scout-essentials 1.8.7 → 1.9.0

data/lib/scout/cmd.rb

@@ -1,12 +1,33 @@
 require_relative 'indiferent_hash'
 require_relative 'concurrent_stream'
 require_relative 'log'
+require_relative 'exceptions'
 require_relative 'open/stream'
 require 'stringio'
 require 'open3'
+require 'fileutils'
 
 module CMD
 
+  # Grace period, in seconds, between sending :INT and escalating to :KILL
+  # when a command exceeds its :timeout option
+  TIMEOUT_KILL_GRACE = 1.0
+
+  # ScoutCoder: exception raised by CMD.cmd when a command exceeds its
+  # :timeout option. It subclasses ProcessFailed, so generic rescue clauses
+  # for ProcessFailed also catch it. In pipe mode the stream is aborted with
+  # this exception (ConcurrentStream#abort), so consumers observe it through
+  # ConcurrentStream#stream_raise_exception when reading or joining the
+  # returned stream.
+  class Timeout < ProcessFailed
+    attr_reader :command, :timeout
+    def initialize(pid = Process.pid, command = nil, timeout = nil)
+      @command = command
+      @timeout = timeout
+      super(pid, "command '#{command}' exceeded timeout of #{timeout} seconds")
+    end
+  end
+
   TOOLS = IndiferentHash.setup({})
   def self.tool(tool, claim = nil, test = nil, cmd = nil, &block)
     TOOLS[tool] = [claim, test, block, cmd]
@@ -124,6 +145,36 @@ module CMD
     string.strip
   end
 
+  # ScoutCoder: process_cmd_options_array mirrors process_cmd_options but
+  # returns an array of separate argument strings instead of a single shell
+  # string, suitable for the no-shell array form of Open3.popen3
+  def self.process_cmd_options_array(options = {})
+    add_dashes = IndiferentHash.process_options options, :add_option_dashes
+
+    result = []
+    options.each do |option, value|
+      raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=.]+$/i
+
+      option = "--" << option.to_s if add_dashes and option.to_s[0] != '-'
+
+      case
+      when value.nil? || FalseClass === value
+        next
+      when TrueClass === value
+        result << option.to_s
+      else
+        if option.to_s.chars.to_a.last == "="
+          result << "#{option}#{value}"
+        else
+          result << option.to_s
+          result << value.to_s
+        end
+      end
+    end
+
+    result
+  end
+
   def self.cmd(tool, cmd = nil, options = {}, &block)
     options, cmd = cmd, nil if Hash === cmd
 
@@ -140,65 +191,226 @@ module CMD
     xvfb       = options.delete(:xvfb)
     bar        = options.delete(:progress_bar)
     save_stderr = options.delete(:save_stderr)
+
+    # :save_stderr also accepts a file path (String, Scout Path or Pathname)
+    # or an IO-like object (anything that responds to :write).  A path is
+    # opened here for writing (truncating, creating missing parent
+    # directories) and CMD owns it: it is closed when the command ends.  An
+    # IO-like object is used as-is and never closed by CMD.  In addition to
+    # writing the destination, std_err keeps accumulating (String and Path are
+    # truthy, so the existing `if save_stderr` behaviour covers this)
+    save_stderr_dst = nil
+    save_stderr_close = false
+    if save_stderr && ! (TrueClass === save_stderr || FalseClass === save_stderr)
+      if String === save_stderr || Pathname === save_stderr
+        path = save_stderr
+        path = path.find if Path === path
+        path = path.to_s unless String === path
+        dir = File.dirname(path)
+        FileUtils.mkdir_p dir unless File.exist?(dir)
+        save_stderr_dst = File.open(path, 'w')
+        save_stderr_close = true
+      elsif save_stderr.respond_to?(:write) || save_stderr.respond_to?(:<<)
+        save_stderr_dst = save_stderr
+      end
+    end
+
     autojoin   = options.delete(:autojoin)
     autojoin   = no_wait if autojoin.nil?
+    timeout    = options.delete(:timeout)
 
     dont_close_in  = options.delete(:dont_close_in)
 
     log = true if log.nil?
 
-    if cmd.nil? && ! Symbol === tool
-      cmd = tool
+    array_mode = Array === tool
+
+    if array_mode
+      cmd_array = tool.dup
+      cmd_array << cmd if cmd.is_a?(String) && !cmd.empty?
+
+      case xvfb
+      when TrueClass
+        cmd_array = ["xvfb-run", "--server-args=-screen 0 1024x768x24", "--auto-servernum"] + cmd_array
+      when String
+        cmd_array = ["xvfb-run", "--server-args=#{xvfb}", "--auto-servernum", "--server-num=1"] + cmd_array
+      end
+
+      if stderr == true
+        stderr = Log::HIGH
+      end
+
+      cmd_array += process_cmd_options_array options
+
+      cmd_array = ["sudo"] + cmd_array if sudo
+
+      # Build cmd string for logging/error messages
+      cmd = cmd_array.map { |e| e.to_s.include?(' ') ? "'#{e}'" : e.to_s }.join(' ')
     else
-      tool = get_tool(tool)
-      if cmd.nil?
+      if cmd.nil? && ! Symbol === tool
         cmd = tool
       else
-        cmd = tool + ' ' + cmd
-      end
+        tool = get_tool(tool)
+        if cmd.nil?
+          cmd = tool
+        else
+          cmd = tool + ' ' + cmd
+        end
 
-    end
+      end
 
-    case xvfb
-    when TrueClass
-      cmd = "xvfb-run --server-args='-screen 0 1024x768x24' --auto-servernum #{cmd}"
-    when String
-      cmd = "xvfb-run --server-args='#{xvfb}' --auto-servernum --server-num=1 #{cmd}"
-    when String
-    end
+      case xvfb
+      when TrueClass
+        cmd = "xvfb-run --server-args='-screen 0 1024x768x24' --auto-servernum #{cmd}"
+      when String
+        cmd = "xvfb-run --server-args='#{xvfb}' --auto-servernum --server-num=1 #{cmd}"
+      when String
+      end
 
-    if stderr == true
-      stderr = Log::HIGH
-    end
+      if stderr == true
+        stderr = Log::HIGH
+      end
 
-    cmd_options = process_cmd_options options
-    if cmd =~ /'\{opt\}'/
-      cmd = cmd.sub('\'{opt}\'', cmd_options)
-    else
-      cmd += " " + cmd_options
-    end
+      cmd_options = process_cmd_options options
+      if cmd =~ /'\{opt\}'/
+        cmd = cmd.sub('\'{opt}\'', cmd_options)
+      else
+        cmd += " " + cmd_options
+      end
 
-    if sudo
-      cmd = "sudo " + cmd
+      if sudo
+        cmd = "sudo " + cmd
+      end
     end
 
     in_content = StringIO.new in_content if String === in_content
 
     sin, sout, serr, wait_thr = begin
-                                  Open3.popen3(ENV, cmd)
+                                  if array_mode
+                                    Open3.popen3(ENV, *cmd_array)
+                                  else
+                                    Open3.popen3(ENV, cmd)
+                                  end
                                 rescue
                                   Log.warn $!.message
+                                  # The command never started; do not leak the
+                                  # file opened for it above
+                                  save_stderr_dst.close if save_stderr_dst && save_stderr_close
                                   raise ProcessFailed, nil, cmd unless no_fail
                                   return
                                 end
     pid = wait_thr.pid
 
-    Log.debug{"CMD: [#{pid}] #{cmd}".strip if log}
+    if Log.severity >= 2
+      Log.medium("CMD [#{pid}] #{Log.fingerprint(cmd)}")
+    else
+      Log.debug("CMD [#{pid}] #{cmd.strip}")
+    end
+
+    # ScoutCoder: :timeout support.  A watchdog thread starts a counter
+    # (deadline) for the command; if the process is still running when the
+    # timeout is reached it kills it and raises CMD::Timeout.
+    #
+    # In pipe mode the exception is routed through ConcurrentStream#abort:
+    # it is stored in stream_exception, the stream is aborted (which
+    # interrupts any consumer blocked reading from it, kills the process and
+    # clears the pid list) and it is re-raised when the consumer reads or
+    # joins the stream.  In non-pipe mode the watchdog raises directly in the
+    # thread running CMD.cmd, unblocking its read, and the internal stream is
+    # aborted in the rescue below.
+    #
+    # The watchdog is deliberately NOT registered in sout.threads, so it never
+    # delays a normal join, and it always ends up reaping the pid (see
+    # reap_timeout_pid below) to avoid leaving a zombie behind when the
+    # Process::Waiter thread is interrupted by the abort.
+    timeout_thread = nil
+    timeout_exception = nil
+    timed_out = false
+    if Numeric === timeout && timeout > 0
+      timeout_exception = Timeout.new(pid, cmd, timeout)
+      caller_thread = Thread.current
+
+      # The helper threads are interrupted with the timeout exception when
+      # the command is killed; do not report that as a thread crash
+      wait_thr.report_on_exception = false
+
+      # Kill the process: :INT first, escalate to :KILL after a grace period,
+      # and reap it so that no zombie is left behind.  Works whether or not
+      # the Process::Waiter thread is still alive: whoever waits first reaps
+      # the pid and the other one gets Errno::ECHILD.
+      reap_timeout_pid = Proc.new do
+        begin
+          Process.kill(:INT, pid)
+        rescue Errno::ESRCH
+        end
+
+        grace_end = Time.now + TIMEOUT_KILL_GRACE
+        reaped = false
+        while Time.now < grace_end
+          begin
+            reaped = ! Process.waitpid(pid, Process::WNOHANG).nil?
+            break if reaped
+          rescue Errno::ECHILD
+            reaped = true
+            break
+          end
+          sleep 0.05
+        end
+
+        unless reaped
+          begin
+            Process.kill(:KILL, pid)
+          rescue Errno::ESRCH
+          end
+          begin
+            Process.waitpid(pid)
+          rescue Errno::ECHILD
+          end
+        end
+      end
+
+      timeout_thread = Thread.new do
+        Thread.current["name"] = "CMD timeout: [#{pid}] #{cmd}"
+        Thread.current.report_on_exception = false
+
+        begin
+          # Counter: deadline of the command, measured from spawn time.  The
+          # extra join(0) discards the boundary race in which the process
+          # finished exactly at the deadline
+          if wait_thr.join(timeout).nil? && wait_thr.join(0).nil?
+            timed_out = true
+            Log.low "CMD: [#{pid}] #{cmd} timed out after #{timeout} seconds. Killing"
+
+            if pipe
+              # Store the exception and abort the stream BEFORE the pid is
+              # gone, so that consumers blocked on a read are interrupted and
+              # get the exception instead of a plain EOF.  abort (rather than
+              # stream_raise_exception) is used so the helper threads are
+              # interrupted with the timeout exception itself: if they died
+              # with any other exception, joining them would overwrite
+              # stream_exception and hide the timeout
+              sout.abort timeout_exception if sout.respond_to?(:abort) && ! sout.joined?
+              reap_timeout_pid.call
+            else
+              # Unblock the thread running CMD.cmd immediately; the kill and
+              # the clean-up of the internal stream happen in its rescue path
+              # and in reap_timeout_pid below
+              caller_thread.raise timeout_exception
+              reap_timeout_pid.call
+            end
+          end
+        rescue Exception
+          # Never let the watchdog crash the process
+          Log.exception $!
+        end
+      end
+      Thread.pass until timeout_thread["name"]
+    end
 
     if in_content.respond_to?(:read)
       in_thread = Thread.new(Thread.current) do |parent|
         begin
-          Thread.current.report_on_exception = false if no_fail
+          Thread.current.report_on_exception = false if no_fail || timeout_exception
           Thread.current["name"] = "CMD in"
           while c = in_content.read(Open::BLOCK_SIZE)
             sin << c
@@ -213,7 +425,9 @@ module CMD
             in_content.join if in_content.respond_to? :join
           end
         rescue Exception
-          Log.error "Error in CMD  [#{pid}] #{cmd}: #{$!.message}" unless no_fail
+          unless no_fail || Aborted === $! || Timeout === $!
+            Log.error "Error in CMD  [#{pid}] #{cmd}: #{$!.message}"
+          end
           sin.close  unless sin.closed?
           sin.join if sin.respond_to? :join
           raise $!
@@ -233,20 +447,45 @@ module CMD
 
       sout.callback = post if post
 
-      if (Integer === stderr and log) || bar
+      if (Integer === stderr and log) || bar || save_stderr_dst
         err_thread = Thread.new do
           Thread.current["name"] = "Error log: [#{pid}] #{ cmd }"
+          Thread.current.report_on_exception = false
           begin
             while line = serr.gets
               bar.process(line) if bar
               sout.log = line
               sout.std_err << line if save_stderr
+              # Write each line to the destination as soon as it arrives and
+              # flush, so that a `tail -f` on the file follows the progress
+              # of the command while the pipe is still being consumed
+              if save_stderr_dst
+                save_stderr_dst << line
+                save_stderr_dst.flush if save_stderr_dst.respond_to?(:flush)
+              end
               Log.log "STDERR [#{pid}]: " +  line, stderr if log
             end
             serr.close
+          rescue Aborted
+            # The stream was aborted (e.g. by a :timeout): stop silently, the
+            # consumer of sout gets the real exception from the stream.  The
+            # destination is flushed and closed, whatever was written so far
+            # is kept in place (the log file is not removed)
+            serr.close unless serr.closed?
           rescue
-            Log.exception $!
+            # When the stream is being aborted (e.g. by a :timeout) the
+            # consumer gets the real exception from the stream itself
+            Log.exception $! unless sout.aborted?
             raise $!
+          ensure
+            # No fd leak: CMD closes what it opened (a caller provided IO is
+            # only flushed, closing is up to the caller)
+            begin
+              save_stderr_dst.flush if save_stderr_dst && save_stderr_dst.respond_to?(:flush) && ! save_stderr_dst.closed?
+            rescue Exception
+              Log.exception $!
+            end
+            save_stderr_dst.close if save_stderr_dst && save_stderr_close && ! save_stderr_dst.closed?
           end
         end
       else
@@ -260,19 +499,45 @@ module CMD
       err = ""
       if bar
         err_thread = Thread.new do
-          while not serr.eof?
-            line = serr.gets
-            bar.process(line)
-            err << line if Integer === stderr and log
+          Thread.current.report_on_exception = false
+          begin
+            while not serr.eof?
+              line = serr.gets
+              bar.process(line)
+              err << line if save_stderr || (Integer === stderr and log)
+            end
+            serr.close
+          rescue Exception
+            # Interrupted while the stream is aborted (e.g. by a :timeout);
+            # the CMD.cmd thread surfaces the real exception
+            serr.close unless serr.closed?
           end
-          serr.close
         end
       elsif log and Integer === stderr
         err_thread = Thread.new do
-          while not serr.eof?
-            err += serr.gets
+          Thread.current.report_on_exception = false
+          begin
+            while not serr.eof?
+              err += serr.gets
+            end
+            serr.close
+          rescue Exception
+            serr.close unless serr.closed?
+          end
+        end
+      elsif save_stderr_dst
+        # Without log level and bar stderr would be discarded; accumulate it
+        # so the destination never ends up silently empty
+        err_thread = Thread.new do
+          Thread.current.report_on_exception = false
+          begin
+            while not serr.eof?
+              err += serr.gets
+            end
+            serr.close
+          rescue Exception
+            serr.close unless serr.closed?
           end
-          serr.close
         end
       else
         Open.consume_stream(serr, true)
@@ -285,11 +550,35 @@ module CMD
         out = StringIO.new sout.read
         status = wait_thr.value
 
+        # Settle the watchdog decision: if the process finished right at the
+        # deadline the watchdog may still be deciding, so wait for it before
+        # reporting success.  A real timeout raises directly in this thread
+        # and is handled by the rescue below.
+        timeout_thread.join if timeout_thread
+
+        # A timeout was detected while waiting; the exception wins over any
+        # exit-status error of the killed process
+        raise timeout_exception if timed_out
+
         sout.join
         sout.close unless sout.closed?
         sout.annotate(out)
 
         out.exit_status = status.exitstatus
+
+        # Save stderr before reporting a failure: the ProcessFailed message
+        # embeds it and the destination must keep it, even when the command
+        # fails (sout.join above already filled err completely)
+        out.std_err = err if save_stderr
+        if save_stderr_dst
+          begin
+            save_stderr_dst << err unless err.empty?
+            save_stderr_dst.flush if save_stderr_dst.respond_to?(:flush) && ! save_stderr_dst.closed?
+          rescue Exception
+            Log.exception $!
+          end
+        end
+
         if status && ! status.success? && ! no_fail
           if !err.empty?
             raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}.\n#{err}"
@@ -299,9 +588,23 @@ module CMD
         else
           Log.log err, stderr if Integer === stderr and log
         end
-        out.std_err = err if save_stderr
         out
+      rescue Timeout
+        # Abort the internal stream so pipes are closed, the input and error
+        # threads are interrupted and the pid list is cleared
+        sout.abort($!) unless sout.aborted?
+        raise $!
       ensure
+        # No fd leak in the non-pipe path either: flush what CMD opened and
+        # close it (a caller provided IO stays open, closing is up to it)
+        if save_stderr_dst
+          begin
+            save_stderr_dst.flush if save_stderr_dst.respond_to?(:flush) && ! save_stderr_dst.closed?
+          rescue Exception
+            Log.exception $!
+          end
+          save_stderr_dst.close if save_stderr_close && ! save_stderr_dst.closed?
+        end
         post.call if post
       end
     end

data/lib/scout/concurrent_stream.rb

@@ -93,7 +93,14 @@ module ConcurrentStream
             if ! (t.value.success? || no_fail)
 
               if log
-                msg = "Error joining #{self.filename || self.inspect}. Last log line: #{log}"
+                if std_err && log.length < 10 && !(log.downcase.include?("error") || log.downcase.include?('exception'))
+                  exception_line = std_err.split("\n").reverse.find{|line| line.downcase.include?("error") || line.downcase.include?('exception') || line.include?('::') }
+                end
+                if exception_line
+                  msg = "Error joining #{self.filename || self.inspect}. Exception line: #{exception_line}"
+                else
+                  msg = "Error joining #{self.filename || self.inspect}. Last log line: #{log}"
+                end
               else
                 msg = "Error joining #{self.filename || self.inspect}"
               end
@@ -164,6 +171,7 @@ module ConcurrentStream
 
     threads = @threads.dup
     @threads.clear
+    aborted_threads = []
     threads.each do |t|
       next if t == Thread.current
       next if t["aborted"]
@@ -171,6 +179,10 @@ module ConcurrentStream
       exception = exception.nil? ? Aborted.new : exception
       Log.debug "Aborting thread #{Log.fingerprint(t)} with exception: #{exception}"
       t.raise(exception)
+      aborted_threads << t
+    end
+
+    aborted_threads.each do |t|
       t.join
     end
   end
@@ -242,6 +254,7 @@ module ConcurrentStream
       super(*args)
     rescue Exception
       @stream_exception ||= $!
+      self.abort
       raise @stream_exception
     ensure
       if ! @stream_exception && autojoin && ! closed?

data/lib/scout/indiferent_hash.rb

@@ -137,7 +137,7 @@ module IndiferentHash
     new
   end
 
-  def prety_print
+  def pretty_print
     Misc.format_definition_list(self, sep: "\n")
   end
 

data/lib/scout/log/fingerprint.rb

@@ -3,6 +3,18 @@ module Log
   FP_MAX_STRING = 150
   FP_MAX_ARRAY = 20
   FP_MAX_HASH = 10
+
+  def self.truncate_string(string, max=FP_MAX_STRING)
+    if string.length > max
+      digest = Digest::MD5.hexdigest(string)
+      middle = "<...#{string.length} - #{digest[0..4]}...>"
+      s = (max - middle.length) / 2
+      string.slice(0,s-1) + middle + string.slice(-s, string.length )
+    else 
+      string
+    end
+  end
+
   def self.fingerprint(obj)
     return obj.fingerprint if obj.respond_to?(:fingerprint)
 
@@ -17,14 +29,7 @@ module Log
       ":" + obj.to_s
     when String
       obj = obj.gsub("\n", '\n')
-      if obj.length > FP_MAX_STRING
-        digest = Digest::MD5.hexdigest(obj)
-        middle = "<...#{obj.length} - #{digest[0..4]}...>"
-        s = (FP_MAX_STRING - middle.length) / 2
-        "'" + obj.slice(0,s-1) + middle + obj.slice(-s, obj.length ) + "'"
-      else 
-        "'" + obj + "'"
-      end
+      "'" + Log.truncate_string(obj) + "'"
     when ConcurrentStream
       name = obj.inspect + " " + obj.object_id.to_s
       name += " #{obj.filename}" if obj.filename

data/lib/scout/log/progress/report.rb

@@ -56,7 +56,7 @@ module Log
         thr = short_mean
       else
         thr = begin
-                d = Time.now - @start
+                d = Time.now - (@start || Time.now)
                 if d == 0
                   1
                 else

data/lib/scout/log.rb

@@ -359,7 +359,10 @@ def ppp(message)
   stack = caller
   puts "#{Log.color :cyan, "PRINT:"} " << stack.first
   puts ""
-  message = message.prety_print if message.respond_to?(:prety_print)
+  begin
+    message = message.pretty_print if message.respond_to?(:pretty_print)
+  rescue
+  end
   if message.length > 200 or message.include? "\n"
     puts Log.color(:cyan, "=>|") << "\n" << message.to_s
   else

data/lib/scout/misc/digest.rb

@@ -7,11 +7,12 @@ module Misc
     else
       case obj
       when String
-        if Path.is_filename?(obj) && Open.exists?(obj)
-          Path.setup(obj).digest_str
-        else
-          obj.dup
-        end
+          if Path.is_filename?(obj) && Open.exists?(obj)
+            @@digest_str_cache ||= {}
+            @@digest_str_cache[obj] ||= Path.setup(obj).digest_str
+          else
+            obj.dup
+          end
       when Integer, Symbol
         obj.to_s
       when Array

data/lib/scout/misc/format.rb

@@ -212,6 +212,30 @@ module Misc
     end
   end
 
+  def self.human_number(n)
+    units = ["", "K", "M", "B", "T"]
+
+    sign = n < 0 ? "-" : ""
+    value = n.abs.to_f
+    unit = 0
+
+    while value >= 999.5 && unit < units.size - 1
+      value /= 1000.0
+      unit += 1
+    end
+
+    str =
+      if value >= 100
+        value.round.to_s
+      elsif value >= 10
+        value.round.to_s
+      else
+        ("%.1f" % value).sub(/\.0$/, "")
+      end
+
+    "#{sign}#{str}#{units[unit]}"
+  end
+
   def self.parse_sql_values(txt)
     io = StringIO.new txt.strip
 

data/lib/scout/named_array.rb

@@ -159,7 +159,7 @@ module NamedArray
     end
   end
 
-  def prety_print
+  def pretty_print
     Misc.format_definition_list(self.to_hash, sep: "\n")
   end
 end

data/lib/scout/open/stream.rb

@@ -121,8 +121,8 @@ module Open
           when (IO === content or StringIO === content or File === content)
             Open.write(tmp_path) do |f|
               while block = content.read(BLOCK_SIZE)
-                f.write block
-                break if content.closed?
+                f.write block if block
+                break if content.closed? || content.eof?
               end
             end
           else