scout 5.1.5 → 5.2.1

data/CHANGELOG

@@ -1,3 +1,9 @@
+== 5.2.1
+
+* Added private-key based code signing
+* Added local plugin overrides
+* Added local ad-hoc plugins
+
 == 5.1.5
 
 * Added sleep interval directive. Agent will only sleep when used in non-interactive mode.

data/data/code_id_rsa.pub

@@ -0,0 +1,9 @@
+-----BEGIN PUBLIC KEY-----
+MIIBIDANBgkqhkiG9w0BAQEFAAOCAQ0AMIIBCAKCAQEA7y0b7SAkaqkKvXSQiN6o
+u1/T56dpx8h74p61Gn4i1dwtogqsa/qSf+K7waFMu7XUeMx3z5ZabLu5ZZLONUFA
+SbV4dN3dtz1ECSMuOleXsM+0HzyV8FqLZ1h6XZ65gUhbmTKRUcIfgse0lLWoVKxX
+gtcaLdv/5nrZwZH9g5asGuzqyVx4JwopR/aedFsXSBZyo+FyLQoINrNL2lVllefR
+3EsVDLyZ0etL+K1MtVu1geS41dw4KBCAZQL6eO/88+A9wlJxCcWjXiBK6Gz55ytP
+1w1QrJ25MGVTrE89KGX0wQ9RJ5SLhuFcvRDhNO2T+4H4VZdEjIsIH68YUDNe5FnR
+cwIBIw==
+-----END PUBLIC KEY-----

data/lib/scout.rb

@@ -1,10 +1,11 @@
 #!/usr/bin/env ruby -wKU
 
 module Scout
-  VERSION = "5.1.5".freeze
+  VERSION = "5.2.1".freeze
 end
 
 require "scout/command"
 require "scout/plugin"
 require "scout/plugin_options"
+require "scout/scout_logger"
 require "scout/server"

data/lib/scout/command.rb

@@ -1,7 +1,6 @@
 #!/usr/bin/env ruby -wKU
 
 require "optparse"
-require "logger"
 require "fileutils"
 
 module Scout
@@ -37,6 +36,9 @@ module Scout
         opts.separator "    #{program_name}"
         opts.separator "    ... OR ..."
         opts.separator "    #{program_name} [OPTIONS] install"
+        opts.separator "  Troubleshoot:"
+        opts.separator "    #{program_name} troubleshoot"
+        opts.separator "    ... print Scout environment info. Recommend directing output to a file."
         opts.separator "  Local plugin testing:"
         opts.separator "    #{program_name} [OPTIONS] test " +
                        "PATH_TO_PLUGIN [PLUGIN_OPTIONS]"
@@ -136,17 +138,18 @@ module Scout
       @force   = options[:force]   || false
 
       @args    = args
-    end
-
-    attr_reader :server, :history
 
-    def config_dir
-      return @config_dir if defined? @config_dir
+      # create config dir if necessary
       @config_dir = File.dirname(history)
       FileUtils.mkdir_p(@config_dir) # ensure dir exists
-      @config_dir
+
+      @log_path = File.join(@config_dir, "latest_run.log")
+
     end
 
+    attr_reader :server, :history, :config_dir, :log_path
+
+
     def verbose?
       @verbose
     end
@@ -154,12 +157,15 @@ module Scout
     def log
       return @log if defined? @log
       @log = if verbose?
-               log                 = Logger.new($stdout)
+               log                 = ScoutLogger.new($stdout)
                log.datetime_format = "%Y-%m-%d %H:%M:%S "
                log.level           = level
                log
              else
-               nil
+               log                 = ScoutLogger.new(nil)
+               log.datetime_format = "%Y-%m-%d %H:%M:%S "
+               log.level           = Logger::DEBUG
+               log
              end
     end
 

data/lib/scout/command/run.rb

@@ -30,8 +30,25 @@ module Scout
           create_pid_file_or_exit
           @scout.run_plugins_by_plan
           @scout.save_history
+
+          begin
+            # Since this is a new checkin, overwrite the existing log
+            File.open(log_path, "w") do|log_file|
+              log_file.puts log.messages # log.messages is an array of every message logged during this run
+            end
+          rescue
+            log.info "Could not write to #{log_path}."
+          end
         else
           log.info "Not time to checkin yet. Next checkin in #{@scout.next_checkin}. Override by passing --force to the scout command" if log
+          begin
+            # Since this a ping, append to the existing log
+            File.open(log_path, "a") do|log_file|
+              log_file.puts log.messages
+            end
+          rescue
+            log.info "Could not write to #{log_path}."
+          end
         end
       end
     end

data/lib/scout/command/troubleshoot.rb

@@ -0,0 +1,65 @@
+#!/usr/bin/env ruby -wKU
+
+require "pp"
+
+module Scout
+  class Command
+    class Troubleshoot < Command
+      def initialize(options, args)
+        @contents=[]
+        super
+      end
+
+      def run
+        puts "Gathering troubleshooting information about your Scout install ... "
+
+        heading "Scout Info"
+        bullet "History file", history
+        bullet "Version", Scout::VERSION
+
+        heading "Latest Log"
+        contents=File.read(log_path) rescue "Log not found at #{log_path}"
+        text contents
+
+        heading "Rubygems Environment"
+        text `gem env`
+
+        heading "Ruby info"
+        bullet "Path to executable", `which ruby`
+        bullet "Version", `ruby -v`
+        bullet "Ruby's internal path",  $:.join(', ')
+
+        heading "Installed Gems"
+        text `gem list --local`
+
+        heading "History file Contents"
+        contents=File.read(history) rescue "History not found at #{log_path}"
+        text contents
+
+        heading "Agent directory Contents"
+        text `ls -la #{config_dir}`
+
+        heading ""
+
+        puts "Done"
+
+        puts @contents.join("\n")
+
+      end
+    end
+
+    private
+    def heading(s)
+      @contents += ["",s,"**************************************************************************************************",""]
+    end
+
+    def bullet(label,s)
+      @contents << " - #{label} :  #{s.chomp}"
+    end
+
+    def text(s)
+      @contents << s
+    end
+
+  end
+end

data/lib/scout/plugin.rb

@@ -59,8 +59,7 @@ module Scout
     end
     
     def option(name)
-      @options[name] ||
-      @options[name.is_a?(String) ? name.to_sym : String(name)]
+      @options[name] || @options[name.is_a?(String) ? name.to_sym : String(name)]
     end
 
     # Builds the data to send to the server.

data/lib/scout/scout_logger.rb

@@ -0,0 +1,19 @@
+# We use this subclass of Logger in the Scout Agent so we can retrieve all the logged messages at the end of the run.
+# This works well only because the Agent is not a long-running process.
+
+require 'logger'
+
+class ScoutLogger < Logger
+  attr_reader :messages
+
+  def initialize(*args)
+    @messages=[]
+    super
+  end
+
+  # this is the method that ultimately gets called whenever you invoke info, debug, etc.
+  def add(severity, message=nil, progname = nil, &block)
+    @messages << "[#{Time.now.strftime('%Y-%m-%d %H:%M:%S ')} ##{$$}] : #{progname}"
+    super
+  end
+end

data/lib/scout/server.rb

@@ -7,6 +7,7 @@ require "timeout"
 require "stringio"
 require "zlib"
 require "socket"
+require "base64"
 
 $LOAD_PATH << File.join(File.dirname(__FILE__), *%w[.. .. vendor json_pure lib])
 require "json"
@@ -37,6 +38,7 @@ module Scout
 
     attr_reader :new_plan
     attr_reader :directives
+    attr_reader :plugin_config
 
     # Creates a new Scout Server connection.
     def initialize(server, client_key, history_file, logger = nil)
@@ -48,6 +50,9 @@ module Scout
       @plugin_plan  = []
       @directives   = {} # take_snapshots, interval, sleep_interval
       @new_plan     = false
+      @local_plugin_path = File.dirname(history_file) # just put overrides and ad-hoc plugins in same directory as history file.
+      @plugin_config_path = File.join(@local_plugin_path, "plugins.properties")
+      @plugin_config = load_plugin_configs(@plugin_config_path)
 
       # the block is only passed for install and test, since we split plan retrieval outside the lockfile for run
       if block_given?
@@ -62,12 +67,13 @@ module Scout
     # to execute, along with all options.
     #
     # This method has a couple of side effects:
-    # 1) it sets the @plugin plan with either A) whatever is in history, B) the results of the /plan retrieval
+    # 1) it sets the @plugin_plan with either A) whatever is in history, B) the results of the /plan retrieval
     # 2) it sets @checkin_to = true IF so directed by the scout server
     def fetch_plan
       url = urlify(:plan)
       info "Pinging server at #{url}..."
       headers = {"x-scout-tty" => ($stdin.tty? ? 'true' : 'false')}
+
       if @history["plan_last_modified"] and @history["old_plugins"]
         headers["If-Modified-Since"] = @history["plan_last_modified"]
       end
@@ -75,7 +81,10 @@ module Scout
         if res.is_a? Net::HTTPNotModified
           info "Plan not modified. Will reuse saved plan."
           @plugin_plan = Array(@history["old_plugins"])
+          # Add local plugins to the plan. Note that local plugins are NOT saved to history file
+          @plugin_plan += get_local_plugins
           @directives = @history["directives"] || Hash.new
+
         else
           info "plan has been modified. Will run the new plan now."
           begin
@@ -85,26 +94,80 @@ module Scout
             end
 
             body_as_hash = JSON.parse(body)
-            @plugin_plan = Array(body_as_hash["plugins"])
-            @directives = body_as_hash["directives"].is_a?(Hash) ? body_as_hash["directives"] : Hash.new
 
-            @history["plan_last_modified"] = res["last-modified"]
-            @history["old_plugins"]        = @plugin_plan
-            @history["directives"]         = @directives
+            # Ensure all the plugins in the new plan are properly signed. Load the public key for this.
+            public_key_text = File.read(File.join( File.dirname(__FILE__), *%w[.. .. data code_id_rsa.pub] ))
+            debug "Loaded public key used for verifying code signatures (#{public_key_text.size} bytes)"
+            code_public_key = OpenSSL::PKey::RSA.new(public_key_text)
 
-            info "Plan loaded.  (#{@plugin_plan.size} plugins:  " +
-                 "#{@plugin_plan.map { |p| p['name'] }.join(', ')})" +
-                 ". Directives: #{@directives.to_a.map{|a|  "#{a.first}:#{a.last}"}.join(", ")}"
+            temp_plugins=Array(body_as_hash["plugins"])
+            plugin_signature_error = false
+            temp_plugins.each do |plugin|
+              signature=plugin['signature']
+              id_and_name = "#{plugin['id']}-#{plugin['name']}".sub(/\A-/, "")
+              if signature
+                code=plugin['code'].gsub(/ +$/,'') # we strip trailing whitespace before calculating signatures. Same here.
+                decoded_signature=Base64.decode64(signature)
+                if !code_public_key.verify(OpenSSL::Digest::SHA1.new, decoded_signature, code)
+                  info "#{id_and_name} signature doesn't match!"
+                  plugin_signature_error=true
+                end
+              else
+                info "#{id_and_name} has no signature!"
+                plugin_signature_error=true
+              end
+            end
 
-            @new_plan = true # used in determination if we should checkin this time or not
-          rescue Exception
-            fatal "Plan from server was malformed."
+
+            if(!plugin_signature_error)
+              @plugin_plan = temp_plugins
+              @directives = body_as_hash["directives"].is_a?(Hash) ? body_as_hash["directives"] : Hash.new
+              @history["plan_last_modified"] = res["last-modified"]
+              @history["old_plugins"]        = @plugin_plan.clone # important that the plan is cloned -- we're going to add local plugins, and they shouldn't go into history
+              @history["directives"]         = @directives
+
+              info "Plan loaded.  (#{@plugin_plan.size} plugins:  " +
+                   "#{@plugin_plan.map { |p| p['name'] }.join(', ')})" +
+                   ". Directives: #{@directives.to_a.map{|a|  "#{a.first}:#{a.last}"}.join(", ")}"
+
+              @new_plan = true # used in determination if we should checkin this time or not
+            else
+              info "There was a problem with plugin signatures. Reusing old plan."
+              @plugin_plan = Array(@history["old_plugins"])
+              @directives = @history["directives"] || Hash.new
+            end
+
+            # Add local plugins to the plan. Note that local plugins are NOT saved to history file
+            @plugin_plan += get_local_plugins
+          rescue Exception =>e
+            fatal "Plan from server was malformed: #{e.message} - #{e.backtrace}"
             exit
           end
         end
       end
     end
-    
+
+    # returns an array of hashes representing local plugins found on the filesystem
+    # The glob pattern requires that filenames begin with a letter,
+    # which excludes plugin overrides (like 12345.rb)
+    def get_local_plugins
+      local_plugin_paths=Dir.glob(File.join(@local_plugin_path,"[a-zA-Z]*.rb"))
+      local_plugin_paths.map do |plugin_path|
+        begin
+          {
+            'name' => File.basename(plugin_path),
+            'local_filename' => File.basename(plugin_path),
+            'origin' => 'LOCAL',
+            'code' => File.read(plugin_path),
+            'interval' => 0
+          }
+        rescue => e
+          info "Error trying to read local plugin: #{plugin_path} -- #{e.backtrace.join('\n')}"
+          nil
+        end
+      end.compact
+    end
+
     # To distribute pings across a longer timeframe, the agent will sleep for a given
     # amount of time. When using the --force option the sleep_interval is ignored.
     def sleep_interval
@@ -153,11 +216,12 @@ module Scout
           process_plugin(plugin)
         rescue Exception
           @checkin[:errors] << build_report(
-            plugin['id'],
+            plugin,
             :subject => "Exception:  #{$!.message}.",
             :body    => $!.backtrace
           )
           error("Encountered an error: #{$!.message}")
+          puts $!.backtrace.join('\n')
         end
       end
       take_snapshot if @directives['take_snapshots']
@@ -172,10 +236,12 @@ module Scout
     # It then loads the plugin and runs it with a PLUGIN_TIMEOUT time limit.
     # The plugin generates data, alerts, and errors. In addition, it will
     # set memory and last_run information in the history file.
-    # 
+    #
+    # The plugin argument is a hash with keys: id, name, code, timeout, options, signature.
     def process_plugin(plugin)
       info "Processing the '#{plugin['name']}' plugin:"
       id_and_name = "#{plugin['id']}-#{plugin['name']}".sub(/\A-/, "")
+      plugin_id = plugin['id']
       last_run    = @history["last_runs"][id_and_name] ||
                     @history["last_runs"][plugin['name']]
       memory      = @history["memory"][id_and_name] ||
@@ -186,21 +252,47 @@ module Scout
       if last_run.nil? or delta.between?(-RUN_DELTA, 0) or delta >= 0
         debug "Plugin is past interval and needs to be run.  " +
               "(last run:  #{last_run || 'nil'})"
+        code_to_run = plugin['code']
+        if plugin_id && plugin_id != ""
+          override_path=File.join(@local_plugin_path, "#{plugin_id}.rb")
+          # debug "Checking for local plugin override file at #{override_path}"
+          if File.exist?(override_path)
+            code_to_run = File.read(override_path)
+            debug "Override file found - Using #{code_to_run.size} chars of code in #{override_path} for plugin id=#{plugin_id}"
+            plugin['origin'] = "OVERRIDE"
+          else
+            plugin['origin'] = nil
+          end
+        end
         debug "Compiling plugin..."
         begin
-          eval( plugin['code'],
+          eval( code_to_run,
                 TOPLEVEL_BINDING,
                 plugin['path'] || plugin['name'] )
           info "Plugin compiled."
         rescue Exception
           raise if $!.is_a? SystemExit
           error "Plugin would not compile: #{$!.message}"
-          @checkin[:errors] << build_report(plugin['id'],:subject => "Plugin would not compile", :body=>"#{$!.message}\n\n#{$!.backtrace}")
+          @checkin[:errors] << build_report(plugin,:subject => "Plugin would not compile", :body=>"#{$!.message}\n\n#{$!.backtrace}")
           return
         end
+
+        # Lookup any local options in plugin_config.properies as needed
+        options=(plugin['options'] || Hash.new)
+        options.each_pair do |k,v|
+          if v=~/^lookup:(.+)$/
+            lookup_key = $1.strip
+            if plugin_config[lookup_key]
+              options[k]=plugin_config[lookup_key]
+            else
+              info "Plugin #{id_and_name}: option #{k} appears to be a lookup, but we can't find #{lookup_key} in #{@plugin_config_path}"
+            end
+          end
+        end
+
+
         debug "Loading plugin..."
-        if job = Plugin.last_defined.load( last_run, (memory || Hash.new),
-                                           plugin['options'] || Hash.new )
+        if job = Plugin.last_defined.load( last_run, (memory || Hash.new), options)
           info "Plugin loaded."
           debug "Running plugin..."
           begin
@@ -212,7 +304,7 @@ module Scout
             end
           rescue Timeout::Error, PluginTimeoutError
             error "Plugin took too long to run."
-            @checkin[:errors] << build_report(plugin['id'],
+            @checkin[:errors] << build_report(plugin,
                                               :subject => "Plugin took too long to run",
                                               :body=>"Execution timed out.")
             return
@@ -220,7 +312,7 @@ module Scout
             raise if $!.is_a? SystemExit
             error "Plugin failed to run: #{$!.class}: #{$!.message}\n" +
                   "#{$!.backtrace.join("\n")}"
-            @checkin[:errors] << build_report(plugin['id'],
+            @checkin[:errors] << build_report(plugin,
                                               :subject => "Plugin failed to run",
                                               :body=>"#{$!.class}: #{$!.message}\n#{$!.backtrace.join("\n")}")
           end
@@ -234,7 +326,7 @@ module Scout
               reports << report
             end
             reports.each do |fields|
-              @checkin[plural] << build_report(plugin['id'], fields)
+              @checkin[plural] << build_report(plugin, fields)
             end
           end
           
@@ -244,7 +336,7 @@ module Scout
           @history["memory"][id_and_name]    = data[:memory]
         else
           @checkin[:errors] << build_report(
-            plugin['id'],
+            plugin,
             :subject => "Plugin would not load."
           )
         end
@@ -285,7 +377,8 @@ module Scout
                    :alerts    => Array.new,
                    :errors    => Array.new,
                    :summaries => Array.new,
-                   :snapshot  => '' }
+                   :snapshot  => '',
+                   :config_path => File.expand_path(File.dirname(@history_file))}
     end
 
     def show_checkin(printer = :p)
@@ -323,10 +416,13 @@ module Scout
 
     private
     
-    def build_report(plugin_id, fields)
-      { :plugin_id  => plugin_id,
+    def build_report(plugin_hash, fields)
+      { :plugin_id  => plugin_hash['id'],
         :created_at => Time.now.utc.strftime("%Y-%m-%d %H:%M:%S"),
-        :fields     => fields }
+        :fields     => fields,
+        :local_filename => plugin_hash['local_filename'], # this will be nil unless it's an ad-hoc plugin
+        :origin => plugin_hash['origin'] # [LOCAL|OVERRIDE|nil]
+      }
     end
 
     def urlify(url_name, options = Hash.new)
@@ -422,5 +518,31 @@ module Scout
         super
       end
     end
+
+    private
+
+    # Called during initialization; loads the plugin_configs (local plugin configurations for passwords, etc)
+    # if the file is there. Returns a hash like {"db.username"=>"secr3t"}
+    def load_plugin_configs(path)
+      temp_configs={}
+      if File.exist?(path)
+        debug "Loading Plugin Configs at #{path}"
+        begin
+          File.open(path,"r").read.each_line do |line|
+            line.strip!
+            next if line[0] == '#'
+            next unless line.include? "="
+            k,v =line.split('=')
+            temp_configs[k]=v
+          end
+          debug("#{temp_configs.size} plugin config(s) loaded.")
+        rescue
+          info "Error loading Plugin Configs at #{path}: #{$!}"
+        end
+      else
+        debug "No Plugin Configs at #{path}"
+      end
+      return temp_configs
+    end
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: scout
 version: !ruby/object:Gem::Version 
-  hash: 57
-  prerelease: 
+  hash: 61
+  prerelease: false
   segments: 
   - 5
+  - 2
   - 1
-  - 5
-  version: 5.1.5
+  version: 5.2.1
 platform: ruby
 authors: 
 - Scout Monitoring
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-02-01 00:00:00 -08:00
+date: 2011-02-28 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -52,12 +52,15 @@ files:
 - lib/scout/command/install.rb
 - lib/scout/command/run.rb
 - lib/scout/command/test.rb
+- lib/scout/command/troubleshoot.rb
 - lib/scout/command.rb
 - lib/scout/plugin.rb
 - lib/scout/plugin_options.rb
+- lib/scout/scout_logger.rb
 - lib/scout/server.rb
 - lib/scout.rb
 - data/cacert.pem
+- data/code_id_rsa.pub
 - data/gpl-2.0.txt
 - data/lgpl-2.1.txt
 - vendor/json_pure/benchmarks/data-p4-3GHz-ruby18/GeneratorBenchmarkComparison.log
@@ -218,7 +221,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: scout
-rubygems_version: 1.4.2
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Scout makes monitoring and reporting on your web applications as flexible and simple as possible.