scout 5.3.5 → 5.4.4.alpha

data/lib/scout/plugin.rb

@@ -49,6 +49,16 @@ module Scout
         code =~ EMBEDDED_OPTIONS_REGEX
         return $2
       end
+
+      def extract_code_class(code)
+        match = /class\s\b(\w*)\s+?<\s+Scout::Plugin/.match(code)
+
+        if match
+          return match[1]
+        else
+          raise ArgumentError, "can't identify plugin class"
+        end
+      end
     end
 
     # Creates a new Scout Plugin to run.
@@ -122,7 +132,16 @@ module Scout
         alias_method :add_#{kind}, :#{kind}
       END
     end
-    
+
+    # resets everything except memory. Memory stays intact. This is used for real-time reporting
+    def reset!
+      @data_for_server = { :reports   => [ ],
+                           :alerts    => [ ],
+                           :errors    => [ ],
+                           :summaries => [ ],
+                           :memory  => @memory }
+    end
+
     #
     # Usage:
     # 

data/lib/scout/server.rb

@@ -1,29 +1,13 @@
-#!/usr/bin/env ruby -wKU
-
-require "net/https"
-require "uri"
-require "yaml"
-require "timeout"
-require "stringio"
-require "zlib"
-require "socket"
-require "base64"
-
-$LOAD_PATH << File.join(File.dirname(__FILE__), *%w[.. .. vendor json_pure lib])
+
+Dir.glob(File.join(File.dirname(__FILE__), *%w[.. .. vendor *])).each do |dir|
+  $LOAD_PATH << File.join(dir,"lib")
+end
+
 require "json"
+require "pusher"
 
 module Scout
-  class Server
-    # A new class for plugin Timeout errors.
-    class PluginTimeoutError < RuntimeError; end
-    # A new class for API Timeout errors.
-    class APITimeoutError < RuntimeError; end
-    
-    # Headers passed up with all API requests.
-    HTTP_HEADERS = { "Client-Version"  => Scout::VERSION,
-                     "Client-Hostname" => Socket.gethostname,
-                     "Accept-Encoding" => "gzip" }
-    
+  class Server < Scout::ServerBase
     # 
     # A plugin cannot take more than DEFAULT_PLUGIN_TIMEOUT seconds to execute, 
     # otherwise, a timeout error is generated.  This can be overriden by
@@ -39,20 +23,20 @@ module Scout
     attr_reader :new_plan
     attr_reader :directives
     attr_reader :plugin_config
+    attr_reader :streamer_command
 
     # Creates a new Scout Server connection.
-    def initialize(server, client_key, history_file, logger = nil, server_name=nil, http_proxy='', https_proxy='')
+    def initialize(server, client_key, history_file, logger = nil, server_name=nil)
       @server       = server
       @client_key   = client_key
       @history_file = history_file
       @history      = Hash.new
       @logger       = logger
       @server_name  = server_name
-      @http_proxy   = http_proxy
-      @https_proxy   = https_proxy
       @plugin_plan  = []
       @plugins_with_signature_errors = []
       @directives   = {} # take_snapshots, interval, sleep_interval
+      @streamer_command = nil
       @new_plan     = false
       @local_plugin_path = File.dirname(history_file) # just put overrides and ad-hoc plugins in same directory as history file.
       @plugin_config_path = File.join(@local_plugin_path, "plugins.properties")
@@ -68,6 +52,7 @@ module Scout
     end
 
     def refresh?
+      #info "called refresh: ping_key=#{ping_key}"
       return true if !ping_key or account_public_key_changed? # fetch the plan again if the account key is modified/created
 
       url=URI.join( @server.sub("https://","http://"), "/clients/#{ping_key}/ping.scout")
@@ -77,6 +62,8 @@ module Scout
         headers["If-Modified-Since"] = @history["plan_last_modified"]
       end
       get(url, "Could not ping #{url} for refresh info", headers) do |res|
+        info "inside 'refresh?' #{res.to_hash.to_json}"
+        @streamer_command = res["x-streamer-command"] # usually will be nil, but can be [start,abcd,1234,5678|stop]
         if res.is_a?(Net::HTTPNotModified)
           return false
         else
@@ -107,7 +94,6 @@ module Scout
             if res["Content-Encoding"] == "gzip" and body and not body.empty?
               body = Zlib::GzipReader.new(StringIO.new(body)).read
             end
-      
             body_as_hash = JSON.parse(body)
             
             temp_plugins=Array(body_as_hash["plugins"])
@@ -152,7 +138,6 @@ module Scout
 
             @new_plan = true # used in determination if we should checkin this time or not
 
-
             # Add local plugins to the plan.
             @plugin_plan += get_local_plugins
           rescue Exception =>e
@@ -165,6 +150,7 @@ module Scout
         @plugin_plan = Array(@history["old_plugins"])
         @plugin_plan += get_local_plugins
         @directives = @history["directives"] || Hash.new
+
       end
       @plugin_plan.reject! { |p| p['code'].nil? }
     end
@@ -245,7 +231,7 @@ module Scout
     def time_to_checkin?
       @history['last_checkin'] == nil ||
               @directives['interval'] == nil ||
-              (Time.now.to_i - Time.at(@history['last_checkin']).to_i).abs+15+sleep_interval > @directives['interval'].to_i*60
+              (Time.now.to_i - Time.at(@history['last_checkin']).to_i).abs+15 > @directives['interval'].to_i*60
     rescue
       debug "Failed to calculate time_to_checkin. @history['last_checkin']=#{@history['last_checkin']}. "+
               "@directives['interval']=#{@directives['interval']}. Time.now.to_i=#{Time.now.to_i}"
@@ -495,7 +481,7 @@ module Scout
       contents=File.read(@history_file)
       begin
         @history = YAML.load(contents)
-      rescue
+      rescue => e
         backup_path=File.join(File.dirname(@history_file), "history.corrupt")
         info "Couldn't parse the history file. Deleting it and resetting to an empty history file. Keeping a backup at #{backup_path}"
         File.open(backup_path,"w"){|f|f.write contents}
@@ -540,70 +526,6 @@ module Scout
       }
     end
 
-    def urlify(url_name, options = Hash.new)
-      return unless @server
-      options.merge!(:client_version => Scout::VERSION)
-      URI.join( @server,
-                "/clients/CLIENT_KEY/#{url_name}.scout".
-                  gsub(/\bCLIENT_KEY\b/, @client_key).
-                  gsub(/\b[A-Z_]+\b/) { |k| options[k.downcase.to_sym] || k } )
-    end
-    
-    def post(url, error, body, headers = Hash.new, &response_handler)
-      return unless url
-      request(url, response_handler, error) do |connection|
-        post = Net::HTTP::Post.new( url.path +
-                                    (url.query ? ('?' + url.query) : ''),
-                                    HTTP_HEADERS.merge(headers) )
-        post.body = body
-        connection.request(post)
-      end
-    end
-
-    def get(url, error, headers = Hash.new, &response_handler)
-      return unless url
-      request(url, response_handler, error) do |connection|
-        connection.get( url.path + (url.query ? ('?' + url.query) : ''),
-                        HTTP_HEADERS.merge(headers) )
-      end
-    end
-
-    def request(url, response_handler, error, &connector)
-      response           = nil
-      Timeout.timeout(5 * 60, APITimeoutError) do
-
-        # take care of http/https proxy, if specified in command line options
-        # Given a blank string, the proxy_uri URI instance's host/port/user/pass will be nil
-        # Net::HTTP::Proxy returns a regular Net::HTTP class if the first argument (host) is nil
-        proxy_uri = URI.parse(url.is_a?(URI::HTTPS) ? @https_proxy : @http_proxy)
-        http=Net::HTTP::Proxy(proxy_uri.host,proxy_uri.port,proxy_uri.user,proxy_uri.port).new(url.host, url.port)
-
-        if url.is_a? URI::HTTPS
-          http.use_ssl     = true
-          http.ca_file     = File.join( File.dirname(__FILE__),
-                                        *%w[.. .. data cacert.pem] )
-          http.verify_mode = OpenSSL::SSL::VERIFY_PEER |
-                             OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
-        end
-        response           = no_warnings { http.start(&connector) }
-      end
-      case response
-      when Net::HTTPSuccess, Net::HTTPNotModified
-        response_handler[response] unless response_handler.nil?
-      else
-        error = "Server says: #{response['x-scout-msg']}" if response['x-scout-msg']
-        fatal error
-        raise SystemExit.new(error)
-      end
-    rescue Timeout::Error
-      fatal "Request timed out."
-      exit
-    rescue Exception
-      raise if $!.is_a? SystemExit
-      fatal "An HTTP error occurred:  #{$!.message}"
-      exit
-    end
-    
     def checkin
       debug """
 #{PP.pp(@checkin, '')}
@@ -626,23 +548,6 @@ module Scout
     end
     
     
-    def no_warnings
-      old_verbose = $VERBOSE
-      $VERBOSE    = false
-      yield
-    ensure
-      $VERBOSE = old_verbose
-    end
-    
-    # Forward Logger methods to an active instance, when there is one.
-    def method_missing(meth, *args, &block)
-      if (Logger::SEV_LABEL - %w[ANY]).include? meth.to_s.upcase
-        @logger.send(meth, *args, &block) unless @logger.nil?
-      else
-        super
-      end
-    end
-
     private
 
     # Called during initialization; loads the plugin_configs (local plugin configurations for passwords, etc)

data/lib/scout/server_base.rb

@@ -0,0 +1,100 @@
+require "net/https"
+require "uri"
+require "yaml"
+require "timeout"
+require "stringio"
+require "zlib"
+require "socket"
+require "base64"
+
+module Scout
+  class ServerBase
+    # A new class for plugin Timeout errors.
+    class PluginTimeoutError < RuntimeError; end
+    # A new class for API Timeout errors.
+    class APITimeoutError < RuntimeError; end
+
+    # Headers passed up with all API requests.
+    HTTP_HEADERS = { "Client-Version"  => Scout::VERSION,
+                     "Client-Hostname" => Socket.gethostname,
+                     "Accept-Encoding" => "gzip" }
+
+
+    private
+
+    def urlify(url_name, options = Hash.new)
+      return unless @server
+      options.merge!(:client_version => Scout::VERSION)
+      URI.join(@server,
+               "/clients/CLIENT_KEY/#{url_name}.scout".
+                   gsub(/\bCLIENT_KEY\b/, @client_key).
+                   gsub(/\b[A-Z_]+\b/) { |k| options[k.downcase.to_sym] || k })
+    end
+
+    def post(url, error, body, headers = Hash.new, &response_handler)
+      return unless url
+      request(url, response_handler, error) do |connection|
+        post = Net::HTTP::Post.new(url.path +
+                                       (url.query ? ('?' + url.query) : ''),
+                                   HTTP_HEADERS.merge(headers))
+        post.body = body
+        connection.request(post)
+      end
+    end
+
+    def get(url, error, headers = Hash.new, &response_handler)
+      return unless url
+      request(url, response_handler, error) do |connection|
+        connection.get(url.path + (url.query ? ('?' + url.query) : ''),
+                       HTTP_HEADERS.merge(headers))
+      end
+    end
+
+    def request(url, response_handler, error, &connector)
+      response = nil
+      Timeout.timeout(5 * 60, APITimeoutError) do
+        http = Net::HTTP.new(url.host, url.port)
+        if url.is_a? URI::HTTPS
+          http.use_ssl = true
+          http.ca_file = File.join(File.dirname(__FILE__),
+                                   *%w[.. .. data cacert.pem])
+          http.verify_mode = OpenSSL::SSL::VERIFY_PEER |
+              OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
+        end
+        response = no_warnings { http.start(&connector) }
+      end
+      case response
+        when Net::HTTPSuccess, Net::HTTPNotModified
+          response_handler[response] unless response_handler.nil?
+        else
+          error = "Server says: #{response['x-scout-msg']}" if response['x-scout-msg']
+          fatal error
+          raise SystemExit.new(error)
+      end
+    rescue Timeout::Error
+      fatal "Request timed out."
+      exit
+    rescue Exception
+      raise if $!.is_a? SystemExit
+      fatal "An HTTP error occurred:  #{$!.message}"
+      exit
+    end
+
+    def no_warnings
+      old_verbose = $VERBOSE
+      $VERBOSE = false
+      yield
+    ensure
+      $VERBOSE = old_verbose
+    end
+
+    # Forward Logger methods to an active instance, when there is one.
+    def method_missing(meth, *args, &block)
+      if (Logger::SEV_LABEL - %w[ANY]).include? meth.to_s.upcase
+        @logger.send(meth, *args, &block) unless @logger.nil?
+      else
+        super
+      end
+    end
+  end
+end

data/lib/scout/streamer.rb

@@ -0,0 +1,162 @@
+require 'rubygems'
+require 'json'
+
+module Scout
+  class Streamer < Scout::ServerBase
+    MAX_DURATION = 60*30 # will shut down automatically after this many seconds
+    SLEEP = 1
+
+    # * history_file is the *path* to the history file
+    # * plugin_ids is an array of integers
+    def initialize(server, client_key, history_file, plugin_ids, streaming_key, logger = nil)
+      @server       = server
+      @client_key   = client_key
+      @history_file = history_file
+      @history      = Hash.new
+      @logger       = logger
+
+      @plugins = []
+
+      Pusher.app_id = '11495'
+      Pusher.key = 'a95aa7293cd158100246'
+      Pusher.secret = '9c13ccfe325fe3ae682d'
+
+      debug "plugin_ids = #{plugin_ids.inspect}"
+      debug "streaming_key = #{streaming_key}"
+
+      streamer_start_time = Time.now
+
+      hostname=Socket.gethostname
+      # load history
+      load_history
+
+      # get the array of plugins, AKA the plugin plan
+      @plugin_plan = Array(@history["old_plugins"])
+
+      # iterate through the plan and compile each plugin. We only compile plugins once at the beginning of the run
+      @plugin_plan.each do |plugin|
+        begin
+          compile_plugin(plugin) # this is what adds to the @plugin array
+        rescue Exception
+          error("Encountered an error: #{$!.message}")
+          puts $!.backtrace.join('\n')
+        end
+      end
+
+      # main loop. Continue running until global $continue_streaming is set to false OR we've been running for MAX DURATION
+      while(streamer_start_time+MAX_DURATION > Time.now && $continue_streaming) do
+        plugins=[]
+        @plugins.each_with_index do |plugin,i|
+          # ignore plugins whose ids are not in the plugin_ids array -- this also ignores local plugins
+          next if !(@plugin_plan[i]['id'] && plugin_ids.include?(@plugin_plan[i]['id'].to_i))
+          start_time=Time.now
+          plugin.reset!
+          plugin.run
+          duration=((Time.now-start_time)*1000).to_i
+
+          plugins << {:duration=>duration,
+                     :fields=>plugin.reports.inject{|memo,hash|memo.merge(hash)},
+                     :name=>@plugin_plan[i]["name"],
+                     :id=>@plugin_plan[i]["id"]}
+        end
+
+        bundle={:hostname=>hostname,
+                 :server_time=>Time.now.strftime("%I:%M:%S %p"),
+                 :num_processes=>`ps -e | wc -l`.chomp.to_i,
+                 :plugins=>plugins }
+
+        begin
+          Pusher[streaming_key].trigger!('server_data', bundle)
+        rescue Pusher::Error => e
+          # (Pusher::AuthenticationError, Pusher::HTTPError, or Pusher::Error)
+          error "Error pushing data: #{e.message}"
+        end
+
+        if false
+          # debugging
+          File.open(File.join(File.dirname(@history_file),"debug.txt"),"w") do |f|
+            f.puts "... sleeping @ #{Time.now.strftime("%I:%M:%S %p")}..."
+            f.puts bundle.to_yaml
+          end
+        end
+
+        sleep(SLEEP)
+      end
+    end
+
+    
+    private
+
+    #def post_bundle(bundle)
+    #  post( urlify(:stream),
+    #        "Unable to stream to server.",
+    #        bundle.to_json,
+    #        "Content-Type"     => "application/json")
+    #rescue Exception
+    #  error "Unable to stream to server."
+    #  debug $!.class.to_s
+    #  debug $!.message
+    #  debug $!.backtrace.join("\n")
+    #end
+
+    # sets up the @plugins array
+    def compile_plugin(plugin)
+      plugin_id = plugin['id']
+
+      # take care of plugin overrides
+      local_path = File.join(File.dirname(@history_file), "#{plugin_id}.rb")
+      if File.exist?(local_path)
+        code_to_run = File.read(local_path)
+      else
+        code_to_run=plugin['code'] || ""
+      end
+
+      id_and_name = "#{plugin['id']}-#{plugin['name']}".sub(/\A-/, "")
+      last_run    = @history["last_runs"][id_and_name] ||
+                    @history["last_runs"][plugin['name']]
+      memory      = @history["memory"][id_and_name] ||
+                    @history["memory"][plugin['name']]
+      options=(plugin['options'] || Hash.new)
+      options.merge!(:tuner_days=>"")
+      code_class=Plugin.extract_code_class(code_to_run)
+      begin
+        eval(code_to_run, TOPLEVEL_BINDING, plugin['path'] || plugin['name'] )
+        klass=Plugin.const_get(code_class)
+        info "Added a #{klass.name} plugin, id = #{plugin_id}"
+        @plugins << klass.load(last_run, (memory || Hash.new), options)
+
+        # turn certain methods into null-ops, so summaries aren't generated. Note that this is ad-hoc, and not future-proof.
+        if klass.name=="RailsRequests"; def klass.analyze;end;end
+        if klass.name=="ApacheAnalyzer"; def klass.generate_log_analysis;end;end
+
+      rescue Exception
+        error "Plugin would not compile: #{$!.message}"
+      end
+    end
+
+
+    def load_history
+      begin
+        debug "Loading history file..."
+        contents=File.read(@history_file)
+        @history = YAML.load(contents)
+      rescue => e
+        info "Couldn't load or parse the history file at #{@history_file}. Exiting."
+        exit(1)
+      end
+      info "History file loaded."
+    end
+
+    # Forward Logger methods to an active instance, when there is one.
+    def method_missing(meth, *args, &block)
+      if (Logger::SEV_LABEL - %w[ANY]).include? meth.to_s.upcase
+        @logger.send(meth, *args, &block) unless @logger.nil?
+      else
+        super
+      end
+    end
+
+    def growl(message)`growlnotify -m '#{message.gsub("'","\'")}'`;end
+
+  end
+end