scout_agent 3.0.0

data/TODO

@@ -0,0 +1,3 @@
+= To Do List
+
+Coming soon...

data/bin/scout_agent

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+$VERBOSE = true # -w
+$KCODE   = "u"  # -Ku
+
+# load the agent
+$LOAD_PATH << File.join(File.expand_path(File.dirname(__FILE__)), *%w[.. lib])
+require "scout_agent"
+
+# dispatch this command to the agent
+ScoutAgent::Dispatcher.dispatch

data/lib/scout_agent.rb

@@ -0,0 +1,73 @@
+#!/usr/bin/env ruby -wKU
+
+# require standard libraries
+require "etc"
+require "fileutils"
+require "optparse"
+require "ostruct"
+require "pathname"
+require "stringio"
+require "thread"
+require "timeout"
+require "uri"
+require "zlib"
+
+# load agent
+require "scout_agent/core_extensions"
+require "scout_agent/wire_tap"
+require "scout_agent/database"
+require "scout_agent/tracked"
+require "scout_agent/server"
+require "scout_agent/plan"
+require "scout_agent/id_card"
+require "scout_agent/assignment"
+require "scout_agent/lifeline"
+require "scout_agent/dispatcher"
+
+# require gems
+require_lib_or_gem "json"
+require_lib_or_gem "amalgalite"
+require_lib_or_gem "rest_client"
+require_lib_or_gem "xmpp4r"
+require_lib_or_gem "xmpp4r/roster"
+
+# The namespace for all agent software.
+module ScoutAgent
+  # Returns the name of the agent executable.
+  def self.agent_name
+    name.snake_case
+  end
+  
+  # Returns agent_name() converted to a proper human-readable name.
+  def self.proper_agent_name
+    agent_name.tr("_", " ").gsub(/\w+/) { |w| w.capitalize }
+  end
+  
+  # 
+  # A helper that prepares a log for +process_name+.  This log will use the
+  # configured log level and be rotated daily.  If the agent is not running as
+  # a daemon, the log will also be teed to <tt>$stdout</tt>, but you can pass
+  # a +true+ value into +skip_stdout+ to prevent this.
+  # 
+  def self.prepare_wire_tap(process_name, skip_stdout = false)
+    wire_tap          = WireTap.new(Plan.log_dir + "#{agent_name}.log", :daily)
+    begin
+      wire_tap.level  = ScoutAgent::WireTap::Severity.const_get(
+        Plan.logging_level
+      )
+    rescue NameError  # unrecognized level
+      # do nothing:  we will stick with the default
+    end
+    wire_tap.progname = process_name
+    wire_tap.tap      = $stdout unless skip_stdout or Plan.run_as_daemon?
+    wire_tap
+  end
+
+  # The version of this agent.
+  VERSION = "3.0.0".freeze
+  # A Pathname reference to the agent code directory, used in dynamic loading.
+  LIB_DIR = Pathname.new(File.dirname(__FILE__)) + agent_name
+end
+
+# An external alias.
+Scout = ScoutAgent

data/lib/scout_agent/agent.rb

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby -wKU
+
+module ScoutAgent
+  class Agent
+    include Tracked
+    
+    def initialize
+      @log = ScoutAgent.prepare_wire_tap(file_name)
+      log.info("Loading.")
+
+      status("Loading", file_name)
+      at_my_exit do
+        clear_status(file_name)
+      end
+    end
+    
+    attr_reader :log
+    
+    def authorize
+      IDCard.new(file_name).authorize
+    end
+    
+    def run
+      raise NotImplementedError,
+            "Subclasses must override ScoutAgent::Agent#run()."
+    end
+    
+    def notice_changes
+      # do nothing:  specific agents can override for their purposes
+    end
+    
+    def finish
+      exit
+    end
+    
+    private
+    
+    def file_name
+      self.class.short_name.sub(/Agent\z/, "").snake_case
+    end
+  end
+end

data/lib/scout_agent/agent/communication_agent.rb

@@ -0,0 +1,85 @@
+#!/usr/bin/env ruby -wKU
+
+# load agent extensions
+require "scout_agent/api"
+require "scout_agent/order"
+
+module ScoutAgent
+  class Agent
+    class CommunicationAgent < Agent
+      def initialize
+        super  # setup our log and status
+
+        Order.log = log  # pass on our log
+        Order.load_all   # load supported orders
+        
+        @agent_jid       = nil
+        @jabber          = nil
+        @roster          = nil
+        @shutdown_thread = nil
+      end
+      
+      def run
+        if Plan.test_mode?
+          login
+          update_status("Online since #{Time.now.utc.to_db_s}")
+          fetch_roster
+          install_subscriptions_callback
+          install_messages_callback
+          listen
+        else
+          loop { sleep 60 }
+        end
+      end
+      
+      def finish
+        if @shutdown_thread
+          @shutdown_thread.run
+        else
+          exit
+        end
+      end
+      
+      private
+      
+      def login
+        Thread.abort_on_exception = true  # make XMPP4R fail fast
+        @agent_jid = Jabber::JID.new("#{Plan.agent_key}@jabber.org/agent")
+        @jabber    = Jabber::Client.new(@agent_jid)
+        no_warnings { @jabber.connect }
+        @jabber.auth(Plan.agent_key)
+      end
+      
+      def update_status(message, status = nil)
+        presence        = Jabber::Presence.new
+        presence.status = message
+        presence.show   = status
+        @jabber.send(presence)
+      end
+      
+      def fetch_roster
+        @roster = Jabber::Roster::Helper.new(@jabber)
+      end
+      
+      def install_subscriptions_callback
+        @roster.add_subscription_request_callback do |_, presence|
+          @roster.accept_subscription(presence.from)
+        end
+      end
+      
+      def install_messages_callback
+        @jabber.add_message_callback do |message|
+          if order = Order.can_handle?(message)
+            order.execute
+          end
+        end
+      end
+      
+      def listen
+        @shutdown_thread = Thread.current
+        Thread.stop
+        @jabber.close
+      end
+    end
+  end
+end

data/lib/scout_agent/agent/master_agent.rb

@@ -0,0 +1,301 @@
+#!/usr/bin/env ruby -wKU
+
+# load agent extensions
+require "scout_agent/mission"
+
+module ScoutAgent
+  class Agent
+    class MasterAgent < Agent
+      def initialize
+        super  # setup our log and status
+        
+        @running   = true
+        @main_loop = nil
+        @server    = Server.new(log)
+        @db        = Database.load(:mission_log, log)
+        @queue     = Database.load(:queue,       log)
+        @snapshots = Database.load(:snapshots,   log)
+        
+        if [@db, @queue, @snapshots].any? { |db| db.nil? }
+          log.fatal("Could not load all required databases.")
+          exit
+        end
+      end
+      
+      def run
+        log.info("Running.")
+        @main_loop = Thread.new do
+          Thread.current.abort_on_exception = true
+          loop do
+            %w[ fetch_plan 
+                execute_missions 
+                checkin
+                perform_maintenance 
+                wait_for_orders ].each do |stage|
+              send(stage)
+              check_running_status
+            end
+          end
+        end
+        @main_loop.join
+      end
+      
+      def notice_changes
+        @main_loop.run if @main_loop
+      rescue ThreadError  # Thread was already killed
+        # do nothing:  we're shutting down and can't notice new things
+      end
+      
+      def finish
+        if @running
+          log.info("Shutting down.")
+        else
+          log.warn("Received multiple shutdown signals.")
+        end
+        @running = false
+        notice_changes
+      end
+      
+      private
+      
+      #############
+      ### Agent ###
+      #############
+      
+      def fetch_plan
+        log.info("Fetching plan from server.")
+        status("Fetching plan from server")
+        headers = {}
+        if not Plan.test_mode? and (old_plan = @db.current_plan)
+          log.debug( "Adding If-Modified-Since for plan fetch:  " +
+                     "#{old_plan[:last_modified]}." )
+          headers[:if_modified_since] = old_plan[:last_modified]
+        end
+        json_plan = @server.get_plan(headers)
+        if json_plan.nil?  # failed to retrieve plan
+          log.warn("Could not retrieve plan from server.")
+          return
+        elsif json_plan.to_s.empty?  # skip empty plans
+          log.info("Received empty plan.")
+          return
+        else
+          log.info("Received plan (#{json_plan.to_s.size} bytes).")
+        end
+        begin
+          ruby_plan = JSON.parse(json_plan.to_s)
+        rescue JSON::ParserError  # bad JSON
+          log.error("Plan from server was malformed JSON.")
+          return  # skip plan update
+        end
+        @db.update_plan( json_plan.headers[:last_modified],
+                         Array(ruby_plan["plugins"]) )
+        @snapshots.update_commands(Array(ruby_plan["commands"]))
+      end
+      
+      def execute_missions
+        status("Running missions")
+        ran_a_mission = false
+        while mission = @db.current_mission
+          log.info("Running #{mission[:name]} mission.")
+          ran_a_mission = true
+          pid           = fork do
+            reset_environment
+            compile_mission(mission)
+            run_mission(mission)
+            complete_mission(mission)
+          end
+          
+          begin
+            Timeout.timeout(mission[:timeout]) do
+              Process.wait(pid)
+            end
+            unless $?.success?
+              log.warn( "#{mission[:name]} exited with an error:  " +
+                        "#{$?.exitstatus}." )
+              @db.write_report(
+                mission[:id],
+                :error,
+                :subject => "#{mission[:name]} exited with an error",
+                :body    => "Exit status:  #{$?.exitstatus}"
+              )
+            end
+          rescue Timeout::Error  # mission exceeded allowed execution
+            status = Process.term_or_kill(pid)
+            log.error( "#{mission[:name]} took too long to run:  " +
+                       "#{status.exitstatus}." )
+            @db.write_report(
+              mission[:id],
+              :error,
+              :subject => "#{mission[:name]} took too long to run",
+              :body    => "Exit status:  #{status.exitstatus}"
+            )
+          end
+          # prevent an infinite loop if we can't complete the mission
+          unless @db.complete_mission(mission)
+            log.error("Mission run cancelled due to imcomplete missions.")
+            break
+          end
+        end
+        log.warn("No missions to run.") unless ran_a_mission
+      end
+      
+      def checkin
+        reports   = @db.current_reports
+        queued    = @queue.queued_reports
+        snapshots = @snapshots.current_runs
+        if reports.empty? and queued.empty? and snapshots.empty?
+          log.warn("No data to report to the server.")
+          return
+        end
+        
+        log.info("Checking in with server.")
+        status("Checking in with server")
+        checkin = { :reports => Array.new,
+                    :hints   => Array.new,
+                    :alerts  => Array.new,
+                    :errors  => Array.new }
+        (reports + queued).each do |report|
+          type                 =  report.delete_at(:type)
+          checkin["#{type}s".to_sym] << report.to_hash
+        end
+        checkin[:snapshots] = snapshots.map { |run| run.to_hash }
+        
+        report_dates = String.new
+        if reports.first or queued.first
+          dates                 = [ [reports.first, queued.first],
+                                    [reports.last,  queued.last] ]
+          first_date, last_date = dates.map { |date_group|
+            date_group.compact.map { |report| report[:created_at] }.min
+          }
+          report_dates          = " from #{first_date} to #{last_date}"
+        end
+        snapshot_dates = String.new
+        if snapshots.first
+          snapshot_dates = " from #{snapshots.first[:created_at]} to " +
+                           "#{snapshots.last[:created_at]}"
+        end
+        log.info( "Sending #{reports.size} reports "                     +
+                  "(#{checkin[:reports].size} reports, "                 +
+                  "#{checkin[:hints].size} hints, "                      +
+                  "#{checkin[:alerts].size} alerts, "                    +
+                  "and #{checkin[:errors].size} errors)#{report_dates} " +
+                  "and #{snapshots.size} snapshot runs#{snapshot_dates}." )
+        if @server.post_checkin(checkin)
+          log.info("Server received data.")
+        else
+          log.warn("Could not get data to server.")
+        end
+      end
+      
+      def perform_maintenance
+        log.info("Running maintenance tasks.")
+        status("Running maintenance tasks")
+        
+        # VACUUM databases
+        [@db, @queue, @snapshots, status_database].compact.each do |db|
+          result = db.maintain
+          if result.nil?
+            log.error("Maintenance for #{db.path} database failed.")
+          elsif result
+            log.debug("VACUUM successful for #{db.path}.")
+          end
+        end
+        
+        # clean out old logs
+        Plan.log_dir.each_entry do |log_file|
+          if log_file.to_s =~ /\.(\d{4})(\d{2})(\d{2})\z/
+            log_day = Time.local(*$~.captures.map { |n| n.to_i })
+            if Time.now - log_day > 60 * 60 * 24 * 7
+              begin
+                (Plan.log_dir + log_file).unlink
+              rescue Exception => error  # file cannot be unlinked
+                log.error( "Failed to unlink old log file '#{log_file}':  " +
+                           "#{error.message} (#{error.class})." )
+                next
+              end
+              log.debug("Successfully unlinked old log file '#{log_file}'.")
+            end
+          end
+        end
+      end
+      
+      def wait_for_orders
+        pause = @db.seconds_to_next_mission
+        log.info("Waiting #{pause} seconds for next mission run.")
+        status("Idle")
+        sleep pause
+      end
+      
+      def check_running_status
+        exit unless @running
+      end
+      
+      ###############
+      ### Mission ###
+      ###############
+      
+      def reset_environment
+        # swap out our parent's signal handlers
+        install_shutdown_handler { exit }
+
+        # clear the parent's identity and assume mine
+        IDCard.me = nil
+        IDCard.new(:mission).authorize or exit(1)
+
+        # get a handle on the log
+        @log = ScoutAgent.prepare_wire_tap(:mission)
+
+        # record the new process status
+        force_status_database_reload
+        at_my_exit do
+          clear_status
+        end
+      end
+      
+      def compile_mission(mission)
+        log.info("Compiling #{mission[:name]} mission.")
+        status("Compiling")
+        begin
+          eval(mission[:code], TOPLEVEL_BINDING, mission[:name])
+        rescue Exception => error  # any compile error
+          raise if $!.is_a? SystemExit  # don't catch exit() calls
+          log.error( "#{mission[:name]} could not be compiled:  " +
+                      "#{error.message} (#{error.class})." )
+          reported = @db.write_report(
+            mission[:id],
+            :error,
+            :subject => "#{mission[:name]} could not be compiled",
+            :body    => "#{error.message}\n#{error.backtrace.join("\n")}"
+          )
+          exit(reported ? 0 : 2)  # warn parent if we can't report
+        end
+      end
+      
+      def run_mission(mission)
+        log.info("Preparing #{mission[:name]} mission.")
+        if prepared = Mission.prepared
+          log.info("Starting #{mission[:name]} mission.")
+          status("Running")
+          prepared.new( *mission.values_at( :id,
+                                            :name,
+                                            :last_run_at,
+                                            :memory,
+                                            :options ) ).run
+        else   # no mission loaded
+          log.error("#{mission[:name]} could not be prepared.")
+          reported = @db.write_report(
+            mission[:id],
+            :error,
+            :subject => "#{mission[:name]} could not be prepared",
+            :body    => "The code didn't define a Scout::Plugin subclass"
+          )
+          exit(reported ? 0 : 3)  # warn parent if we can't report
+        end
+      end
+      
+      def complete_mission(mission)
+        log.info("#{mission[:name]} mission complete.")
+      end
+    end
+  end
+end