jobmanager 1.0.0

data/Rakefile

@@ -0,0 +1,17 @@
+# -*- ruby -*-
+require 'rubygems'
+require 'hoe'
+
+Hoe.new('jobmanager', "1.0.0") do |p|
+  p.remote_rdoc_dir = ''
+  p.developer('DesigningPatterns', 'technical.inquiries@designingpatterns.com')
+
+  p.extra_deps << ['simpleemail']
+  p.extra_deps << ['configtoolkit', '>= 2.2']
+  p.extra_deps << ['relative']
+  p.extra_deps << ['logrotate']
+  p.extra_deps << ['SyslogLogger']
+  p.extra_deps << ['sys-host']
+end
+
+# vim: syntax=Ruby

data/bin/jobmanager

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'relative'
+require 'jobmanager'
+
+PROGRAM_NAME = File.basename($0)
+
+begin
+  env_config_file = ENV["JOBMANAGER_CONFIG_FILE"]
+  config_file = env_config_file ? env_config_file : "/etc/jobmanager.yaml"
+    
+  jobmanager = JobManager::Application.new(PROGRAM_NAME, ARGV, config_file)
+  jobmanager.run()
+rescue => error
+  # All relevant errors have already been printed.
+  exit(1)
+end
+
+

data/examples/email_settings.rb

@@ -0,0 +1,14 @@
+ActionMailer::Base.delivery_method = :smtp
+
+ActionMailer::Base.smtp_settings = {
+  :address => 'smtp.gmail.com',
+  :port   => 587,
+  :domain => "gmail.com",
+  :authentication => "login",
+  :user_name => "sender_address@gmail.com",
+  :password => "password",
+  :tls => :auto
+}
+
+SimpleEmail::Email.default_from = "Sender Address <sender_address@gmail.com>"
+SimpleEmail::Email.default_to = "Receiver Address <receiver_address@gmail.com>"

data/examples/example.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+#
+# These example programs use the 'relative' gem in order to express
+# paths relative to __FILE__ cleanly, allowing them to be run from any
+# directory.  In particular, they use
+# File.expand_path_relative_to_caller in order to refer to
+# configuration files within the examples directory.
+#
+require 'rubygems'
+require 'relative'
+
+CONFIGURATION_FILE = File.expand_path_relative_to_caller("jobmanager.yaml")
+
+ENV["JOBMANAGER_CONFIG_FILE"] = CONFIGURATION_FILE
+
+command = "jobmanager --job_name ls_usr \"ls /usr\""
+print "Command: #{command}\n"
+
+output = %x[ #{command} ]
+
+print "Output:\n"
+print output

data/examples/jobmanager.yaml

@@ -0,0 +1,66 @@
+################################################################################
+# The jobmanager configuration file.
+#
+# For a full description of each of the fields, and their default values,
+# see the CONFIGURATION FILE section in README.txt.
+################################################################################
+
+# The number of log files to be kept per job.
+number_of_job_logs: 5
+
+# The directory in which the job logs will be kept, to be interpreted
+# by ERB.  Allowed variables: user_name.
+job_logs_directory: /tmp/logs
+
+# jobmanager logs operational information (when jobs are launched,
+# exit, etc).  This can be done via syslog or directly to a log file.
+# Possible values are: syslog, file.
+central_log_mode: file
+
+# If the central_log_mode is set to "file", the log file must be
+# specified.  This parameter will be interpreted by ERB.  Allowed
+# variables: user_name.
+central_log_file: /tmp/logs/jobmanager.log
+
+# The path to search for the command that jobmanager is invoked with.  
+#command_path: /usr/bin:/usr/designingpatterns/bin
+
+# Whether jobmanager should print debug trace to the central log file.
+debug: false
+
+# The condition upon which condition results should be emailed.
+# Possible values are: always, never, on_failure.  
+email_condition: always
+
+# The configuration file for the simpleemail gem.  This parameter will
+# be interpreted by ERB.  Allowed variables: user_name.  Note: If a
+# relative path is specified, it will be considered to be relative to
+# the location of this configuration file.
+email_settings_file: email_settings.rb
+
+# The email subject, to be interpreted by ERB.
+# Allowed variables: job_name, command, host_name, results, user_name.
+email_subject: "jobmanager results for <%=job_name%> on <%=host_name%> : <%=result%>"
+
+# The extension of the rotated job log file.  If true, the extension
+# is a date_time.  Otherwise, it is a simple count (.1, .2, etc.).
+date_time_extension: true
+
+# If date_time_extension is true, this field will be used as the
+# format for the date/time extension of the rotated job log file
+# name.
+date_time_extension_format: '%F_%T'
+
+# Whether the rotated job log file should be zipped.
+zip_rotated_log_file: false
+
+# The timeout (in seconds).  If not present, no timeout will be used.
+timeout: 1800
+
+# The sender of emails sent from jobmanager.  This can also be
+# specified in the email_settings_file (attribute default_from).
+#email_from_address: "Sender <sender@gmail.com>"
+
+# The receiver of emails sent from jobmanager.  This can also be
+# specified in the email_settings_file (attributes default_to).
+#email_to_address: "Receiver <receiver@gmail.com>"

data/examples/mysql_backup.rb

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+print "=================================================\n"
+print "Backing up Database\n"
+print "=================================================\n"
+
+database = ARGV[0]
+
+command     = "mysqldump #{database} > /tmp/#{database}.mysql"
+print command, "\n"
+
+if (system("#{command}"))
+  print"\nSuccess!\n"
+else
+  print "\nFailure!\n"
+  exit(1)
+end
+

data/lib/jobmanager.rb

@@ -0,0 +1 @@
+require 'jobmanager/application'

data/lib/jobmanager/application.rb

@@ -0,0 +1,455 @@
+#!/usr/bin/env ruby
+
+require 'erb'
+require 'ostruct'
+require 'stringio'
+
+require 'rubygems'
+require 'logrotate'
+require 'relative'
+require 'sys/host'
+require 'configtoolkit'
+require 'configtoolkit/yamlreader'
+require 'configtoolkit/hashreader'
+require 'configtoolkit/overridereader'
+
+require 'jobmanager/util'
+
+# The simpleemail gem will bring in a bunch of ActionMailer warnings.
+# Disable this temporarily.
+JobManager.disable_warnings { require 'simpleemail' }
+
+require 'jobmanager/applicationconfig'
+require 'jobmanager/applicationoptionparser'
+require 'jobmanager/system'
+require 'jobmanager/teestream'
+require 'jobmanager/applicationlogger'
+
+module JobManager
+
+  class Application
+
+    # Inside the Application class, two classes of exceptions are
+    # raised (Error and ComposedError).  This is done so that when an
+    # exception is caught, the exception handling code can discern
+    # between an exception that this Application has raised and an
+    # unexpected exception (raised in code other than this class).
+    # The exception handling code can than log this information.
+    
+    # A simple exception class used in this class.
+    class Error < StandardError
+    end
+    
+    # A simple exception class that repackages an existing exception.
+    # This class often catches exceptions raised by other libraries,
+    # and repackages them, adding additional trace (via the msg
+    # parameter).
+    class ComposedError < Error
+      def initialize(msg, contained_exception, backtrace = caller())
+        @msg = msg
+        @contained_exception = contained_exception
+        
+        super(msg)
+        set_backtrace(backtrace)
+      end
+      
+      def contained_exception()
+        return @contained_exception
+      end
+    end
+    
+    # 
+    # ====Description: 
+    # This method creates an Application instance.  It also reads in
+    # the configuration file and the command line options.
+    # 
+    # ====Parameters:
+    # [program_name]
+    #      The name of the program.
+    # [argv]
+    #      A list of command line arguments.
+    # [config_file]
+    #      The jobmanager configuration file name.
+    # [override_terminal_shell]
+    #      Whether to override this method's internal determination of
+    #      whether the calling process has been invoked by a shell
+    #      attached to a terminal.  If the value is nil, this method
+    #      will make its own determination.  This is purely for
+    #      debugging purposes.
+    #
+    def initialize(program_name, 
+                   argv, 
+                   config_file,
+                   override_terminal_shell = nil)
+
+      begin
+        @mgr_logger = nil
+        @email_ready = false
+        @program_name = program_name
+        @argv = argv
+        @config_file = config_file
+        @user_name = Etc.getpwuid(Process::Sys.getuid).name()
+        
+        @terminal_shell = STDIN.tty? || STDOUT.tty? || STDERR.tty?
+        if (override_terminal_shell != nil)
+          @terminal_shell = override_terminal_shell
+        end
+        
+        # If any of this process's standard iostreams are connected to
+        # a terminal, this process has been invoked directly or
+        # indirectly by the command line shell.  As a result, let's log
+        # central log messages to STDOUT.
+        if (@terminal_shell)
+          @mgr_logger = ApplicationIOLogger.new(STDOUT,
+                                                @user_name)
+          
+          str =  "\n"
+          str << "*NOTE* "
+          str << "jobmanager has been invoked interactively.  Ignoring the "
+          str << "central_log_* configuration parameters, and logging all output to "
+          str << "the terminal.\n\n"
+          
+          print str
+          
+        else
+          # None of this process's standard iostreams are connected to
+          # a terminal.  This process has most likely been invoked by
+          # cron.  Log all central log messages to syslog.  The user
+          # may have specified in the configuration file for all
+          # central log messages to be directed to a file.  However,
+          # until we successfully process the configuration file, we
+          # do not know the log mode they have chosen.  Thus,
+          # temporarily log central log messages to syslog until we
+          # have read this configuration parameter.  No messages will
+          # actually be written to syslog unless an error condition
+          # occurs (such as not being able to read or parse the
+          # configuration file itself!)
+          @mgr_logger = ApplicationSyslogLogger.new(@program_name,
+                                                    @user_name)
+        end
+      
+        # @mgr_logger is now valid! We can print to @mgr_logger in
+        # rescue statements going forwards.
+        # This method populates @config with the configuration from
+        # the configuration file and the command line parameters.
+        gather_options(@program_name, @argv, @config_file)
+        
+        # This is done so that if any methods called by this
+        # application call any of the standard print commands (print,
+        # puts, etc) directly, these messages will be printed to
+        # @mgr_logger.
+        $stderr = $stdout = @mgr_logger
+        
+        # Print the config in debug mode.
+        if (@mgr_logger.level <= Logger::DEBUG)
+          @mgr_logger.debug("Configuration:\n #{@config}\n")
+        end
+      rescue => e
+        handle_exception(e)
+        # Re-raise the exception to notify bin/jobmanager to exit!
+        raise e
+      end
+    end
+    
+    def handle_exception(e)
+      if (@mgr_logger)
+        case e
+        when ComposedError
+          @mgr_logger.record_tagged_exception(e.message, e.contained_exception)
+        when Error
+          @mgr_logger.record_exception(e)
+        else
+          @mgr_logger.record_tagged_exception("Unexpected Error", e)
+        end
+        
+        begin
+          if (@email_ready) then email_results(false) end
+        rescue => e
+          @mgr_logger.record_tagged_exception("Error emailing results", e)
+        end
+        
+        @mgr_logger.close() 
+        @mgr_logger = nil
+      end
+      
+      $stderr = STDERR
+      $stdout = STDOUT
+    end
+    
+    # 
+    # ====Description:
+    # This method runs the jobmanager application.
+    # 
+    def run()      
+      success = false
+      
+      begin
+        FileUtils.mkdir_p(@config.job_logs_directory)
+      rescue => e
+        raise ComposedError.new("Error creating job_logs_directory", e)
+      end
+      
+      job_log_file = File.join(@config.job_logs_directory, "#{@config.job_name}.log")
+      @mgr_logger.debug("Opening job log file #{job_log_file}.")
+      
+      begin
+        job_log_file_stream = File.open(job_log_file, "w")
+      rescue => e
+        raise ComposedError.new("Error opening job log file", e)
+      end
+      
+      #
+      # The output of the job that will be invoked will be sent back
+      # to this process via a pipe (inside invoke_command).  This
+      # process will use the job's output for 2 purposes.  It will be
+      # forwarded to a job log file and the output will be used later
+      # when emailing results.  invoke_command takes only 1
+      # job_output_stream.  We could pass a string stream to
+      # invoke_command, and after write the contents to the job's log
+      # file, and also include it in the emailed results.  However,
+      # this is less than optimal, as the job output will not be
+      # written to the log file in real time; it will not be written
+      # until after the job has completed.  For long jobs, this can
+      # leave the job owner with no way of determining the status of
+      # the job while it is running.  Instead, we construct a
+      # TeeStream which is a stream that forwards the messages to a
+      # list of streams.  In this case, we will initialize the
+      # TeeStream to forward strings to the job log file stream and a
+      # string stream (which will be later used for emailing results).
+      #
+      begin
+        job_tee_stream = TeeStream.new(:file => job_log_file_stream,
+                                       :stringio => StringIO.new())
+        
+        optional_args = {}
+        if @config.timeout then optional_args[:timeout] = @config.timeout end
+        optional_args[:command_path] = @config.command_path
+        
+        # invoke the command - fork and exec the process, and wait til it finishes to reap the status.
+        success = System::invoke_command(@config.command, 
+                                         job_tee_stream,
+                                         @mgr_logger,
+                                         optional_args)
+        
+        job_output = job_tee_stream.get_stream(:stringio).string()
+        job_tee_stream.close()
+        
+      rescue => e
+        @mgr_logger.record_tagged_exception("Error running job", e)
+      end
+      
+      # Rotate the job log file.
+      begin
+        if (File.file?(job_log_file))
+          
+          before_entries = Dir.glob("#{@config.job_logs_directory}/#{@config.job_name}*")
+          
+          logrotate_options = { 
+            :count => @config.number_of_job_logs,
+            :date_time_ext => @config.date_time_extension,
+            :date_time_format => @config.date_time_extension_format,
+            :gzip => @config.zip_rotated_log_file
+          }
+          
+          LogRotate.rotate_file(job_log_file, logrotate_options)
+          
+          rotated_file = (Dir.glob("#{@config.job_logs_directory}/#{@config.job_name}*") - before_entries)[0]
+          @mgr_logger.info("Job log rotated to #{rotated_file}.")
+        end
+      rescue => e
+        success = false
+        @mgr_logger.record_tagged_exception("Error rotating file #{job_log_file}", e)
+      end
+      
+      # Email the results!
+      begin
+        if (@config.email_condition == :always || 
+            (@config.email_condition == :on_failure && !success))
+          
+          email_results(success, job_output)
+        end
+      rescue => e
+        @mgr_logger.record_tagged_exception("Error emailing results", e)
+      end
+      
+    rescue => e
+      handle_exception(e)
+      raise e
+      
+    ensure
+      if (@mgr_logger) then 
+        @mgr_logger.close() 
+        @mgr_logger = nil
+      end
+      
+      $stderr = STDERR
+      $stdout = STDOUT
+    end
+    
+    private
+    
+    #
+    # ====Description: 
+    # This method reads the configuration file and the command line
+    # options into a configuration class (ApplicationConfig).
+    #
+    def gather_options(program_name, argv, config_file)
+      yaml_reader = nil
+      begin
+        yaml_reader = ConfigToolkit::YAMLReader.new(config_file)
+      rescue => e
+        raise ComposedError.new("Error loading #{config_file}", e)
+      end
+      
+      command_parameters_hash = nil
+      begin
+        command_parameters_hash = JobManager::ApplicationOptionParser.parse(program_name, argv)
+        hash_reader = ConfigToolkit::HashReader.new(command_parameters_hash)
+      rescue => e
+        raise ComposedError.new("Error Parsing Command Line", e)
+      end
+      
+      @config = nil
+      begin
+        # Union together the configuration file and the command line options.
+        override_reader = ConfigToolkit::OverrideReader.new(yaml_reader, hash_reader)
+        @config = JobManager::ApplicationConfig.load(override_reader)
+      rescue => e
+        raise ComposedError.new("Error validating configuration", e)
+      end
+      
+      # if jobmanager is configured to email results, process email
+      # related configuration parameters.
+      if (@config.email_condition != :never)
+        
+        # Do not email on error, email is not yet setup!
+        @config.email_settings_file = evaluate_configuration_parameter(@config, :email_settings_file)
+        
+        # if the email_settings_file is a relative path, convert it to
+        # an absolute path (or a relative path relative to the
+        # process's current working directory).  Assume that the
+        # email_settings_file is specified relative to the jobmanager
+        # configuration file's directory.
+        if (value = @config.email_settings_file)
+          if (!value.absolute?)
+            value = value.expand_path(File.dirname(config_file))
+          end
+          
+          if (!value.file? || !value.readable?)
+            raise Error.new("email_settings_file (#{value}) does not exist or is not readable.\n")
+          end
+          
+          @config.email_settings_file = value
+        end
+
+        # Note: From/To email addresses can be specified in either the
+        # jobmanager configuration file or the simpleemail gem
+        # configuration file.
+        begin
+          SimpleEmail::Email.email_settings_file = @config.email_settings_file
+          if (@config.email_from_address) 
+            SimpleEmail::Email.default_from = @config.email_from_address 
+          end
+          if (@config.email_to_address)
+            SimpleEmail::Email.default_to = @config.email_to_address
+          end
+        rescue => e
+          raise ComposedError.new("Error evaluating email settings file (#{@config.email_settings_file}, ", e)
+        end
+        
+        @email_ready = true
+      end
+      
+      if (@config.central_log_mode == :file && !@terminal_shell)
+        # Note: Earlier we set up @mgr_logger to log via syslog
+        # regardless of the central_log_mode configuration parameter
+        # (we hadn't read the configuration file at that point!).
+        # Close the syslog logger, and set @mgr_logger up to log
+        # directly to a file.
+
+        # Evaluate the string parameters that allow ERB syntax.
+        @config.central_log_file = evaluate_configuration_parameter(@config, :central_log_file)
+        
+        begin
+          new_logger = ApplicationFileLogger.new(@config.central_log_file,
+                                                 @user_name)
+
+          old_logger = @mgr_logger
+          @mgr_logger = new_logger
+          old_logger.close
+          
+        rescue => e
+          raise ComposedError.new("Error Initializing Central Log", e)
+        end
+      end
+      
+      @mgr_logger.job_name = @config.job_name
+      @mgr_logger.level = @config.debug ? Logger::DEBUG : Logger::INFO
+      
+      @config.job_logs_directory = evaluate_configuration_parameter(@config, :job_logs_directory)
+    end
+
+    #
+    # ====Description: 
+    # This method emails the results of the job to the configured email address.
+    #
+    def email_results(success,
+                      job_output = nil)
+
+      email_banner = "---------------------------------------------------------\n"
+      
+      mgr_output = @mgr_logger.string()
+      
+      # Evaluate the email_subject configuration parameter string with
+      # ERB.  Resolve the following variables to their values listed
+      # below.
+      result = success ? "Success" : "Failure"
+      host_name = Sys::Host.hostname
+      job_name = @config.job_name
+      command = @config.command
+      user_name = @user_name
+      
+      subject = nil
+      begin
+        subject = ERB.new(@config.email_subject).result(binding)
+      rescue => e
+        @mgr_logger.record_tagged_exception("Error evaluating email subject with ERB", e)
+        subject = "Invalid Subject - Check Logs!"
+      end
+      
+      body =""
+      body += email_banner
+      body += "jobmanager Output:\n"
+      body += email_banner
+      body += "#{mgr_output}\n"
+      
+      if (job_output && job_output.length > 0)
+        body += email_banner
+        body += "Job Output:\n"
+        body += email_banner
+        body += "#{job_output}\n"
+      end
+      
+      ActionMailer::Base.logger = @mgr_logger
+      SimpleEmail::Email.send_email(nil, nil, subject, body)  
+    end
+
+    def evaluate_configuration_parameter(config, parameter)
+      # Evaluate the parameter string with ERB, replacing the
+      # user_name variable reference with the user name that invoked
+      # this application.
+      evaluated_parameter = nil
+
+      begin
+        user_name = @user_name
+        evaluated_parameter = ERB.new(config.send(parameter)).result(binding)
+      rescue => e
+        raise ComposedError.new("Error evaluating #{parameter} with ERB", e)
+      end
+      
+      return evaluated_parameter
+    end
+    
+  end
+
+end