jobmanager 1.0.0

data/test/configs/jobmanager_1.yaml

@@ -0,0 +1,9 @@
+job_logs_directory: temp/logs/<%=user_name%>
+
+central_log_mode: file
+
+central_log_file: temp/logs/<%=user_name%>/jobmanager.log
+
+number_of_job_logs: 5
+
+date_time_extension: false

data/test/configs/jobmanager_2.yaml

@@ -0,0 +1,11 @@
+email_condition: always
+
+job_logs_directory: "temp/logs"
+
+central_log_mode: syslog
+
+email_settings_file: email_settings.rb
+
+email_subject: "jobmanager results for job <%=job_name%>, command <%=command%>, user <%=user_name%>, host <%=host_name%>, result <%=result%>"
+
+zip_rotated_log_file: true

data/test/configs/jobmanager_3.yaml

@@ -0,0 +1,13 @@
+email_condition: on_failure
+
+job_logs_directory: temp/logs
+
+central_log_mode: syslog
+
+email_settings_file: email_settings.rb
+
+email_subject: "jobmanager results for <%=job_name%> : <%=result%>"
+
+debug: true
+
+

data/test/configs/jobmanager_4_bad_central_log_file.yaml

@@ -0,0 +1,9 @@
+job_logs_directory: temp/logs/<%=user_name%>
+
+central_log_mode: file
+
+central_log_file: temp/permissioned_directory/jobmanager.log
+
+email_condition: on_failure
+
+email_settings_file: email_settings.rb

data/test/configs/jobmanager_4_bad_email_settings_file.yaml

@@ -0,0 +1,9 @@
+job_logs_directory: temp/permissioned_directory/job_logs_directory
+
+central_log_mode: file
+
+central_log_file: temp/logs/jobmanager.log
+
+email_condition: always
+
+email_settings_file: /no_directory/no_file.rb

data/test/configs/jobmanager_4_bad_job_logs_directory_1.yaml

@@ -0,0 +1,9 @@
+job_logs_directory: temp/permissioned_directory/job_logs_directory
+
+central_log_mode: file
+
+central_log_file: temp/logs/jobmanager.log
+
+email_condition: on_failure
+
+email_settings_file: email_settings.rb

data/test/configs/jobmanager_4_bad_job_logs_directory_2.yaml

@@ -0,0 +1,9 @@
+job_logs_directory: temp/permissioned_directory
+
+central_log_mode: file
+
+central_log_file: temp/logs/jobmanager.log
+
+email_condition: on_failure
+
+email_settings_file: email_settings.rb

data/test/configs/minimum_plus_email_settings.yaml

@@ -0,0 +1,9 @@
+job_logs_directory: temp/logs
+
+job_name: ze_echoer
+
+command: "echo hello"
+
+email_condition: always
+
+email_settings_file: email_settings.rb

data/test/configs/minimum_required.yaml

@@ -0,0 +1,5 @@
+job_logs_directory: temp/logs
+
+job_name: ze_echoer
+
+command: "echo hello"

data/test/helpers.rb

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+module TestHelpers
+
+  TEMP_DIR = "temp"
+
+end

data/test/mock_syslog.rb

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+
+require 'etc'
+require 'logger'
+
+require 'rubygems'
+require 'syslog_logger'
+
+#
+# Global Variables for testing purposes
+#
+$test_syslog_logger = nil
+
+#
+# This class is a mock up, or redefinition of the methods in
+# SyslogLogger. The implementation simply saves the log messages
+# written to this class, instead of writing them to syslog.  It
+# facilitates the unit tests to confirm that the messages logged to
+# SyslogLogger are correct without requiring the test to be dependent
+# on the system's syslog facilities.  This class is only included by
+# unit test files.
+#
+class SyslogLogger
+  attr_accessor :level
+  
+  def string()
+    return @stringio.string()
+  end
+  
+  def initialize(program_name = "unknown")
+    @stringio = StringIO.new
+    @level = Logger::INFO
+
+    $test_syslog_logger = self
+  end
+  
+  def add(severity, message = nil)
+    return true if severity < @level
+    @stringio << message << "\n"
+    return true
+  end
+    
+  def close()
+  end
+  
+  def self.make_methods(method)
+    str =  "def #{method}(message = nil)\n"
+    str << "  if (SyslogLogger::LOGGER_MAP.keys.find() {|key| :#{method} == key})\n"
+    str << "    if (SyslogLogger::LOGGER_LEVEL_MAP[:#{method}] < @level) then return true end\n"
+    str << "    @stringio << message << \"\\n\"\n"
+    str << "    return true\n"
+    str << "  end\n"
+    str << "end\n"
+
+    str << "def #{method}?\n"
+    str << "  @level <= Logger::#{method.to_s.upcase}\n"
+    str << "end"
+
+    eval(str)
+
+  end
+
+  LOGGER_MAP.each_key do |level|
+    make_methods(level)
+  end
+
+end
+

data/test/test_applicationlogger.rb

@@ -0,0 +1,145 @@
+require 'jobmanager/applicationlogger.rb'
+
+require 'test/unit'
+require_relative('mock_syslog')
+require_relative('helpers')
+
+#
+# This class tests the assorted JobManager::ApplicationLogger* classes.
+#
+
+class TestApplicationLogger < Test::Unit::TestCase
+
+  include TestHelpers
+  
+  LOG_DIR = "#{TEMP_DIR}/logs"
+  LOG_FILE = "#{LOG_DIR}/application.log"
+
+  USER_NAME = Etc.getpwuid(Process::Sys.getuid()).name()
+  JOB_NAME  = "ze_echoer"
+  
+
+  LevelMessagePair = Struct.new(:level, :message)
+  
+  def setup
+    teardown()
+
+    FileUtils.mkdir_p(LOG_DIR)
+  end
+  
+  def teardown
+    FileUtils::rm_r(TEMP_DIR, :force => true)
+  end
+  
+  def get_log_em_all_list()
+    
+    return [ LevelMessagePair.new("DEBUG", "debug"),
+             LevelMessagePair.new("INFO", "info"),
+             LevelMessagePair.new("WARN", "warn"),
+             LevelMessagePair.new("ERROR", "error"),
+             LevelMessagePair.new("FATAL", "fatal"),
+             LevelMessagePair.new("", "unknown"),
+             LevelMessagePair.new("INFO", "write"),
+             LevelMessagePair.new("INFO", "<<"),
+             LevelMessagePair.new("INFO", "print"),
+             LevelMessagePair.new("INFO", "puts_1"),
+             LevelMessagePair.new("INFO", "puts_2"),
+             LevelMessagePair.new("INFO", "add"),
+             LevelMessagePair.new("INFO", "log"),
+             LevelMessagePair.new("WARN", "warn"),
+             LevelMessagePair.new("ERROR", "error"),
+             LevelMessagePair.new("FATAL", "fatal"),
+             LevelMessagePair.new("", "unknown")
+           ]
+  end
+  
+  def log_em_all(logger)
+    logger.debug("debug")
+    logger.info("info")
+    logger.warn("warn")
+    logger.error("error")
+    logger.fatal("fatal")
+    logger.unknown("unknown")
+    
+    logger.write("write")
+    logger << "<<"
+    logger.print "print"
+    logger.puts ["puts_1", "puts_2"]
+    logger.add(Logger::INFO, "add")
+    logger.log(Logger::INFO, "log")
+  end
+  
+  def test_log_file
+    logger = JobManager::ApplicationFileLogger.new(LOG_FILE, USER_NAME, JOB_NAME)
+    
+    logger.level = Logger::DEBUG
+    assert_equal(Logger::DEBUG, logger.level)
+    log_em_all(logger)
+    
+    logger.level = Logger::WARN
+    log_em_all(logger)
+    
+    logger.close
+    
+    file_contents = File.read(LOG_FILE)
+    lines = file_contents.split("\n")
+
+    print "\ntest_log_file file contents:\n"
+    print file_contents, "\n"
+
+    expected_list = get_log_em_all_list()
+
+    assert_equal(expected_list.length(), lines.length())
+    assert_equal(file_contents, logger.string())
+    
+    0.upto(expected_list.length() - 1).each do |i|
+      assert_match(/\[#{USER_NAME}. #{JOB_NAME}\]/, lines[i])
+      assert_match(expected_list[i].level, lines[i])
+      assert_match(expected_list[i].message, lines[i])
+    end
+  end
+  
+  def test_io
+    # We don't need to test much with this class as this was tested
+    # indirectly by the ApplicationFileLogger test.  However, lets
+    # make sure calling close on ApplicationIOLogger will not close
+    # STDOUT.
+    logger = JobManager::ApplicationIOLogger.new(STDOUT, USER_NAME, JOB_NAME)
+    
+    logger.close
+    assert_equal(false, STDOUT.closed?())
+  end
+
+  def test_syslog
+    program_name = "program_name"
+    logger = JobManager::ApplicationSyslogLogger.new(program_name, USER_NAME, JOB_NAME)
+    
+    logger.level = Logger::DEBUG
+    assert_equal(Logger::DEBUG, logger.level)
+    log_em_all(logger)
+    
+    logger.level = Logger::WARN
+    log_em_all(logger)
+    
+    logger.close
+    
+    syslog_contents = $test_syslog_logger.string()
+    lines = syslog_contents.split("\n")
+    
+    print "\ntest_syslog contents:\n"
+    print syslog_contents, "\n"
+
+    expected_list = get_log_em_all_list()
+
+    assert_equal(expected_list.length(), lines.length)
+    assert_equal(syslog_contents, logger.string())
+
+    0.upto(expected_list.length() - 1).each do |i|
+      assert_match(/\[#{USER_NAME}. #{JOB_NAME}\]/, lines[i])
+      assert_match(expected_list[i].level, lines[i])
+      assert_match(expected_list[i].message, lines[i])
+    end
+  end
+
+end
+

data/test/test_config.rb

@@ -0,0 +1,208 @@
+#!/usr/bin/env ruby
+
+require 'erb'
+require 'fileutils'
+
+require 'rubygems'
+require 'relative'
+require 'assertions'
+
+require 'jobmanager/applicationconfig.rb'
+require 'jobmanager/applicationoptionparser.rb'
+
+require 'configtoolkit/yamlreader'
+require 'configtoolkit/hashreader'
+require 'configtoolkit/overridereader'
+
+require 'test/unit'
+require_relative('helpers')
+
+#
+# This class tests the configuration related classes:
+# JobManager::ApplicationConfig and
+# JobManager::ApplicationOptionParser.
+#
+
+class ConfigTest < Test::Unit::TestCase
+  include TestHelpers
+  
+  CONFIG_DIR =  PathRelativeToCaller.new("./configs")
+  
+  def load_good_config(config_file)
+    config_file = File.join(CONFIG_DIR, config_file)
+    config = JobManager::ApplicationConfig.load(ConfigToolkit::YAMLReader.new(config_file))
+    
+    config
+  end
+  
+  def confirm_all_values(config)
+    assert_equal(:always, config.email_condition)
+    assert_equal(4, config.number_of_job_logs)
+    assert_equal(Pathname.new("temp/logs"), config.job_logs_directory)
+    assert_equal("/usr/designingpatterns/bin", config.command_path)
+    assert_equal(true, config.date_time_extension)
+    assert_equal('%F', config.date_time_extension_format)
+    assert_equal(true, config.zip_rotated_log_file)
+    assert_equal(1800, config.timeout)
+    assert_equal(Pathname.new("email_settings.rb"), config.email_settings_file)
+    assert_equal(:file, config.central_log_mode)
+    assert_equal(Pathname.new("temp/logs/jobmanager.log"), config.central_log_file)
+    assert_equal("ze_echoer", config.job_name)
+    assert_equal("echo hello", config.command)
+    assert_equal("sender_address@gmail.com", config.email_from_address)
+    assert_equal("receiver_address@gmail.com", config.email_to_address)
+
+    result = 'Success'
+    host_name = "jackfruit"
+    job_name = "mysql_backup"
+    
+    email_subject = ERB.new(config.email_subject).result(binding)
+    assert_equal("jobmanager results for mysql_backup on jackfruit : Success", 
+                 email_subject)
+  end
+
+  def confirm_minimum_required(config)
+    assert_equal("ze_echoer", config.job_name)
+    assert_equal("echo hello", config.command)
+    assert_equal(Pathname.new("temp/logs"), config.job_logs_directory)
+    
+    assert_equal(:on_failure, config.email_condition)
+    assert_equal(Pathname.new("email_settings.rb"), config.email_settings_file)
+    assert_equal(:syslog, config.central_log_mode)
+    assert_equal(3, config.number_of_job_logs)
+    assert_equal(true, config.date_time_extension)
+    assert_equal("%F_%T", config.date_time_extension_format)
+    assert_equal(false, config.zip_rotated_log_file)
+    assert_equal(false, config.debug)
+    assert_equal(nil, config.central_log_file)
+    assert_equal("jobmanager results for <%=job_name%> on <%=host_name%> : <%=result%>", 
+                 config.email_subject)
+        assert_equal(ENV['PATH'], config.command_path)
+
+    assert_equal(nil, config.timeout)
+    assert_equal(nil, config.email_to_address)
+    assert_equal(nil, config.email_from_address)
+  end
+
+  def test_good_configs
+    confirm_all_values(load_good_config("all_values.yaml"))
+  
+    confirm_minimum_required(load_good_config("minimum_required.yaml"))
+  
+    # just make sure no errors are thrown
+    config = load_good_config("minimum_plus_email_settings.yaml")
+  end
+
+  def test_bad_configs
+    msg = "error setting timeout with value 0: timeout must be a positive integer (seconds)."
+    confirm_bad_config_throws_error("bad_timeout_zero.yaml", msg)
+    
+    msg = "error setting number_of_job_logs with value 0: value must be > 0."
+    confirm_bad_config_throws_error("bad_number_of_job_logs.yaml", msg)
+    
+    msg = "error setting email_condition with value invalid: "
+    msg << "email_condition must be one of the following values: always, never, on_failure."
+    confirm_bad_config_throws_error("bad_email_condition.yaml", msg)
+    
+    msg = "JobManager::ApplicationConfig#validate_all_values error: "
+    msg << "If central_log_mode is set to file, central_log_file must be specified."
+    confirm_bad_config_throws_error("bad_no_central_log_file.yaml",
+                                    msg)
+  end
+  
+  def confirm_bad_config_throws_error(config_file, msg)
+    config_file = File.join(CONFIG_DIR, config_file)
+    
+    assert_raise_message(msg, ConfigToolkit::Error) do
+      config = JobManager::ApplicationConfig.load(ConfigToolkit::YAMLReader.new(config_file))
+    end
+  end
+
+  def test_good_option_parser()
+    program_name = "jobmanager"
+
+    # make sure that the job_name is calculated if it isn't provided
+    args = [ "--job_logs_directory",       "temp/logs",
+             "/bin/echo hello" ]
+
+    hash = JobManager::ApplicationOptionParser.parse(program_name, args)
+    config = JobManager::ApplicationConfig.load(ConfigToolkit::HashReader.new(hash))
+    assert_equal("echo", config.job_name)
+    
+    # match the config specified in minimum_required.yaml
+    args = [ "--job_name",                 "ze_echoer",
+             "--job_logs_directory",       "temp/logs",
+             "echo hello" ]
+
+    hash = JobManager::ApplicationOptionParser.parse(program_name, args)
+    config = JobManager::ApplicationConfig.load(ConfigToolkit::HashReader.new(hash))
+    confirm_minimum_required(config)
+    
+    # match the config specified in all_values.yaml
+    email_subject = "jobmanager results for <%=job_name%> on <%=host_name%> : <%=result%>"
+    args = [ "--job_name",                 "ze_echoer",
+             "--job_logs_directory",       "temp/logs",
+             "--email_condition",               "always",
+             "--central_log_mode",              "file",
+             "--central_log_file",              "temp/logs/jobmanager.log",
+             "--number_of_job_logs",       "4",
+             "--date_time_extension",
+             "--date_time_extension_format",    "%F",
+             "--zip_rotated_log_file",
+             "--timeout",                       "1800",
+             "--email_settings_file",           "email_settings.rb",
+             "--email_from_address",            "sender_address@gmail.com",
+             "--email_to_address",              "receiver_address@gmail.com",
+             "--command_path",                  "/usr/designingpatterns/bin",
+             "--email_subject",                 email_subject,
+             "--debug",
+             "echo hello"
+           ]
+
+    hash = JobManager::ApplicationOptionParser.parse(program_name, args)
+    config = JobManager::ApplicationConfig.load(ConfigToolkit::HashReader.new(hash))
+    confirm_all_values(config)
+
+    # confirm the no- boolean configuration parameters work
+    args = [ "--job_logs_directory",       "temp/logs",
+             "--no-date_time_extension",
+             "--no-zip_rotated_log_file",
+             "--no-debug",
+             "echo hello"
+           ]
+    hash = JobManager::ApplicationOptionParser.parse(program_name, args)
+    config = JobManager::ApplicationConfig.load(ConfigToolkit::HashReader.new(hash))
+    assert_equal(false, config.date_time_extension())
+    assert_equal(false, config.zip_rotated_log_file())
+    assert_equal(false, config.debug())
+
+  end
+  
+  def test_yaml_and_option_parser()
+    program_name = "jobmanager"
+
+    # test the optionparser and the yaml reader together - both configs are incomplete on their own.
+    config_file = File.join(CONFIG_DIR, "incomplete_1.yaml")
+    yaml_reader = ConfigToolkit::YAMLReader.new(config_file)
+    
+    args = [ "--job_name",                 "ze_echoer",
+             "--email_to_address",              "receiver_address@gmail.com",
+             "--central_log_file",              "temp/logs/jobmanager.log",
+             "--email_settings_file",           "email_settings.rb",
+             "--timeout",                       "1800",
+             "--zip_rotated_log_file",
+             "--debug",
+             "echo hello" 
+           ]
+    
+    hash = JobManager::ApplicationOptionParser.parse(program_name, args)
+    hash_reader = ConfigToolkit::HashReader.new(hash)
+    
+    override_reader = ConfigToolkit::OverrideReader.new(yaml_reader, hash_reader)
+    confirm_all_values(JobManager::ApplicationConfig.load(override_reader))
+  end
+  
+end
+
+
+