jobmanager 1.0.0

data/test/test_jobmanager.rb

@@ -0,0 +1,443 @@
+#!/usr/bin/env ruby
+
+require 'yaml'
+require 'stringio'
+require 'logger'
+require 'zlib'
+require 'fileutils'
+
+require 'rubygems'
+require 'relative'
+require 'assertions'
+require 'sys/host'
+
+require 'configtoolkit'
+require 'configtoolkit/yamlreader'
+require 'configtoolkit/yamlwriter'
+require 'configtoolkit/overridereader'
+
+require 'jobmanager'
+
+require 'test/unit'
+require_relative('helpers')
+require 'mock_syslog'
+
+#
+# This class essentially tests the bin/jobmanager application since
+# the bin/jobmanager application does little more than call into the
+# JobManager::Application class, which this test class directly tests.
+#
+
+class TestJobManager < Test::Unit::TestCase
+  include TestHelpers
+
+  CONFIG_DIR =  PathRelativeToCaller.new("./configs")
+  
+  LOG_DIR = "#{TEMP_DIR}/logs"
+  
+  PROGRAM_NAME = $0
+  USER_NAME = Etc.getpwuid(Process::Sys.getuid()).name()
+  HOST_NAME = Sys::Host.hostname
+
+  # override the interactive shell calculation, and set to false
+  INTERACTIVE_SHELL_OVERRIDE = false 
+
+  def setup
+    teardown()
+
+    FileUtils.mkdir_p(TEMP_DIR)
+    FileUtils.mkdir_p(LOG_DIR)
+    
+    ActionMailer::Base.delivery_method = :test
+    ActionMailer::Base.deliveries = []
+  end
+  
+  def teardown
+    FileUtils::rm_r(TEMP_DIR, :force => true)
+  end
+  
+  # Invoke the JobManager::Application a number of times to launch the
+  # echo command.  Confirm the output logs produced (the job log file
+  # and the central log file).  Also, confirm that the job logs are
+  # properly rotated.
+  def test_jobmanager_1
+    config_file = File.join(CONFIG_DIR, "jobmanager_1.yaml")
+    config = File.open(config_file) { |fs| YAML::load(fs) }
+    
+    user_name = USER_NAME
+    central_log_file = ERB.new(config["central_log_file"]).result(binding)
+    job_logs_directory = ERB.new(config["job_logs_directory"]).result(binding)
+    
+    1.upto(config["number_of_job_logs"] + 1) do |iteration|
+      args = [ "echo #{iteration}" ]
+          
+      jobmanager = JobManager::Application.new(PROGRAM_NAME, 
+                                               args, 
+                                               config_file, 
+                                               INTERACTIVE_SHELL_OVERRIDE)
+      jobmanager.run()
+      
+      # Confirm that nothing was logged via syslog - The central log
+      # should be written to a file, not via syslog!
+      assert_equal("", $test_syslog_logger.string())
+      STDERR.print "\nJob Manager Pre Log Contents #1:\n"
+      STDERR.print $test_syslog_logger.string(), "\n"
+      
+      jobmanager_log_contents = File.read(central_log_file)
+      
+      print "\nJob Manager Log Contents #1:\n"
+      print jobmanager_log_contents, "\n"
+      
+      lines = jobmanager_log_contents.split("\n")
+      assert_match(/^I, .* INFO .* \[#{USER_NAME}, echo\] Exited successfully$/, lines[1])
+      
+      lines.each { |line| assert_no_match(/^ERROR/, line) }
+      
+      # confirm correctness of the job log file.
+      job_log_file = "#{job_logs_directory}/echo.log.1"
+      assert(File.file?(job_log_file))
+      
+      print "\nContents of #{job_log_file}:\n"
+      print File.read(job_log_file), "\n"
+
+      expected_contents = "#{iteration}\n"
+      assert_equal(expected_contents, File.read(job_log_file))
+    end
+    
+    # confirm only number_of_job_logs log files have been kept (the
+    # rest discarded via the log rotation mechanism inside
+    # JobManager::Application).
+    assert_equal(config["number_of_job_logs"],
+                 Dir.glob("#{job_logs_directory}/echo.log.*").length)    
+  end
+  
+  # Invoke the JobManager::Application to launch the echo command.
+  # Confirm the output logs produced (the job log file and the central
+  # log) are correct.  This test configures JobManager::Application to
+  # log central log information via syslog, which has been mocked out
+  # for testing purposes.  Confirm the email generated is correct, and
+  # that the job log is properly rotated and zipped.
+  def test_jobmanager_2
+    config_file = File.join(CONFIG_DIR, "jobmanager_2.yaml")
+    command_line_args = [ "-j", "ze_echoer",
+                          "echo 1" 
+                        ]
+    
+    # Load the ApplicationConfig class ourselves (the
+    # JobManager::Application class will also load the configuration
+    # separately).
+    yaml_reader = ConfigToolkit::YAMLReader.new(config_file)
+    command_parameters_hash = JobManager::ApplicationOptionParser.parse(PROGRAM_NAME, command_line_args.dup())
+    hash_reader = ConfigToolkit::HashReader.new(command_parameters_hash)
+    override_reader = ConfigToolkit::OverrideReader.new(yaml_reader, hash_reader)
+    config = JobManager::ApplicationConfig.load(override_reader)
+    
+    # We need to test embedding the user_name variable in the
+    # email_settings_file parameter.  Edit the email_settings_file to
+    # include this variable, and then write the new configuration to a
+    # temporary location.  We will then pass this newly created
+    # configuration file to the JobManager::Application class.
+    config.email_settings_file = File.expand_path(TEMP_DIR) + "/<%=user_name%>/email_settings.rb"
+    new_config_file = "#{TEMP_DIR}/jobmanager_2.yaml"
+    
+    # write the new config to a temporary location
+    File.open(new_config_file, "w") do |file_stream|
+      config.dump(ConfigToolkit::YAMLWriter.new(file_stream, true))
+    end
+    
+    # copy the email settings file in the standard config directory to
+    # the email settings file referenced in the new config.
+    user_name = USER_NAME
+    evaluated_file = ERB.new(config.email_settings_file).result(binding)
+    FileUtils.mkdir_p(File.dirname(evaluated_file))
+    FileUtils.cp("#{CONFIG_DIR}/email_settings.rb", evaluated_file)
+
+    jobmanager = JobManager::Application.new(PROGRAM_NAME, 
+                                             command_line_args.dup(), 
+                                             new_config_file, 
+                                             INTERACTIVE_SHELL_OVERRIDE)
+    jobmanager.run()
+    
+    user_name = USER_NAME
+    job_logs_directory = ERB.new(config.job_logs_directory).result(binding)
+    
+    # confirm correctness of the central log.
+    jobmanager_log_contents = $test_syslog_logger.string()
+    
+    print "\nJob Manager Log Contents #2:\n"
+    print jobmanager_log_contents, "\n"
+    
+    lines = jobmanager_log_contents.split("\n")
+    assert_match(/INFO \[#{USER_NAME}, ze_echoer\] Exited successfully$/, lines[1])
+    lines.each { |line| assert_no_match(/ERROR/, line) }
+    
+    # confirm the correctness of the job log file.
+    job_log_files = Dir.glob("#{job_logs_directory}/ze_echoer.log.*.gz")
+    
+    assert_equal(1, job_log_files.length)
+    job_log_file = job_log_files[0]
+    
+    Zlib::GzipReader.open(job_log_file)  do |gz|
+      expected_contents = "1\n"
+      actual_contents = gz.read
+      assert_equal(expected_contents, actual_contents)
+
+      print "\nContents of #{job_log_file}:\n"
+      print actual_contents, "\n"
+    end
+    
+    #confirm the email generated is correct.
+    deliveries = ActionMailer::Base.deliveries
+    assert_equal(1, deliveries.length)
+    
+    delivery = deliveries[0]
+
+    assert_equal("jobmanager results for job ze_echoer, command echo 1, user janet, host #{HOST_NAME}, result Success",
+                 delivery.subject)
+    assert_match(/^INFO \[#{USER_NAME}, ze_echoer\] Exited successfully$/, delivery.body)
+  end
+  
+  # test config w/ email + date extensions, command that exists but fails
+
+  # Invoke the JobManager::Application to launch the cat command with
+  # a bogus file, which will cause the command to fail.  Confirm the
+  # output logs produced (the job log file and the central log) are
+  # correct.  Confirm the email generated is correct.
+  def test_jobmanager_3
+    config_file = File.join(CONFIG_DIR, "jobmanager_3.yaml")
+    config = File.open(config_file) { |fs| YAML::load(fs) }
+    
+    assert(JobManager::System.which("cat"))
+    args = [ "cat #{TEMP_DIR}/no_file" ]
+    jobmanager = JobManager::Application.new(PROGRAM_NAME, 
+                                             args, 
+                                             config_file,
+                                             INTERACTIVE_SHELL_OVERRIDE)
+    jobmanager.run()
+    
+    # confirm the correctness of jobmanager.log.
+    jobmanager_log_contents = $test_syslog_logger.string()
+    
+    print "\nJob Manager Log Contents #3:\n"
+    print jobmanager_log_contents, "\n"
+    
+    lines = jobmanager_log_contents.split("\n")
+    assert_match(/^ERROR \[janet, cat\] Failed! - exited normally with exit code:/, jobmanager_log_contents)
+    lines.each { |line| assert_no_match(/Exited successfully/, line) }
+    
+    # confirm the email generated is correct.
+    deliveries = ActionMailer::Base.deliveries
+    assert_equal(1, deliveries.length)
+    
+    delivery = deliveries[0]
+    
+    assert_equal("jobmanager results for cat : Failure", delivery.subject)
+    assert_match(/^ERROR \[#{USER_NAME}, cat\] Failed! - exited normally with exit code:/, 
+                 delivery.body)
+    
+  end
+
+  #
+  # Test the following error cases:
+  # 1) The central log file can't be created/opened.
+  # 2) The email results file can't be created/opened.
+  # 3) The job logs directory can't be created.
+  # 4) The job logs directory exists but is permissioned such that a file can't be created inside.
+  # 
+  def test_jobmanager_exceptions()
+    jobmanager_exception_bad_central_log_file()  
+    teardown(); setup();
+
+    jobmanager_exception_bad_email_settings_file()
+    teardown(); setup();
+
+    jobmanager_exception_bad_job_logs_directory_1()
+    teardown(); setup();
+
+    jobmanager_exception_bad_job_logs_directory_2()
+    
+  end
+  
+  # In this test case, the job logs directory can't be created.
+  # Test that the error is properly raised, logged, and emailed.
+  def jobmanager_exception_bad_job_logs_directory_1()
+    
+    # Permission the directory so that the central log file can't be created inside this directory.
+    FileUtils.mkdir_p("#{TEMP_DIR}/permissioned_directory")
+    FileUtils.chmod(0000, "#{TEMP_DIR}/permissioned_directory")
+    
+    config_file = File.join(CONFIG_DIR, "jobmanager_4_bad_job_logs_directory_1.yaml")
+    
+    assert_raise_message("Error creating job_logs_directory", JobManager::Application::ComposedError) do
+      jobmanager = JobManager::Application.new(PROGRAM_NAME, 
+                                               args = [ "ls" ],
+                                               config_file,
+                                               INTERACTIVE_SHELL_OVERRIDE)
+      
+      jobmanager.run()
+    end
+    
+    # return the permissions
+    FileUtils.chmod(0755, "#{TEMP_DIR}/permissioned_directory")
+    
+    # confirm nothing was written via syslog.
+    assert_equal("", $test_syslog_logger.string())
+    
+    jobmanager_log_contents = File.read("temp/logs/jobmanager.log")
+    
+    print "\nJob Manager Log Contents #4 (jobmanager_4_bad_job_logs_directory_1.yaml):\n"
+    print jobmanager_log_contents, "\n"
+
+    lines = jobmanager_log_contents.split("\n")
+    assert_match(/^E, .* : \[#{USER_NAME}, ls\] Error creating job_logs_directory: Error \(Errno::EACCES\): Permission denied - temp\/permissioned_directory\/job_logs_directory/, 
+                 lines[0])
+
+    # confirm the email generated is correct.
+    deliveries = ActionMailer::Base.deliveries
+    assert_equal(1, deliveries.length)
+    
+    delivery = deliveries[0]
+    
+    assert_equal("jobmanager results for ls on #{HOST_NAME} : Failure", delivery.subject)
+    assert_match(/^E, .* : \[#{USER_NAME}, ls\] Error creating job_logs_directory: Error \(Errno::EACCES\): Permission denied - temp\/permissioned_directory\/job_logs_directory/,
+                 delivery.body)
+
+  end  
+
+
+  # In this test case, the job logs directory exists but permissions prevent a job log from being created.
+  # Test that the error is properly raised, logged, and emailed.
+  def jobmanager_exception_bad_job_logs_directory_2()
+    
+    # Permission the directory so that the central log file can't be created inside this directory.
+    FileUtils.mkdir_p("#{TEMP_DIR}/permissioned_directory")
+    FileUtils.chmod(0000, "#{TEMP_DIR}/permissioned_directory")
+    
+    config_file = File.join(CONFIG_DIR, "jobmanager_4_bad_job_logs_directory_2.yaml")
+    
+    assert_raise_message("Error opening job log file", JobManager::Application::ComposedError) do
+      jobmanager = JobManager::Application.new(PROGRAM_NAME, 
+                                               args = [ "ls" ],
+                                               config_file,
+                                               INTERACTIVE_SHELL_OVERRIDE)
+      
+      jobmanager.run()
+    end
+    
+    # return the permissions
+    FileUtils.chmod(0755, "#{TEMP_DIR}/permissioned_directory")
+    
+    # confirm nothing was written via syslog.
+    assert_equal("", $test_syslog_logger.string())
+    
+    jobmanager_log_contents = File.read("temp/logs/jobmanager.log")
+    
+    print "\nJob Manager Log Contents #4 (jobmanager_4_bad_job_logs_directory_2.yaml):\n"
+    print jobmanager_log_contents, "\n"
+
+    lines = jobmanager_log_contents.split("\n")
+    assert_match(/^E, .* : \[#{USER_NAME}, ls\] Error opening job log file: Error \(Errno::EACCES\): Permission denied - temp\/permissioned_directory\/ls\.log/, 
+                 lines[0])
+
+    # confirm the email generated is correct.
+    deliveries = ActionMailer::Base.deliveries
+    assert_equal(1, deliveries.length)
+    
+    delivery = deliveries[0]
+    
+    assert_equal("jobmanager results for ls on #{HOST_NAME} : Failure", delivery.subject)
+    assert_match(/^E, .* : \[#{USER_NAME}, ls\] Error opening job log file: Error \(Errno::EACCES\): Permission denied - temp\/permissioned_directory\/ls\.log/,
+                 delivery.body)
+
+  end  
+  
+  
+  # In this test case, the email settings file can't be opened.
+  # Test that the error is properly raised, logged, and not emailed.
+  def jobmanager_exception_bad_email_settings_file()
+    
+    # Permission the directory so that the central log file can't be
+    # created inside this directory.
+    FileUtils.mkdir_p("#{TEMP_DIR}/permissioned_directory")
+    FileUtils.chmod(0000, "#{TEMP_DIR}/permissioned_directory")
+    
+    config_file = File.join(CONFIG_DIR, "jobmanager_4_bad_email_settings_file.yaml")
+    
+    assert(JobManager::System.which("cat"))
+
+    assert_raise_message("email_settings_file (/no_directory/no_file.rb) does not exist or is not readable.\n", JobManager::Application::Error) do
+      jobmanager = JobManager::Application.new(PROGRAM_NAME, 
+                                               args = [ "ls" ],
+                                               config_file,
+                                               INTERACTIVE_SHELL_OVERRIDE)
+      
+      jobmanager.run()
+    end
+
+    # return the permissions
+    FileUtils.chmod(0755, "#{TEMP_DIR}/permissioned_directory")
+    
+    # confirm correctness of syslog.
+    jobmanager_log_contents = $test_syslog_logger.string()
+    
+    print "\nJob Manager Log Contents #4 (jobmanager_4_bad_email_settings_file.yaml):\n"
+    print jobmanager_log_contents, "\n"
+
+    lines = jobmanager_log_contents.split("\n")
+    assert_equal("ERROR [janet, UNKNOWN] email_settings_file (/no_directory/no_file.rb) does not exist or is not readable.",
+                 lines[0])
+    
+    # confirm no email was generated.
+    deliveries = ActionMailer::Base.deliveries
+    assert_equal(0, deliveries.length)
+  end
+  
+  # In this test case, the central log file can't be opened/created.
+  # Test that the error is properly raised, logged, and emailed.
+  def jobmanager_exception_bad_central_log_file()
+
+    # Permission the directory so that the central log file can't be created inside this directory.
+    FileUtils.mkdir_p("#{TEMP_DIR}/permissioned_directory")
+    FileUtils.chmod(0000, "#{TEMP_DIR}/permissioned_directory")
+    
+    config_file = File.join(CONFIG_DIR, "jobmanager_4_bad_central_log_file.yaml")
+    
+    assert_raise_message("Error Initializing Central Log", 
+                         JobManager::Application::ComposedError) do
+      jobmanager = JobManager::Application.new(PROGRAM_NAME, 
+                                               args = [ "ls" ],
+                                               config_file,
+                                               INTERACTIVE_SHELL_OVERRIDE)
+      
+      jobmanager.run()
+    end
+    
+    # return the permissions
+    FileUtils.chmod(0755, "#{TEMP_DIR}/permissioned_directory")
+    
+    # confirm correctness of syslog.
+    jobmanager_log_contents = $test_syslog_logger.string()
+    
+    print "\nJob Manager Log Contents #4 (jobmanager_4_bad_central_log_file.yaml):\n"
+    print jobmanager_log_contents, "\n"
+
+    lines = jobmanager_log_contents.split("\n")
+    assert_equal("ERROR [#{USER_NAME}, UNKNOWN] Error Initializing Central Log: Error (Errno::EACCES): Permission denied - temp/permissioned_directory/jobmanager.log", 
+                 lines[0])
+    
+    # confirm the email generated is correct.
+    deliveries = ActionMailer::Base.deliveries
+    assert_equal(1, deliveries.length)
+    
+    delivery = deliveries[0]
+    
+    assert_equal("jobmanager results for ls on #{HOST_NAME} : Failure", delivery.subject)
+    assert_match(/ERROR \[#{USER_NAME}, UNKNOWN\] Error Initializing Central Log: Error \(Errno::EACCES\): Permission denied - temp\/permissioned_directory\/jobmanager.log/, 
+                 delivery.body)
+
+  end
+
+end
+
+
+

data/test/test_system.rb

@@ -0,0 +1,206 @@
+#!/usr/bin/env ruby
+
+require 'stringio'
+require 'fileutils'
+
+require 'rubygems'
+require 'relative'
+
+require 'test/unit'
+require_relative('helpers')
+require_relative('mock_syslog')
+
+require 'jobmanager/system'
+require 'jobmanager/applicationlogger'
+
+#
+# This class tests the JobManager::System module.
+#
+
+class TestSystem < Test::Unit::TestCase
+  include TestHelpers
+  
+  SAMPLE_DIR = "#{TEMP_DIR}/samples"
+  
+  SAMPLE_FILE = File.join(SAMPLE_DIR, "sample.dat")
+  EXPECTED_SAMPLE_CONTENTS = (1..10000).map { "This is a sample file." }.join("\n")
+
+  USER_NAME = Etc.getpwuid(Process::Sys.getuid()).name()
+
+  def setup
+    teardown()
+
+    FileUtils.mkdir_p(TEMP_DIR)
+    FileUtils.mkdir_p(SAMPLE_DIR)
+    
+    File.open(SAMPLE_FILE, "w") do |file_stream|
+      file_stream << EXPECTED_SAMPLE_CONTENTS
+    end    
+  end
+  
+  def teardown
+    FileUtils::rm_r(TEMP_DIR, :force => true)
+  end
+  
+  # Command w/ relative path (cat) - success
+  def test_simple_cat_success
+    command = "cat #{SAMPLE_FILE}"
+    run_simple_cat(command)
+  end
+  
+  # Command w/ absolute path (eg. /bin/cat) - success
+  def test_absolute_path_command_success
+    exe = JobManager::System.which("cat")
+    command = "#{exe} #{SAMPLE_FILE}"
+    
+    run_simple_cat(command)
+  end
+
+  # Command (cat) w/ large timeout - success
+  def test_simple_cat_with_timeout_success
+    command = "cat #{SAMPLE_FILE}"
+    optional_args = { :timeout => 100 }
+    
+    run_simple_cat(command)
+  end
+  
+  # Command w/ relative path (cat) - options (good path)
+  def test_simple_cat_with_environment_path_specified_success
+    exe = JobManager::System.which("cat")
+    path = exe.split()[0]
+    
+    command = "#{exe} #{SAMPLE_FILE}"
+    optional_args = { :command_path => path }
+    
+    run_simple_cat(command)
+  end
+  
+  def create_logger(job_name)
+    return JobManager::ApplicationSyslogLogger.new("job_manager", 
+                                                   USER_NAME,
+                                                   job_name)
+  end
+  
+  # Command (cat) - cat nonexistent file
+  def test_simple_cat__failure
+    command = "cat #{SAMPLE_DIR}/no_file.dat"
+    job = "catter"
+    log_stream = StringIO.new
+    logger = create_logger(job)
+    success = JobManager::System.invoke_command(command, 
+                                                log_stream, 
+                                                logger)
+    
+    logger.close
+    
+    control_log_lines = logger.string.split("\n")
+    print "\n", control_log_lines.join("\n"), "\n"
+    
+    assert_equal(false, success)
+    
+    assert_match(/ERROR \[#{USER_NAME}, catter\] Failed! - exited normally with exit code:/,
+                 control_log_lines[-1])
+  end
+  
+  # Command (cat) - options (bad path)
+  def test_simple_cat_with_environment_path_specified_failure
+    command = "cat #{SAMPLE_FILE}"
+    job = "catter"
+    log_stream = StringIO.new
+    logger = create_logger(job)
+    optional_args = { :command_path => "/bad_directory_1:/bad_directory_2" }
+    
+    success = JobManager::System.invoke_command(command, 
+                                                log_stream, 
+                                                logger,
+                                                optional_args)
+    
+    logger.close
+    
+    control_log_lines = logger.string.split("\n")
+    print "\n", control_log_lines.join("\n"), "\n"
+
+    assert_equal(false, success)
+    assert_equal("", log_stream.string())
+    
+    assert_match(/ERROR \[#{USER_NAME}, catter\] File cat does not exist in path .* or is not executable!/,
+                 control_log_lines[-1])
+  end
+  
+  def run_simple_cat(command, optional_args = {})
+    job = "catter"
+    log_stream = StringIO.new
+    logger = create_logger(job)
+    
+    success = JobManager::System.invoke_command(command, 
+                                                log_stream, 
+                                                logger,
+                                                optional_args)
+    
+    logger.close
+    
+    control_log_lines = logger.string.split("\n")
+    print "\n", control_log_lines.join("\n"), "\n"
+
+    assert_equal(true, success)
+    assert_equal(EXPECTED_SAMPLE_CONTENTS, log_stream.string())
+    
+    assert_match(/INFO \[#{USER_NAME}, catter\] Exited successfully/,
+                 control_log_lines[-1])
+  end
+  
+  # Launch a sleep command w/ a timeout shorter than the sleep time.
+  def test_sleep_past_timeout
+    command = "sleep 100"
+    job = "catter"
+    log_stream = StringIO.new
+    logger = create_logger(job)
+    optional_args = { :timeout => 1 }
+    
+    success = JobManager::System.invoke_command(command, 
+                                                log_stream, 
+                                                logger,
+                                                optional_args)
+    
+    logger.close
+    
+    control_log_lines = logger.string.split("\n")
+    print "\n", control_log_lines.join("\n"), "\n"
+    
+    # verify that the sleep process has been timed out!
+    assert_equal(false, success)
+    
+    assert_match(/ERROR \[#{USER_NAME}, catter\] Timed out after 1 seconds./,
+                 control_log_lines[-3])
+  end
+  
+  # Test whether the invoke_command properly recognizes when the process that has been
+  # forked has died due to a signal.
+  def test_sleep_and_kill
+    job_pid = nil
+    
+    command = "sleep 1000"
+    job = "catter"
+    log_stream = StringIO.new
+    logger = create_logger(job)
+    optional_args = { :timeout => 100 }
+    
+    success = JobManager::System.invoke_command(command, 
+                                                log_stream, 
+                                                logger,
+                                                optional_args) do |job_pid|
+      Process.kill("TERM", job_pid)
+    end
+    
+    logger.close
+    
+    control_log_lines = logger.string.split("\n")
+    print "\n", control_log_lines.join("\n"), "\n"
+    
+    assert_equal(false, success)
+    
+    assert_match(/ERROR \[#{USER_NAME}, catter\] Failed! - exited abnormally due to signal:/,
+                 control_log_lines[-1])
+  end
+end
+