tengine_job_agent 0.3.17

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+source "http://rubygems.org"
+
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+gem "tengine_support", "~> 0.3.0"
+gem "tengine_event", "~> 0.4.0"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "rspec", "~> 2.6.0"
+  gem "yard", "~> 0.7.2"
+  gem "bundler", "~> 1.0.18"
+  gem "jeweler", "~> 1.6.4"
+  # gem "rcov", ">= 0"
+  gem "simplecov", "~> 0.5.3"
+  gem "ZenTest", "~> 4.6.2"
+  gem "ci_reporter", "~>1.6.5"
+end

data/Gemfile.lock

@@ -0,0 +1,67 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    ZenTest (4.6.2)
+    activesupport (3.2.1)
+      i18n (~> 0.6)
+      multi_json (~> 1.0)
+    amq-client (0.8.7)
+      amq-protocol (>= 0.8.4)
+      eventmachine
+    amq-protocol (0.8.4)
+    amqp (0.8.4)
+      amq-client (~> 0.8.7)
+      amq-protocol (~> 0.8.4)
+      eventmachine
+    builder (3.0.0)
+    ci_reporter (1.6.9)
+      builder (>= 2.1.2)
+    diff-lcs (1.1.3)
+    eventmachine (0.12.10)
+    git (1.2.5)
+    i18n (0.6.0)
+    jeweler (1.6.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    macaddr (1.5.0)
+      systemu (>= 2.4.0)
+    multi_json (1.0.4)
+    rake (0.9.2.2)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+    simplecov (0.5.4)
+      multi_json (~> 1.0.3)
+      simplecov-html (~> 0.5.3)
+    simplecov-html (0.5.3)
+    systemu (2.4.2)
+    tengine_event (0.4.6)
+      activesupport (>= 3.0.0)
+      amqp (~> 0.8.0)
+      tengine_support (>= 0.3.24)
+      uuid (~> 2.3.4)
+    tengine_support (0.3.24)
+      activesupport (>= 3.0.0)
+    uuid (2.3.5)
+      macaddr (~> 1.0)
+    yard (0.7.5)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  ZenTest (~> 4.6.2)
+  bundler (~> 1.0.18)
+  ci_reporter (~> 1.6.5)
+  jeweler (~> 1.6.4)
+  rspec (~> 2.6.0)
+  simplecov (~> 0.5.3)
+  tengine_event (~> 0.4.0)
+  tengine_support (~> 0.3.0)
+  yard (~> 0.7.2)

data/README.rdoc

@@ -0,0 +1,20 @@
+= tengine_job_agent
+
+Description goes here.
+
+== Contributing to tengine_job_agent
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== License
+tengine_job_agent is distributed under the MPL2.0 or LGPLv3 or the dual license of MPL2.0/LGPLv3
+
+== Copyright
+
+Copyright (c) 2011 nautilus-technologies.com

data/Rakefile

@@ -0,0 +1,42 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "tengine_job_agent"
+  gem.homepage = "http://github.com/tengine/tengine_job_agent"
+  gem.license = "MPL2.0/LGPLv3"
+  gem.summary = %Q{tengine_job_agent invoke job, watches it and notify its finish to tengine server}
+  gem.description = %Q{tengine_job_agent works with tengine_job}
+  gem.email = "tengine@nautilus-technologies.com"
+  gem.authors = %w[taigou totty g-morita shyouhei akm]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'yard'
+YARD::Rake::YardocTask.new

data/VERSION

@@ -0,0 +1 @@
+0.3.17

data/bin/tengine_job_agent_kill

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+target_pid = ARGV.shift
+interval = ARGV.shift.to_i # seconds
+signals = ARGV.join(',').split(/,/).compact
+
+first_time = true
+signals.each do |signal|
+  begin
+    result = Process.kill(signal, -1 * target_pid.to_i) # killing process group
+    puts "result: #{result.inspect}"
+    first_time = false
+  rescue Errno::ESRCH
+    if first_time
+      puts "#{$!.message}."
+    else
+      # previous signal must have succeeded to kill the process
+      break
+    end
+  end
+  sleep(interval)
+end

data/bin/tengine_job_agent_run

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+$LOAD_PATH << File.expand_path("../lib", File.dirname(__FILE__))
+require "tengine_job_agent"
+
+exit_code = TengineJobAgent::Run.process(*ARGV)
+exit(exit_code)

data/bin/tengine_job_agent_watchdog

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+unless RUBY_VERSION >= "1.9.2"
+  raise "RUBY_VERSION must be >= 1.9.2 but was #{RUBY_VERSION}"
+end
+
+# Process.daemon(true, true)
+Process.daemon(true)
+
+$LOAD_PATH << File.expand_path("../lib", File.dirname(__FILE__))
+require "tengine_job_agent"
+
+exit_code = TengineJobAgent::Watchdog.process(*ARGV)
+exit(exit_code)

data/lib/tengine_job_agent.rb

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+
+module TengineJobAgent
+
+  autoload :Run         , "tengine_job_agent/run"
+  autoload :Watchdog    , 'tengine_job_agent/watchdog'
+
+  autoload :CommandUtils, 'tengine_job_agent/command_utils'
+
+end

data/lib/tengine_job_agent/command_utils.rb

@@ -0,0 +1,38 @@
+require 'tengine_job_agent'
+require 'logger'
+require 'yaml'
+require 'tengine/support/yaml_with_erb'
+
+module TengineJobAgent::CommandUtils
+  def self.included(mod)
+    mod.extend(ClassMethods)
+  end
+
+  module ClassMethods
+    def load_config
+      config_path = Dir["{.,./config,/etc}/tengine_job_agent{.yml,.yml.erb}"].first
+      YAML.load_file(config_path)
+    end
+
+    def process(*args)
+      config = load_config
+      logger = new_logger(config)
+      begin
+        return new(logger, args, config).process
+      rescue Exception => e
+        logger.error("error: [#{e.class.name}] #{e.message}\n  " << e.backtrace.join("\n"))
+        return false
+      end
+    end
+
+    def new_logger(config)
+      logfile = config['logfile']
+      unless logfile
+        prefix = self.name.split('::').last.downcase
+        logfile = File.expand_path("#{prefix}-#{Process.pid}.log", config['log_dir'])
+      end
+      Logger.new(logfile)
+    end
+
+  end
+end

data/lib/tengine_job_agent/run.rb

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+require 'tengine_job_agent'
+require 'timeout'
+require 'rbconfig'
+
+class TengineJobAgent::Run
+  include TengineJobAgent::CommandUtils
+
+  def initialize(logger, args, config = {})
+    @logger = logger
+    @pid_output = STDOUT
+    @error_output = STDERR
+    @args = args
+    @config = config
+    @pid_path = File.expand_path("pid_for_#{Process.pid}", @config['log_dir'])
+    @timeout       = (config[:timeout      ] || ENV["MM_SYSTEM_AGENT_RUN_TIMEOUT"      ] || 600).to_i # seconds
+    @timeout_alert = (config[:timeout_alert] || ENV["MM_SYSTEM_AGENT_RUN_TIMEOUT_ALERT"] || 30 ).to_i # seconds
+  end
+
+  def process
+    validate_environment
+    line = nil
+    process_spawned = false
+    begin
+      timeout(@timeout) do #タイムアウト(秒)
+        @logger.info("watchdog process spawning for #{@args.join(' ')}")
+        pid = spawn_watchdog # watchdogプロセスをspawnで起動
+        @logger.info("watchdog process spawned. PID: #{pid.inspect}")
+        File.open(@pid_path, "r") do |f|
+          sleep(0.1) until line = f.gets
+          process_spawned = true
+          @logger.info("watchdog process returned first result: #{line.inspect}")
+          if line =~ /\A\d+\n?\Z/ # 数字と改行のみで構成されるならそれはPIDのはず。
+            @pid_output.puts(line.strip)
+            @logger.info("return PID: #{pid.inspect}")
+          else
+            f.rewind
+            msg = f.read
+            @logger.error("error occurred:\n#{msg}")
+            @error_output.puts(msg)
+            return false
+          end
+        end
+      end
+    rescue Timeout::Error => e
+      @error_output.puts("[#{e.class.name}] #{e.message}")
+      raise e # raiseしたものはTengineJobAgent::Run.processでloggerに出力されるので、ここでは何もしません
+    end
+  end
+
+  # 引数に@pid_pathを渡してwatchdogを起動します。戻り値は起動したwatchdogのPIDです
+  def spawn_watchdog
+    @logger.info("pid file creating: #{@pid_path}")
+    File.open(@pid_path, "w"){ } # ファイルをクリア
+    @logger.info("pid file created: #{@pid_path}")
+    # http://doc.ruby-lang.org/ja/1.9.2/method/Kernel/m/spawn.html を参考にしています
+    args = @args # + [{:out => stdout_w}] #, :err => stderr_w}]
+    watchdog = File.expand_path("../../bin/tengine_job_agent_watchdog", File.dirname(__FILE__))
+    @logger.info("spawning watchdog: #{@pid_path}")
+    pid = Process.spawn(RbConfig.ruby, watchdog, @pid_path, *args)
+    @logger.info("spawned watchdog: #{pid}")
+    return pid
+  end
+
+  # ジョブ実行時に使用されるRubyが1.8系の場合でもtengine_job_agent_runがエラーを起こさない
+  def validate_environment
+    if RUBY_VERSION >= "1.9.2"
+      @logger.info("RUBY_VERSION is #{RUBY_VERSION}")
+    else
+      raise "RUBY_VERSION must be >= 1.9.2 but was #{RUBY_VERSION}"
+    end
+  end
+
+end

data/lib/tengine_job_agent/watchdog.rb

@@ -0,0 +1,162 @@
+# -*- coding: utf-8 -*-
+require 'tengine_job_agent'
+
+require 'fileutils'
+require 'tempfile'
+require 'tengine_event'
+require 'eventmachine'
+require 'uuid'
+
+class TengineJobAgent::Watchdog
+  include TengineJobAgent::CommandUtils
+
+  def initialize(logger, args, config = {})
+    @uuid = UUID.new.generate
+    @logger = logger
+    @pid_output = config['pid_output'] || STDOUT
+    @pid_path, @program, *@args = *args
+    @config = config
+  end
+
+  def process
+    pid, process_status = nil, nil
+    with_tmp_outs do |stdout, stderr|
+      EM.run do
+        sender.mq_suite.send :ensures, :connection do
+          sender.wait_for_connection do
+            begin
+              pid = spawn_process
+              File.open(@pid_path, "a"){|f| f.puts(pid)} # 起動したPIDを呼び出し元に返す
+              detach_and_wait_process(pid)
+            rescue Exception => e
+              File.open(@pid_path, "a"){|f| f.puts("[#{e.class.name}] #{e.message}")}
+              @logger.error("[#{e.class.name}] #{e.message}")
+              EM.stop
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def spawn_process
+    @logger.info("spawning process " << [@program, @args].flatten.join(" "))
+    options = {
+      :out => @stdout.path,
+      :err => @stderr.path,
+      :pgroup => true}
+    pid = Process.spawn(@program, *(@args + [options]))
+    @logger.info("spawned process PID: #{pid}")
+    return pid
+  end
+
+  def detach_and_wait_process(pid)
+    @logger.info("detaching process PID: #{pid}")
+    fire_heartbeat pid do
+      timer = nil
+      int = @config["heartbeat"]["job"]["interval"]
+      if int and int > 0
+        timer = EM.add_periodic_timer int do
+          fire_heartbeat pid do end # <- rspecを黙らせるための無駄なブロック
+        end
+      end
+      EM.defer(lambda { Process.waitpid2 pid }, lambda {|a|
+        @logger.info("process finished: " << a[1].exitstatus.inspect)
+        EM.cancel_timer timer if timer
+        fire_finished(*a)
+      })
+    end
+  end
+
+  def fire_finished(pid, process_status)
+    exit_status = process_status.exitstatus # killされた場合にnilの可能性がある
+    level_key = exit_status == 0 ? :info : :error
+    @logger.info("fire_finished starting #{pid} #{level_key}(#{exit_status})")
+    event_properties = {
+      "execution_id"     => ENV['MM_SCHEDULE_ID'],
+      "root_jobnet_id"   => ENV['MM_ROOT_JOBNET_ID'],
+      "target_jobnet_id" => ENV['MM_TARGET_JOBNET_ID'],
+      "target_job_id"    => ENV['MM_ACTUAL_JOB_ID'],
+      "pid" => pid,
+      "exit_status" => exit_status,
+      "command"   => [@program, @args].flatten.join(" "),
+    }
+    user_stdout_path = output_filepath("stdout", pid)
+    user_stderr_path = output_filepath("stderr", pid)
+    FileUtils.cp(@stdout.path, user_stdout_path)
+    FileUtils.cp(@stderr.path, user_stderr_path)
+    event_properties[:stdout_log] = user_stdout_path
+    event_properties[:stderr_log] = user_stderr_path
+    if level_key == :error
+      event_properties[:message] =
+        "Job process failed. STDOUT and STDERR were redirected to files.\n" <<
+        "You can see them at '#{user_stdout_path}' and '#{user_stderr_path}'\n" <<
+        "on the server '#{ENV['MM_SERVER_NAME']}'"
+    end
+    sender.fire("finished.process.job.tengine", {
+      :key => @uuid,
+      :level_key => level_key,
+      :source_name => source_name(pid),
+      :sender_name => sender_name,
+      :properties => event_properties,
+    })
+    @logger.info("fire_finished complete")
+    sender.stop
+  end
+
+  def fire_heartbeat pid, &block
+    sender.fire("job.heartbeat.tengine", {
+      :key => @uuid,
+      :level_key => :debug,
+      :sender_name => sender_name,
+      :source_name => source_name(pid),
+      :occurred_at => Time.now,
+      :properties => {
+        "root_jobnet_id"   => ENV['MM_ROOT_JOBNET_ID'],
+        "target_jobnet_id" => ENV['MM_TARGET_JOBNET_ID'],
+        "target_job_id"    => ENV['MM_ACTUAL_JOB_ID'],
+        "pid" => pid,
+        "command"   => [@program, @args].flatten.join(" "),
+      },
+      :keep_connection => true,
+      :retry_count => 0,
+    }, &block)
+    @logger.debug("fire_heartbeat #{pid}")
+  end
+
+  def sender
+    @sender ||= Tengine::Event::Sender.new(@config)
+  end
+
+  private
+  def sender_name
+    @sender_name ||= sprintf "agent:%s/%d/tengine_job_agent", Tengine::Event.host_name, Process.pid
+  end
+
+  def source_name pid
+    sprintf "job:%s/%d/%s/%s", ENV['MM_SERVER_NAME'], pid, ENV['MM_ROOT_JOBNET_ID'], ENV['MM_ACTUAL_JOB_ID']
+  end
+
+  def output_filepath(prefix, pid)
+    File.expand_path("#{prefix}-#{pid}.log", @config['log_dir'])
+  end
+
+  def with_tmp_outs
+    Tempfile.open("stdout-#{Process.pid}.log") do |tmp_stdout|
+      @stdout = tmp_stdout
+      begin
+        Tempfile.open("stderr-#{Process.pid}.log") do |tmp_stderr|
+          @stderr = tmp_stderr
+          begin
+            yield
+          ensure
+            @stderr = nil
+          end
+        end
+      ensure
+        @stdout = nil
+      end
+    end
+  end
+
+end