tengine_job 0.6.9

data/spec/tengine/job/drivers/job_controll_driver/expansion_spec.rb

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+require 'tengine/rspec'
+
+describe 'job_control_driver' do
+  include Tengine::RSpec::Extension
+
+  target_dsl File.expand_path("../../../../../lib/tengine/job/drivers/job_control_driver.rb", File.dirname(__FILE__))
+  driver :job_control_driver
+
+  shared_examples_for "/rjn0008/rjn0001/j11を実行する際の環境変数" do |dsl_version|
+    it "expansionだったジョブネットよりも上位のジョブの情報は出力されない" do
+      @rjn0001 = @root.vertex_by_name_path("/rjn0008/rjn0001")
+      @j11 = @root.vertex_by_name_path("/rjn0008/rjn0001/j11")
+      @root.phase_key = :running
+      @rjn0001.phase_key = :running
+      @j11.phase_key = :ready
+      @j11.prev_edges.each{|edge| edge.phase_key = :transmitting}
+      @root.save!
+      @root.reload
+      tengine.should_not_fire
+      mock_ssh = mock(:ssh)
+      mock_channel = mock(:channel)
+      Net::SSH.should_receive(:start).
+        with("localhost", an_instance_of(Tengine::Resource::Credential), an_instance_of(Hash)).and_yield(mock_ssh)
+      mock_ssh.should_receive(:open_channel).and_yield(mock_channel)
+      mock_channel.should_receive(:exec) do |*args|
+        args.length.should == 1
+        # args.first.should =~ %r<source \/etc\/profile && export MM_ACTUAL_JOB_ID=[0-9a-f]{24} MM_ACTUAL_JOB_ANCESTOR_IDS=\\"[0-9a-f]{24}\\" MM_FULL_ACTUAL_JOB_ANCESTOR_IDS=\\"[0-9a-f]{24}\\" MM_ACTUAL_JOB_NAME_PATH=\\"/rjn0001/j11\\" MM_ACTUAL_JOB_SECURITY_TOKEN= MM_SCHEDULE_ID=[0-9a-f]{24} MM_SCHEDULE_ESTIMATED_TIME= MM_TEMPLATE_JOB_ID=[0-9a-f]{24} MM_TEMPLATE_JOB_ANCESTOR_IDS=\\"[0-9a-f]{24}\\" && tengine_job_agent_run -- \$HOME/j11\.sh>
+        args.first.should =~ %r<source \/etc\/profile>
+          t_rjn1001 = Tengine::Job::RootJobnetTemplate.find_by_name("rjn0001")
+        t_rjn1001.dsl_version.should == dsl_version
+        t_j11 = t_rjn1001.vertex_by_name_path("/rjn0001/j11")
+        args.first.should =~ %r<MM_TEMPLATE_JOB_ID=#{t_j11.id.to_s}>
+          args.first.should_not =~ %r<MM_TEMPLATE_JOB_ANCESTOR_IDS=\"#{@template.id.to_s};#{t_rjn1001.id.to_s}\">
+          args.first.should =~ %r<MM_TEMPLATE_JOB_ANCESTOR_IDS=\"#{t_rjn1001.id.to_s}\">
+          args.first.should =~ %r<job_test j11>
+      end
+      tengine.receive("start.job.job.tengine", :properties => {
+          :execution_id => @execution.id.to_s,
+          :root_jobnet_id => @root.id.to_s,
+          :target_jobnet_id => @rjn0001.id.to_s,
+          :target_job_id => @j11.id.to_s,
+        })
+      @root.reload
+      @rjn0001 = @root.vertex_by_name_path("/rjn0008/rjn0001")
+      @j11 = @root.vertex_by_name_path("/rjn0008/rjn0001/j11")
+      @root.phase_key = :running
+      @rjn0001.phase_key = :running
+    end
+  end
+
+  # in [rjn0008]
+  # (S1) --e1-->(rjn0001)--e2-->(rjn0002)--e3-->(E1)
+  #
+  # in [rjn0001]
+  # (S1) --e1-->(j11)--e2-->(j12)--e3-->(E1)
+  #
+  # in [rjn0002]
+  #              |--e2-->(j11)--e4-->|
+  # (S1)--e1-->[F1]                [J1]--e6-->(E1)
+  #              |--e3-->(j12)--e5-->|
+  context "rjn0008" do
+    before do
+      Tengine::Core::Setting.delete_all
+      Tengine::Core::Setting.create!(:name => "dsl_version", :value => "1")
+      Tengine::Job::Vertex.delete_all
+      Rjn0001SimpleJobnetBuilder.new.create_template
+      Rjn0002SimpleParallelJobnetBuilder.new.create_template
+      builder = Rjn0008ExpansionFixture.new
+      @template = builder.create_template
+      @root = @template.generate
+      @ctx = builder.context
+      @execution = Tengine::Job::Execution.create!({
+          :root_jobnet_id => @root.id,
+        })
+      @base_props = {
+        :execution_id => @execution.id.to_s,
+        :root_jobnet_id => @root.id.to_s,
+        :target_jobnet_id => @root.id.to_s,
+      }
+    end
+
+    it_should_behave_like "/rjn0008/rjn0001/j11を実行する際の環境変数", "1"
+  end
+
+  context "複数のバージョンのデータがある場合" do
+    before do
+      Tengine::Core::Setting.delete_all
+      Tengine::Core::Setting.create!(:name => "dsl_version", :value => "2")
+      Tengine::Job::Vertex.delete_all
+      Rjn0001SimpleJobnetBuilder.new.tap do |f|
+        f.create_template(:dsl_version => "1")
+        f.create_template(:dsl_version => "2")
+      end
+      Rjn0002SimpleParallelJobnetBuilder.new.tap do |f|
+        f.create_template(:dsl_version => "1")
+        f.create_template(:dsl_version => "2")
+      end
+      builder = Rjn0008ExpansionFixture.new
+      builder.create_template(:dsl_version => "1")
+      @template = builder.create_template(:dsl_version => "2")
+      @root = @template.generate
+      @ctx = builder.context
+      @execution = Tengine::Job::Execution.create!({
+          :root_jobnet_id => @root.id,
+        })
+      @base_props = {
+        :execution_id => @execution.id.to_s,
+        :root_jobnet_id => @root.id.to_s,
+        :target_jobnet_id => @root.id.to_s,
+      }
+    end
+
+    it_should_behave_like "/rjn0008/rjn0001/j11を実行する際の環境変数", '2'
+  end
+
+end
+
+

data/spec/tengine/job/drivers/job_controll_driver/stop_spec.rb

@@ -0,0 +1,159 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+require 'tengine/rspec'
+
+require 'net/ssh'
+
+describe "<BUG>(tengined複数起動)強制停止すると、ステータスが「強制停止済」ではなく「エラー終了」になる" do
+  include Tengine::RSpec::Extension
+
+  driver_path = File.expand_path("../../../../../lib/tengine/job/drivers/job_control_driver.rb", File.dirname(__FILE__))
+
+  #
+  # in [jn0004]
+  #                         |--e3-->(j2)--e5-->|
+  # (S1)--e1-->(j1)--e2-->[F1]                [J1]--e7-->(j4)--e8-->(E1)
+  #                         |--e4-->(j3)--e6-->|
+  #
+  # in [jn0004/finally]
+  # (S2) --e9-->(jn0004_f)-e10-->(E2)
+  #
+  # 現象:
+  # j1を強制停止した際に、プロセスが２つ動いている場合、その片方のプロセスAが
+  # stop.job.job.tengineイベントを受け取りSSHでtengine_job_agent_killを実行します。
+  # その実行の戻り値を得るまでの間に、もう片方のプロセスBが、プロセスが終了して発火される
+  # finished.process.job.tengineイベントを処理すると、stop_reasonがuser_stopでなくなってしまっていました。
+  #
+  context "jn0004" do
+    before do
+      Tengine::Resource::Server.delete_all
+      Tengine::Job::Execution.delete_all
+      Tengine::Job::Vertex.delete_all
+      TestCredentialFixture.test_credential1
+      TestServerFixture.test_server1
+      TestServerFixture.test_server2
+      builder = Rjn0004ParallelJobnetWithFinally.new
+      @root = builder.create_actual
+      @ctx = builder.context
+      @execution = Tengine::Job::Execution.create!({
+          :root_jobnet_id => @root.id,
+        })
+      @base_props = {
+        :execution_id => @execution.id.to_s,
+        :root_jobnet_id => @root.id.to_s,
+        :root_jobnet_name_path => @root.name_path,
+        :target_jobnet_id => @root.id.to_s,
+        :target_jobnet_name_path => @root.name_path,
+      }
+
+      # 2つのプロセスの代わりに、2つのカーネルを別のFiberで動かす
+      @bootstrap1 = Tengine::Core::Bootstrap.new(:tengined => { :load_path => driver_path })
+      @bootstrap1.kernel.tap{|k| k.bind; k.evaluate}
+      @tengine1 = Tengine::RSpec::ContextWrapper.new(@bootstrap1.kernel)
+      #
+      @bootstrap2 = Tengine::Core::Bootstrap.new(:tengined => { :load_path => driver_path })
+      @bootstrap2.kernel.tap{|k| k.bind; k.evaluate}
+      @tengine2 = Tengine::RSpec::ContextWrapper.new(@bootstrap2.kernel)
+    end
+
+    before do
+      @pid = "123"
+      @ctx[:e1].phase_key = :transmitted
+      @ctx[:j1].tap do |j|
+        j.phase_key = :running
+        j.executing_pid = @pid
+      end
+      @root.phase_key = :running
+      @root.version = 4
+      @root.save!
+
+      @f1 = Fiber.new do
+        @tengine1.should_not_fire
+        ssh1 = mock(:ssh1)
+        Net::SSH.should_receive(:start).with("localhost",
+          an_instance_of(Tengine::Resource::Credential),
+          an_instance_of(Hash)).once.and_yield(ssh1)
+        channel1 = mock(:channel1)
+        ssh1.stub(:open_channel).and_yield(channel1)
+        channel1.stub(:exec).with(any_args).and_yield(channel1, true)
+        channel1.should_receive(:on_close) do
+          Tengine.logger.debug( ("!" * 100) << "\non_close: Fiber.yield #{Process.pid} #{__FILE__}##{__LINE__}")
+          Fiber.yield
+        end # on_dataが呼び出される前に止める
+        channel1.stub(:on_data)
+        channel1.stub(:on_extended_data)
+        @tengine1.receive("stop.job.job.tengine", :properties => {
+            :stop_reason => "user_stop",
+            :target_job_id => @ctx.vertex(:j1).id.to_s,
+            :target_job_name_path => @ctx.vertex(:j1).name_path,
+          }.update(@base_props))
+        :end
+      end
+
+      @f2 = Fiber.new do
+        @tengine2.should_fire(:"error.job.job.tengine", {
+            :source_name => @ctx[:j1].name_as_resource,
+            :properties=>{
+              :target_job_id => @ctx.vertex(:j1).id.to_s,
+              :target_job_name_path => @ctx.vertex(:j1).name_path,
+              :exit_status=>nil,
+            }.update(@base_props)
+          })
+        @tengine2.receive("finished.process.job.tengine", :properties => {
+            :pid=>17485,
+            :exit_status=>nil,
+            :target_job_id => @ctx.vertex(:j1).id.to_s,
+            :target_job_name_path => @ctx.vertex(:j1).name_path,
+          }.update(@base_props))
+        :end
+      end
+
+      @j1 = @root.element("j1")
+
+      @root.reload
+      @root.version.should == 4
+      Tengine::Job.test_harness_clear
+    end
+
+    it "tengine_job_agent_killの戻り値の前にfinished.process.job.tengineが来ても強制終了となるべき" do
+      # f1-1.
+      Tengine.logger.info("1" * 100)
+      # Tengine::Job.should_receive(:test_harness).with(1, "before yield in update_with_lock").once
+      # Tengine::Job.should_receive(:test_harness).with(2, "after yield in update_with_lock").once{ Fiber.yield}
+      @f1.resume.should_not == :end
+      @root.reload
+      @root.version.should == 5
+      @root.element("j1").tap do |j|
+        j.phase_key.should == :dying
+        j.executing_pid.should == @pid
+        j.stop_reason.should == "user_stop"
+      end
+
+      # f2
+      Tengine.logger.info("2" * 100)
+      # Tengine::Job.should_receive(:test_harness).with(3, "before yield in update_with_lock").once
+      # Tengine::Job.should_receive(:test_harness).with(4, "after yield in update_with_lock").once{ Fiber.yield}
+      @f2.resume.should == :end
+      @root.reload
+      @root.version.should == 6
+      @root.element("j1").tap do |j|
+        j.phase_key.should == :error
+        j.executing_pid.should == @pid
+        j.stop_reason.should == "user_stop"
+      end
+
+      # f1-2.
+      Tengine.logger.info("3" * 100)
+      # Tengine::Job.should_receive(:test_harness).with(5, "after update_with_lock").once{ Fiber.yield}
+      @f1.resume.should == :end
+      @root.reload
+      @root.version.should == 6
+      @root.element("j1").tap do |j|
+        j.phase_key.should == :error
+        j.executing_pid.should == @pid
+        j.stop_reason.should == "user_stop"
+      end
+    end
+
+  end
+end

data/spec/tengine/job/drivers/job_controll_driver_spec.rb

@@ -0,0 +1,623 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+require 'tengine/rspec'
+
+require 'net/ssh'
+
+describe 'job_control_driver' do
+  include Tengine::RSpec::Extension
+
+  target_dsl File.expand_path("../../../../lib/tengine/job/drivers/job_control_driver.rb", File.dirname(__FILE__))
+  driver :job_control_driver
+
+  context "rjn0001" do
+    before do
+      Tengine::Job::Vertex.delete_all
+      builder = Rjn0001SimpleJobnetBuilder.new
+      @jobnet = builder.create_actual
+      @ctx = builder.context
+      @execution = Tengine::Job::Execution.create!({
+          :root_jobnet_id => @jobnet.id,
+        })
+    end
+
+    context "ジョブの起動イベントを受け取ったら" do
+      it "通常の場合" do
+        @jobnet.phase_key = :starting
+        @ctx.edge(:e1).phase_key = :transmitting
+        @ctx.vertex(:j11).phase_key = :ready
+        @jobnet.save!
+        @jobnet.reload
+        tengine.should_not_fire
+        mock_ssh = mock(:ssh)
+        mock_channel = mock(:channel)
+        Net::SSH.should_receive(:start).
+          with("localhost", an_instance_of(Tengine::Resource::Credential), an_instance_of(Hash)).and_yield(mock_ssh)
+        mock_ssh.should_receive(:open_channel).and_yield(mock_channel)
+        mock_channel.should_receive(:exec) do |*args|
+          args.length.should == 1
+          # args.first.should =~ %r<source \/etc\/profile && export MM_ACTUAL_JOB_ID=[0-9a-f]{24} MM_ACTUAL_JOB_ANCESTOR_IDS=\\"[0-9a-f]{24}\\" MM_FULL_ACTUAL_JOB_ANCESTOR_IDS=\\"[0-9a-f]{24}\\" MM_ACTUAL_JOB_NAME_PATH=\\"/rjn0001/j11\\" MM_ACTUAL_JOB_SECURITY_TOKEN= MM_SCHEDULE_ID=[0-9a-f]{24} MM_SCHEDULE_ESTIMATED_TIME= MM_TEMPLATE_JOB_ID=[0-9a-f]{24} MM_TEMPLATE_JOB_ANCESTOR_IDS=\\"[0-9a-f]{24}\\" && tengine_job_agent_run -- \$HOME/j11\.sh>
+          args.first.should =~ %r<source \/etc\/profile>
+          args.first.should =~ %r<MM_ACTUAL_JOB_ID=[0-9a-f]{24} MM_ACTUAL_JOB_ANCESTOR_IDS=\"[0-9a-f]{24}\" MM_FULL_ACTUAL_JOB_ANCESTOR_IDS=\"[0-9a-f]{24}\" MM_ACTUAL_JOB_NAME_PATH=\"/rjn0001/j11\" MM_ACTUAL_JOB_SECURITY_TOKEN= MM_SCHEDULE_ID=[0-9a-f]{24} MM_SCHEDULE_ESTIMATED_TIME= MM_TEMPLATE_JOB_ID=[0-9a-f]{24} MM_TEMPLATE_JOB_ANCESTOR_IDS=\"[0-9a-f]{24}\">
+          args.first.should =~ %r<job_test j11>
+        end
+        tengine.receive("start.job.job.tengine", :properties => {
+            :execution_id => @execution.id.to_s,
+            :root_jobnet_id => @jobnet.id.to_s,
+            :root_jobnet_name_path => @jobnet.name_path,
+            :target_jobnet_id => @jobnet.id.to_s,
+            :target_jobnet_name_path => @jobnet.name_path,
+            :target_job_id => @ctx.vertex(:j11).id.to_s,
+            :target_job_name_path => @ctx.vertex(:j11).name_path,
+          })
+        @jobnet.reload
+        @ctx.edge(:e1).phase_key.should == :transmitted
+        @ctx.edge(:e2).phase_key.should == :active
+        @ctx.vertex(:j11).phase_key.should == :starting
+      end
+
+      context "starting直前stopによってinitializedになっている場合" do
+        [:starting, :running].each do |root_phase_key|
+
+          it "ルートが#{root_phase_key}" do
+            @jobnet.phase_key = root_phase_key
+            @ctx[:e1].phase_key = :closing
+            @ctx[:e2].phase_key = :closing
+            @ctx[:e3].phase_key = :closing
+            @ctx[:j11].phase_key = :initialized
+            @jobnet.save!
+            @jobnet.reload
+            tengine.should_fire(:"error.jobnet.job.tengine", {
+                :source_name => @ctx[:root].name_as_resource,
+                :properties=>{
+                  :execution_id => @execution.id.to_s,
+                  :root_jobnet_id => @jobnet.id.to_s,
+                  :root_jobnet_name_path => @jobnet.name_path,
+                  :target_jobnet_id => @jobnet.id.to_s,
+                  :target_jobnet_name_path => @jobnet.name_path,
+                }
+              })
+            tengine.receive("start.job.job.tengine", :properties => {
+                :execution_id => @execution.id.to_s,
+                :root_jobnet_id => @jobnet.id.to_s,
+                :root_jobnet_name_path => @jobnet.name_path,
+                :target_jobnet_id => @jobnet.id.to_s,
+                :target_jobnet_name_path => @jobnet.name_path,
+                :target_job_id => @ctx.vertex(:j11).id.to_s,
+                :target_job_name_path => @ctx.vertex(:j11).name_path,
+              })
+            @jobnet.reload
+            @ctx.edge(:e1).phase_key.should == :closing
+            @ctx.edge(:e2).phase_key.should == :closed
+            @ctx.edge(:e3).phase_key.should == :closed
+            @ctx.vertex(:j11).phase_key.should == :initialized
+            @jobnet.phase_key.should == :error
+          end
+        end
+
+      end
+
+      it "存在しないスクリプトを実行しようとした場合、標準エラー出力にエラーメッセージが返されるので、それを保持する" do
+        @jobnet.phase_key = :starting
+        @ctx.edge(:e1).phase_key = :transmitting
+        @ctx.vertex(:j11).phase_key = :ready
+        @jobnet.save!
+        @jobnet.reload
+        mock_ssh = mock(:ssh)
+        Net::SSH.stub(:start).with(any_args).and_yield(mock_ssh)
+        mock_channel = mock(:channel)
+        mock_ssh.stub(:open_channel).and_yield(mock_channel)
+        mock_channel.stub(:exec).with(any_args).and_yield(mock_channel, true)
+        mock_channel.stub(:on_data)
+        mock_channel.should_receive(:on_extended_data).and_yield(mock_channel,
+          "session", "[Errno::ENOENT] No such file or directory - /home/goku/unexist_script.sh")
+        mock_channel.stub(:on_close)
+        tengine.should_fire(:"error.job.job.tengine", {
+            :source_name => @ctx[:j11].name_as_resource,
+            :properties=>{
+              :execution_id => @execution.id.to_s,
+              :root_jobnet_id => @jobnet.id.to_s,
+              :root_jobnet_name_path => @jobnet.name_path,
+              :target_jobnet_id => @jobnet.id.to_s,
+              :target_jobnet_name_path => @jobnet.name_path,
+              :target_job_id => @ctx.vertex(:j11).id.to_s,
+              :target_job_name_path => @ctx.vertex(:j11).name_path,
+              :exit_status=>nil,
+              :message=>"Failure to execute /rjn0001/j11 via SSH: [Errno::ENOENT] No such file or directory - /home/goku/unexist_script.sh"
+            }
+          })
+        tengine.receive("start.job.job.tengine", :properties => {
+            :execution_id => @execution.id.to_s,
+            :root_jobnet_id => @jobnet.id.to_s,
+            :root_jobnet_name_path => @jobnet.name_path,
+            :target_jobnet_id => @jobnet.id.to_s,
+            :target_jobnet_name_path => @jobnet.name_path,
+            :target_job_id => @ctx.vertex(:j11).id.to_s,
+            :target_job_name_path => @ctx.vertex(:j11).name_path,
+          })
+        @jobnet.reload
+        @ctx.edge(:e1).phase_key.should == :transmitted
+        @ctx.edge(:e2).phase_key.should == :active
+        @ctx.vertex(:j11).tap do |job|
+          job.phase_key.should == :error
+          job.error_messages.should == [
+            "[Errno::ENOENT] No such file or directory - /home/goku/unexist_script.sh",
+            "Failure to execute /rjn0001/j11 via SSH: [Errno::ENOENT] No such file or directory - /home/goku/unexist_script.sh"
+          ]
+        end
+        @jobnet.phase_key.should == :running
+      end
+
+    end
+
+
+    it "PIDを取得できたら" do
+      @ctx.edge(:e1).phase_key = :transmitted
+      @ctx.edge(:e2).phase_key = :active
+      @ctx.vertex(:j11).phase_key = :starting
+      @jobnet.save!
+      @jobnet.reload
+      tengine.should_not_fire
+      mock_event = mock(:event)
+      @pid = "123"
+      signal = Tengine::Job::Signal.new(mock_event)
+      signal.data = {:executing_pid => @pid}
+      @ctx.vertex(:j11).ack(signal) # このメソッド内ではsaveされないので、ここでreloadもしません。
+      @ctx.vertex(:j11).executing_pid.should == @pid
+      @ctx.edge(:e1).phase_key.should == :transmitted
+      @ctx.edge(:e2).phase_key.should == :active
+      @ctx.vertex(:j11).phase_key.should == :running
+    end
+
+    test_error_message1 = "Job process failed. STDOUT and STDERR were redirected to files. You can see them at /home/goku/stdout-1234.log and /home/goku/stderr-1234.log on the server test_server1"
+    {
+      :success => ["0", {}],
+      :error => ["1", {
+          :stdout_log => "/home/goku/stdout-1234.log",
+          :stderr_log => "/home/goku/stderr-1234.log",
+          :message => test_error_message1
+        }]
+    }.each do |phase_key, (exit_status, extra_props)|
+      it "ジョブ実行#{phase_key}の通知" do
+        test_key = "test_key.finished.process.job.tengine"
+        Tengine::Core::Event.delete_all(:conditions => {:key => test_key})
+        Tengine::Core::Event.create!(:event_type_name => "job.heartbeat.tengine", :key => test_key)
+        @jobnet.reload
+        j11 = @jobnet.find_descendant_by_name_path("/rjn0001/j11")
+        j11.executing_pid = "123"
+        j11.phase_key = :running
+        j11.previous_edges.length.should == 1
+        j11.previous_edges.first.phase_key = :transmitted
+        @ctx[:root].save!
+        tengine.should_fire(:"#{phase_key}.job.job.tengine",
+          :source_name => @ctx[:j11].name_as_resource,
+          :properties => {
+            :execution_id => @execution.id.to_s,
+            :root_jobnet_id => @jobnet.id.to_s,
+            :root_jobnet_name_path => @jobnet.name_path,
+            :target_jobnet_id => @jobnet.id.to_s,
+            :target_jobnet_name_path => @jobnet.name_path,
+            :target_job_id => @ctx[:j11].id.to_s,
+            :target_job_name_path => @ctx[:j11].name_path,
+            :exit_status => exit_status
+          })
+        tengine.receive(:"finished.process.job.tengine",
+          :key => test_key,
+          :source_name => @ctx[:j11].name_as_resource,
+          :properties => {
+            :execution_id => @execution.id.to_s,
+            :root_jobnet_id => @jobnet.id.to_s,
+            :root_jobnet_name_path => @jobnet.name_path,
+            :target_jobnet_id => @jobnet.id.to_s,
+            :target_jobnet_name_path => @jobnet.name_path,
+            :target_job_id => @ctx[:j11].id.to_s,
+            :target_job_name_path => @ctx[:j11].name_path,
+            :exit_status => exit_status
+          }.merge(extra_props))
+        @jobnet.reload
+        @ctx.edge(:e1).phase_key.should == :transmitted
+        @ctx.edge(:e2).phase_key.should == :active
+        @ctx.vertex(:j11).tap do |j|
+          j.phase_key.should == phase_key
+          j.exit_status.should == exit_status
+          if phase_key == :error
+            j.error_messages.should == [test_error_message1]
+          end
+        end
+      end
+    end
+
+    it "強制停止" do
+      @pid = "123"
+      @jobnet.reload
+      j11 = @jobnet.find_descendant_by_name_path("/rjn0001/j11")
+      j11.executing_pid = @pid
+      j11.phase_key = :running
+      j11.previous_edges.length.should == 1
+      j11.previous_edges.first.phase_key = :transmitted
+      @ctx[:root].save!
+
+      tengine.should_not_fire
+      mock_ssh = mock(:ssh)
+      mock_channel = mock(:channel)
+      Net::SSH.should_receive(:start).
+        with("localhost", an_instance_of(Tengine::Resource::Credential), an_instance_of(Hash)).and_yield(mock_ssh)
+      mock_ssh.should_receive(:open_channel).and_yield(mock_channel)
+      mock_channel.should_receive(:exec) do |*args|
+        interval = Tengine::Job::Killing::DEFAULT_KILLING_SIGNAL_INTERVAL
+        args.length.should == 1
+        args.first.should =~ %r<source \/etc\/profile>
+        args.first.should =~ %r<tengine_job_agent_kill #{@pid} #{interval} KILL$>
+      end
+      tengine.receive(:"stop.job.job.tengine",
+        :source_name => @ctx[:j11].name_as_resource,
+        :properties => {
+          :execution_id => @execution.id.to_s,
+          :root_jobnet_id => @jobnet.id.to_s,
+          :target_jobnet_id => @jobnet.id.to_s,
+          :target_job_id => @ctx[:j11].id.to_s,
+        })
+      @jobnet.reload
+      @ctx.edge(:e1).phase_key.should == :transmitted
+      @ctx.edge(:e2).phase_key.should == :active
+      @ctx.vertex(:j11).tap do |j|
+        j.phase_key.should == :dying
+        j.exit_status.should == nil
+      end
+    end
+
+    it "強制停止(ジョブネット)" do
+      @pid11 = "11"
+      @pid12 = "12"
+      @jobnet.reload
+      j11 = @jobnet.find_descendant_by_name_path("/rjn0001/j11")
+      j11.executing_pid = @pid11
+      j11.phase_key = :success
+      j11.previous_edges.length.should == 1
+      j11.previous_edges.first.phase_key = :transmitted
+      j12 = @jobnet.find_descendant_by_name_path("/rjn0001/j12")
+      j12.executing_pid = @pid12
+      j12.phase_key = :running
+      j12.previous_edges.length.should == 1
+      j12.previous_edges.first.phase_key = :transmitted
+      @ctx[:root].save!
+
+      # phase_key が success の j11 は fireされない
+      tengine.should_not_fire(:"stop.job.job.tengine")
+      # phase_key が running の j12 は fireされる
+      tengine.should_fire(:"stop.job.job.tengine",
+        :source_name => @ctx[:j12].name_as_resource,
+        :properties => {
+          :stop_reason => "user_stop",
+          :target_jobnet_id => @jobnet.id.to_s,
+          :target_jobnet_name_path => "/rjn0001",
+          :target_job_id => @ctx[:j12].id.to_s,
+          :target_job_name_path => "/rjn0001/j12",
+          :execution_id => @execution.id.to_s,
+          :root_jobnet_id => @jobnet.id.to_s,
+          :root_jobnet_name_path => "/rjn0001",
+        })
+      # jobnet に対して強制停止された
+      tengine.receive(:"stop.jobnet.job.tengine",
+        :source_name => @jobnet.name_as_resource,
+        :properties => {
+          :stop_reason => "user_stop",
+          :target_jobnet_id => @jobnet.id.to_s,
+          :target_jobnet_name_path => "/rjn0001",
+          :execution_id => @execution.id.to_s,
+          :root_jobnet_id => @jobnet.id.to_s,
+          :root_jobnet_name_path => "/rjn0001",
+        })
+    end
+
+    it "強制停止(後続のジョブ)" do
+      @pid11 = "11"
+      @pid12 = "12"
+      @jobnet.reload
+      j11 = @jobnet.find_descendant_by_name_path("/rjn0001/j11")
+      j11.executing_pid = @pid11
+      j11.phase_key = :success
+      j11.previous_edges.length.should == 1
+      j11.previous_edges.first.phase_key = :transmitted
+      j12 = @jobnet.find_descendant_by_name_path("/rjn0001/j12")
+      j12.executing_pid = @pid12
+      j12.phase_key = :running
+      j12.previous_edges.length.should == 1
+      j12.previous_edges.first.phase_key = :transmitted
+      @ctx[:root].save!
+
+      mock_ssh = mock(:ssh)
+      mock_channel = mock(:channel)
+      Net::SSH.should_receive(:start).
+        with("localhost", an_instance_of(Tengine::Resource::Credential), an_instance_of(Hash)).and_yield(mock_ssh)
+      mock_ssh.should_receive(:open_channel).and_yield(mock_channel)
+      mock_channel.should_receive(:exec) do |*args|
+        interval = Tengine::Job::Killing::DEFAULT_KILLING_SIGNAL_INTERVAL
+        args.length.should == 1
+        args.first.should =~ %r<source \/etc\/profile>
+        args.first.should =~ %r<tengine_job_agent_kill #{@pid12} #{interval} KILL$>
+      end
+
+      # job12 に対して強制停止
+      tengine.receive(:"stop.job.job.tengine",
+        :source_name => @ctx[:j12].name_as_resource,
+        :properties => {
+          :stop_reason => "user_stop",
+          :target_jobnet_id => @jobnet.id.to_s,
+          :target_jobnet_name_path => "/rjn0001",
+          :target_job_id => @ctx[:j12].id.to_s,
+          :target_job_name_path => "/rjn0001/j12",
+          :execution_id => @execution.id.to_s,
+          :root_jobnet_id => @jobnet.id.to_s,
+          :root_jobnet_name_path => "/rjn0001",
+        })
+      @jobnet.reload
+      @ctx.edge(:e1).phase_key.should == :transmitted
+      @ctx.edge(:e2).phase_key.should == :transmitted
+      @ctx.edge(:e3).phase_key.should == :active
+      @ctx.vertex(:j11).tap do |j|
+        j.phase_key.should == :success
+        j.stop_reason.should == nil
+      end
+      @ctx.vertex(:j12).tap do |j|
+        j.phase_key.should == :dying
+        j.stop_reason.should == "user_stop"
+      end
+    end
+
+
+    if ENV['PASSWORD']
+    context "実際にSSHで接続", :ssh_actual => true do
+      before do
+        resource_fixture = GokuAtEc2ApNortheast.new
+        credential = resource_fixture.goku_ssh_pw
+        credential.auth_values = {:username => ENV['USER'], :password => ENV['PASSWORD']}
+        credential.save!
+        server = resource_fixture.hadoop_master_node
+        server.local_ipv4 = "127.0.0.1"
+        server.save!
+      end
+
+      it do
+        tengine.should_not_fire
+        tengine.receive("start.job.job.tengine", :properties => {
+            :execution_id => @execution.id.to_s,
+            :root_jobnet_id => @jobnet.id.to_s,
+            :target_jobnet_id => @jobnet.id.to_s,
+          })
+        @jobnet.reload
+        j11 = @jobnet.find_descendant_by_name_path("/rjn0001/j11")
+        j11.executing_pid.should_not be_nil
+        j11.exit_status.should == nil
+        j11.phase_key.should == :running
+        j11.previous_edges.length.should == 1
+        j11.previous_edges.first.phase_key.should == :transmitted
+      end
+
+    end
+    end
+  end
+
+  context "再実行" do
+    context "ジョブを再実行" do
+      {
+        false => "後続も実行",
+        true => "スポット再実行"
+      }.each do |spot, caption|
+        context(caption) do
+
+          before do
+            Tengine::Job::Vertex.delete_all
+            builder = Rjn0001SimpleJobnetBuilder.new
+            @root = builder.create_actual
+            @ctx = builder.context
+            @execution = Tengine::Job::Execution.create!({
+                :root_jobnet_id => @root.id,
+                :spot => spot, :retry => true,
+                :target_actual_ids => [@ctx[:j11].id.to_s]
+              })
+            @root.phase_key = :running
+            @ctx[:j11].phase_key = :success
+            @ctx[:j12].phase_key = :error
+            @ctx[:e1].phase_key = :transmitted
+            @ctx[:e2].phase_key = :transmitted
+            @ctx[:e3].phase_key = :active
+          end
+
+          [:initialized, :success, :error, :stuck].each do |phase_key|
+            it "phase_keyが#{phase_key}ならば再実行できるので、startのイベントを発火する" do
+              @ctx[:j11].phase_key = phase_key
+              @root.save!
+              tengine.should_fire(:"start.job.job.tengine", {
+                  :source_name => @ctx[:j11].name_as_resource,
+                  :properties=>{
+                    :execution_id => @execution.id.to_s,
+                    :root_jobnet_name_path => @root.name_path,
+                    :root_jobnet_id => @root.id.to_s,
+                    :target_jobnet_name_path => @root.name_path,
+                    :target_jobnet_id => @root.id.to_s,
+                    :target_job_name_path => @ctx.vertex(:j11).name_path,
+                    :target_job_id => @ctx.vertex(:j11).id.to_s,
+                  }
+                })
+              tengine.receive("restart.job.job.tengine", :properties => {
+                  :execution_id => @execution.id.to_s,
+                  :root_jobnet_id => @root.id.to_s,
+                  :root_jobnet_name_path => @root.name_path,
+                  :target_jobnet_id => @root.id.to_s,
+                  :target_jobnet_name_path => @root.name_path,
+                  :target_job_id => @ctx.vertex(:j11).id.to_s,
+                  :target_job_name_path => @ctx.vertex(:j11).name_path,
+                })
+              @root.reload
+              @root.phase_key.should == :running
+              @ctx.edge(:e1).phase_key.should == :transmitted
+              @ctx.vertex(:j11).phase_key.should == :ready
+              if spot
+                @ctx.vertex(:j12).phase_key.should == :error
+                @ctx.edge(:e2).phase_key.should == :transmitted
+                @ctx.edge(:e3).phase_key.should == :active
+              else
+                @ctx.vertex(:j12).phase_key.should == :initialized
+                @ctx.edge(:e2).phase_key.should == :active
+                @ctx.edge(:e3).phase_key.should == :active
+              end
+            end
+          end
+
+          [:ready, :starting, :running, :dying].each do |phase_key|
+            it "phase_keyが#{phase_key}ならば再実行できず、エラーのイベントを発火する" do
+              @ctx[:j11].phase_key = phase_key
+              @root.save!
+              tengine.should_fire("restart.job.job.tengine.error.tengined").with(any_args)
+              Tengine::Core::Kernel.temp_exception_reporter(:except_test) do
+                tengine.receive("restart.job.job.tengine", :properties => {
+                    :execution_id => @execution.id.to_s,
+                    :root_jobnet_id => @root.id.to_s,
+                    :target_jobnet_id => @root.id.to_s,
+                    :target_job_id => @ctx.vertex(:j11).id.to_s,
+                  })
+              end
+              # 再実行に失敗したのでルートジョブネット以下何も状態は変更されません
+              @root.reload
+              @root.phase_key.should == :running
+              @ctx.edge(:e1).phase_key.should == :transmitted
+              @ctx.vertex(:j11).phase_key.should == phase_key
+            end
+
+          end
+        end
+      end
+
+    end
+
+  end
+
+
+  context "<BUG>同じジョブネットが複数バージョン存在する際、ジョブ実行時にスクリプトに渡される環境変数の「MM_TEMPLATE_JOB_ID」「MM_TEMPLATE_JOB_ANCESTOR_IDS」が実行しているバージョン以外のものがセットされている" do
+    shared_examples_for "最新のバージョンのルートジョブネットを参照する" do |dsl_version|
+
+      it do
+        @root.phase_key = :starting
+        @root.element("prev!j11").phase_key = :transmitting
+        @root.element('j11').phase_key = :ready
+        @root.save!
+        @root.reload
+        tengine.should_not_fire
+        mock_ssh = mock(:ssh)
+        mock_channel = mock(:channel)
+        Net::SSH.should_receive(:start).
+          with("localhost", an_instance_of(Tengine::Resource::Credential), an_instance_of(Hash)).and_yield(mock_ssh)
+        mock_ssh.should_receive(:open_channel).and_yield(mock_channel)
+        mock_channel.should_receive(:exec) do |*args|
+          args.length.should == 1
+          # args.first.should =~ %r<source \/etc\/profile && export MM_ACTUAL_JOB_ID=[0-9a-f]{24} MM_ACTUAL_JOB_ANCESTOR_IDS=\\"[0-9a-f]{24}\\" MM_FULL_ACTUAL_JOB_ANCESTOR_IDS=\\"[0-9a-f]{24}\\" MM_ACTUAL_JOB_NAME_PATH=\\"/rjn0001/j11\\" MM_ACTUAL_JOB_SECURITY_TOKEN= MM_SCHEDULE_ID=[0-9a-f]{24} MM_SCHEDULE_ESTIMATED_TIME= MM_TEMPLATE_JOB_ID=[0-9a-f]{24} MM_TEMPLATE_JOB_ANCESTOR_IDS=\\"[0-9a-f]{24}\\" && tengine_job_agent_run -- \$HOME/j11\.sh>
+          args.first.should =~ %r<source \/etc\/profile>
+          args.first.should =~ %r<MM_ACTUAL_JOB_ID=[0-9a-f]{24} MM_ACTUAL_JOB_ANCESTOR_IDS=\"[0-9a-f]{24}\" MM_FULL_ACTUAL_JOB_ANCESTOR_IDS=\"[0-9a-f]{24}\" MM_ACTUAL_JOB_NAME_PATH=\"/rjn0001/j11\" MM_ACTUAL_JOB_SECURITY_TOKEN= MM_SCHEDULE_ID=[0-9a-f]{24} MM_SCHEDULE_ESTIMATED_TIME= MM_TEMPLATE_JOB_ID=[0-9a-f]{24} MM_TEMPLATE_JOB_ANCESTOR_IDS=\"[0-9a-f]{24}\">
+          @template.dsl_version.should == dsl_version
+          template_job = @template.element("/rjn0001/j11")
+          args.first.should =~ %r<MM_TEMPLATE_JOB_ID=#{template_job.id.to_s}>
+          args.first.should =~ %r<MM_TEMPLATE_JOB_ANCESTOR_IDS=\"#{@template.id.to_s}\">
+          args.first.should =~ %r<job_test j11>
+        end
+        tengine.receive("start.job.job.tengine", :properties => {
+            :execution_id => @execution.id.to_s,
+            :root_jobnet_id => @root.id.to_s,
+            :root_jobnet_name_path => @root.name_path,
+            :target_jobnet_id => @root.id.to_s,
+            :target_jobnet_name_path => @root.name_path,
+            :target_job_id => @root.element('j11').id.to_s,
+            :target_job_name_path => @root.element('j11').name_path,
+          })
+        @root.reload
+        @root.element('prev!j11').phase_key.should == :transmitted
+        @root.element('next!j11').phase_key.should == :active
+        @root.element('j11').phase_key.should == :starting
+      end
+    end
+
+    context "バージョン1つだけ" do
+      before do
+        Tengine::Core::Setting.delete_all
+        Tengine::Core::Setting.create!(:name => "dsl_version", :value => "1")
+        Tengine::Job::Vertex.delete_all
+        Rjn0001SimpleJobnetBuilder.new.tap do |builder|
+          @template = builder.create_template(:dsl_version => "1")
+          @root = @template.generate
+          @ctx = builder.context
+        end
+        @execution = Tengine::Job::Execution.create!({
+            :root_jobnet_id => @root.id,
+          })
+      end
+      it{ @root.template.dsl_version.should == "1" }
+      it_should_behave_like "最新のバージョンのルートジョブネットを参照する", "1"
+    end
+
+    context "バージョン2つ" do
+      before do
+        Tengine::Core::Setting.delete_all
+        Tengine::Core::Setting.create!(:name => "dsl_version", :value => "2")
+        Tengine::Job::Vertex.delete_all
+        Rjn0001SimpleJobnetBuilder.new.tap do |builder|
+          builder.create_template(:dsl_version => "1")
+          @template = builder.create_template(:dsl_version => "2")
+          @root = @template.generate
+          @ctx = builder.context
+        end
+        @execution = Tengine::Job::Execution.create!({
+            :root_jobnet_id => @root.id,
+          })
+      end
+      it{ @root.template.dsl_version.should == "2" }
+      it_should_behave_like "最新のバージョンのルートジョブネットを参照する", "2"
+    end
+
+    context "バージョン10個" do
+      before do
+        Tengine::Core::Setting.delete_all
+        Tengine::Core::Setting.create!(:name => "dsl_version", :value => "10")
+        Tengine::Job::Vertex.delete_all
+        Rjn0001SimpleJobnetBuilder.new.tap do |builder|
+          (1..9).each do |idx|
+            builder.create_template(:dsl_version => idx.to_s)
+          end
+          @template = builder.create_template(:dsl_version => "10")
+          @root = @template.generate
+          @ctx = builder.context
+        end
+        @execution = Tengine::Job::Execution.create!({
+            :root_jobnet_id => @root.id,
+          })
+      end
+      it{ @root.template.dsl_version.should == "10" }
+      it_should_behave_like "最新のバージョンのルートジョブネットを参照する", "10"
+    end
+  end
+
+  context "https://www.pivotaltracker.com/story/show/22624209" do
+    it "stuckにする" do
+      Tengine::Core::Schedule.delete_all
+      Tengine::Job::Vertex.delete_all
+      builder = Rjn0001SimpleJobnetBuilder.new
+      @root = builder.create_actual
+      @ctx = builder.context
+      @execution = Tengine::Job::Execution.create!({
+          :root_jobnet_id => @root.id,
+        })
+      @root.phase_key = :initialized
+      @root.save!
+      EM.run_block do
+        tengine.receive("expired.job.heartbeat.tengine", :properties => {
+            :execution_id => @execution.id.to_s,
+            :root_jobnet_id => @root.id.to_s,
+            :target_job_id => @root.children[1].id.to_s,
+          })
+      end
+      @root.reload
+      @root.children[1].phase_key.should == :stuck
+      @root.phase_key.should_not == :stuck # initialized
+    end
+  end
+
+end