tengine_job 0.6.9 → 0.6.10

data/Gemfile.lock

@@ -61,7 +61,7 @@ GEM
       simplecov-html (~> 0.5.3)
     simplecov-html (0.5.3)
     systemu (2.4.2)
-    tengine_core (0.5.28)
+    tengine_core (0.5.30)
       activemodel (~> 3.1.0)
       activesupport (~> 3.1.0)
       bson (~> 1.5.2)
@@ -72,7 +72,7 @@ GEM
       selectable_attr (~> 0.3.15)
       tengine_event (~> 0.4.0)
       tengine_support (~> 0.3.12)
-    tengine_event (0.4.6)
+    tengine_event (0.4.7)
       activesupport (>= 3.0.0)
       amqp (~> 0.8.0)
       tengine_support (>= 0.3.24)
@@ -83,14 +83,15 @@ GEM
       tengine_core (~> 0.5.23)
       tengine_support (~> 0.3.24)
       wakame-adapters-tengine (~> 0.0.0)
-    tengine_support (0.3.24)
+    tengine_support (0.3.25)
       activesupport (>= 3.0.0)
     tzinfo (0.3.31)
     uuid (2.3.5)
       macaddr (~> 1.0)
-    wakame-adapters-tengine (0.0.7)
+    wakame-adapters-tengine (0.0.1)
       json
-      right_aws (~> 2.1.0)
+      right_aws (= 2.1.0)
+      rspec
     yard (0.7.5)
 
 PLATFORMS

data/VERSION

@@ -1 +1 @@
-0.6.9
+0.6.10

data/lib/tengine/job/drivers/job_control_driver.rb

@@ -1,5 +1,16 @@
 # -*- coding: utf-8 -*-
 
+[
+ :'start.job.job.tengine',
+ :'stop.job.job.tengine',
+ :'finished.process.job.tengine',
+ :'expired.job.heartbeat.tengine',
+ :'restart.job.job.tengine',
+].each do |i|
+  ack_policy :after_all_handler_submit, i
+end
+
+
 # ジョブ制御ドライバ
 driver :job_control_driver do
 
@@ -25,6 +36,26 @@ driver :job_control_driver do
       root_jobnet.update_with_lock(&signal.callback)
     end
     signal.reservations.each{|r| fire(*r.fire_args)}
+    submit
+  end
+
+  on :'start.job.job.tengine.failed.tengined' do
+    # このイベントは壊れていたからfailedなのかもしれない。多重送信によ
+    # りfailedなのかもしれない。あまりへんな仮定を置かない方が良い。
+    e = event
+    f = e.properties           or next
+    g = f["original_event"]    or next
+    h = g["properties"]        or next
+    i = h["root_jobnet_id"]    or next
+    j = h["target_jobnet_id"]  or next
+    k = h["target_job_id"]     or next
+    l = Tengine::Job::RootJobnetActual.find(i) or next
+
+    l.update_with_lock do
+      m = l.find_descendant(j)  || l
+      n = m.find_descendant(k)
+      n.phase_key = :stuck
+    end
   end
 
   on :'stop.job.job.tengine' do
@@ -43,6 +74,26 @@ driver :job_control_driver do
       signal.callback.call
     end
     signal.reservations.each{|r| fire(*r.fire_args)}
+    submit
+  end
+
+  on :'stop.job.job.tengine.failed.tengined' do
+    # このイベントは壊れていたからfailedなのかもしれない。多重送信によ
+    # りfailedなのかもしれない。あまりへんな仮定を置かない方が良い。
+    e = event
+    f = e.properties           or next
+    g = f["original_event"]    or next
+    h = g["properties"]        or next
+    i = h["root_jobnet_id"]    or next
+    j = h["target_jobnet_id"]  or next
+    k = h["target_job_id"]     or next
+    l = Tengine::Job::RootJobnetActual.find(i) or next
+
+    l.update_with_lock do
+      m = l.find_descendant(j)  || l
+      n = m.find_descendant(k)
+      n.phase_key = :stuck
+    end
   end
 
   on :'finished.process.job.tengine' do
@@ -52,9 +103,29 @@ driver :job_control_driver do
     root_jobnet.update_with_lock do
       signal.reset
       job = root_jobnet.find_descendant(event[:target_job_id])
-      job.finish(signal)
+      job.finish(signal) unless job.phase_key == :stuck
     end
     signal.reservations.each{|r| fire(*r.fire_args)}
+    submit
+  end
+
+  on :'finished.process.job.tengine.failed.tengined' do
+    # このイベントは壊れていたからfailedなのかもしれない。多重送信によ
+    # りfailedなのかもしれない。あまりへんな仮定を置かない方が良い。
+    e = event
+    f = e.properties           or next
+    g = f["original_event"]    or next
+    h = g["properties"]        or next
+    i = h["root_jobnet_id"]    or next
+    j = h["target_jobnet_id"]  or next
+    k = h["target_job_id"]     or next
+    l = Tengine::Job::RootJobnetActual.find(i) or next
+
+    l.update_with_lock do
+      m = l.find_descendant(j)  || l
+      n = m.find_descendant(k)
+      n.phase_key = :stuck
+    end
   end
 
   on :'expired.job.heartbeat.tengine' do
@@ -65,18 +136,60 @@ driver :job_control_driver do
         end
       end
     end
+    submit
+  end
+
+  on :'expired.job.heartbeat.tengine.failed.tengined' do
+    # このイベントは壊れていたからfailedなのかもしれない。多重送信によ
+    # りfailedなのかもしれない。あまりへんな仮定を置かない方が良い。
+    e = event
+    f = e.properties           or next
+    g = f["original_event"]    or next
+    h = g["properties"]        or next
+    i = h["root_jobnet_id"]    or next
+    j = h["target_jobnet_id"]  or next
+    k = h["target_job_id"]     or next
+    l = Tengine::Job::RootJobnetActual.find(i) or next
+
+    l.update_with_lock do
+      m = l.find_descendant(j)  || l
+      n = m.find_descendant(k)
+      n.phase_key = :stuck
+    end
   end
 
   on :'restart.job.job.tengine' do
-    signal = Tengine::Job::Signal.new(event)
-    root_jobnet = Tengine::Job::RootJobnetActual.find(event[:root_jobnet_id])
-    root_jobnet.update_with_lock do
-      signal.reset
-      job = root_jobnet.find_descendant(event[:target_job_id])
-      job.reset(signal)
-      job.transmit(signal)
+    begin
+      signal = Tengine::Job::Signal.new(event)
+      root_jobnet = Tengine::Job::RootJobnetActual.find(event[:root_jobnet_id])
+      root_jobnet.update_with_lock do
+        signal.reset
+        job = root_jobnet.find_descendant(event[:target_job_id])
+        job.reset(signal)
+        job.transmit(signal)
+      end
+      signal.reservations.each{|r| fire(*r.fire_args)}
+    ensure
+      submit
     end
-    signal.reservations.each{|r| fire(*r.fire_args)}
   end
 
+  on :'restart.job.job.tengine.failed.tengined' do
+    # このイベントは壊れていたからfailedなのかもしれない。多重送信によ
+    # りfailedなのかもしれない。あまりへんな仮定を置かない方が良い。
+    e = event
+    f = e.properties           or next
+    g = f["original_event"]    or next
+    h = g["properties"]        or next
+    i = h["root_jobnet_id"]    or next
+    j = h["target_jobnet_id"]  or next
+    k = h["target_job_id"]     or next
+    l = Tengine::Job::RootJobnetActual.find(i) or next
+
+    l.update_with_lock do
+      m = l.find_descendant(j)  || l
+      n = m.find_descendant(k)
+      n.phase_key = :stuck
+    end
+  end
 end

data/lib/tengine/job/drivers/job_execution_driver.rb

@@ -1,4 +1,8 @@
 # -*- coding: utf-8 -*-
+include Tengine::Core::SafeUpdatable
+
+ack_policy :after_all_handler_submit, :'start.execution.job.tengine'
+ack_policy :after_all_handler_submit, :'stop.execution.job.tengine'
 
 # ジョブ起動ドライバ
 driver :job_execution_driver do
@@ -11,8 +15,22 @@ driver :job_execution_driver do
       signal.reset
       execution.transmit(signal)
     end
-    execution.save!
+    execution.safely(safemode(Tengine::Job::Execution.collection)).save!
     signal.reservations.each{|r| fire(*r.fire_args)}
+    submit
+  end
+
+  on :'start.execution.job.tengine.failed.tengined' do
+    # このイベントは壊れていたからfailedなのかもしれない。多重送信によ
+    # りfailedなのかもしれない。あまりへんな仮定を置かない方が良い。
+    e = event
+    f = e.properties                     or next
+    g = f["original_event"]              or next
+    h = g["properties"]                  or next
+    i = h["execution_id"]                or next
+    j = Tengine::Job::Execution.find(i)  or next
+
+    j.update_attributes :phase_key => :stuck
   end
 
   on :'stop.execution.job.tengine' do
@@ -23,8 +41,21 @@ driver :job_execution_driver do
       signal.reset
       execution.stop(signal)
     end
-    execution.save!
+    execution.safely(safemode(Tengine::Job::Execution.collection)).save!
     signal.reservations.each{|r| fire(*r.fire_args)}
+    submit
   end
 
+  on :'stop.execution.job.tengine.failed.tengined' do
+    # このイベントは壊れていたからfailedなのかもしれない。多重送信によ
+    # りfailedなのかもしれない。あまりへんな仮定を置かない方が良い。
+    e = event
+    f = e.properties                     or next
+    g = f["original_event"]              or next
+    h = g["properties"]                  or next
+    i = h["execution_id"]                or next
+    j = Tengine::Job::Execution.find(i)  or next
+
+    j.update_attributes :phase_key => :stuck
+  end
 end

data/lib/tengine/job/drivers/jobnet_control_driver.rb

@@ -1,4 +1,17 @@
 # -*- coding: utf-8 -*-
+include Tengine::Core::SafeUpdatable
+
+[
+ :'start.jobnet.job.tengine',
+ :'success.job.job.tengine',
+ :'error.job.job.tengine',
+ :'success.jobnet.job.tengine',
+ :'error.jobnet.job.tengine',
+ :'stop.jobnet.job.tengine',
+].each do |i|
+  ack_policy :after_all_handler_submit, i
+end
+
 
 # ジョブネット制御ドライバ
 driver :jobnet_control_driver do
@@ -13,8 +26,26 @@ driver :jobnet_control_driver do
         target_jobnet.activate(signal)
       end
     end
-    signal.execution.save! if event[:root_jobnet_id] == event[:target_jobnet_id]
+    signal.execution.safely(safemode(Tengine::Job::Execution.collection)).save! if event[:root_jobnet_id] == event[:target_jobnet_id]
     signal.reservations.each{|r| fire(*r.fire_args)}
+    submit
+  end
+
+  on :'start.jobnet.job.tengine.failed.tengined' do
+    # このイベントは壊れていたからfailedなのかもしれない。多重送信によ
+    # りfailedなのかもしれない。あまりへんな仮定を置かない方が良い。
+    e = event
+    f = e.properties           or next
+    g = f["original_event"]    or next
+    h = g["properties"]        or next
+    i = h["root_jobnet_id"]    or next
+    j = h["target_jobnet_id"]  or next
+    k = Tengine::Job::RootJobnetActual.find(i) or next
+
+    k.update_with_lock do
+      l = k.find_descendant(j) || k
+      l.phase_key = :stuck
+    end
   end
 
   on :'success.job.job.tengine' do
@@ -28,7 +59,33 @@ driver :jobnet_control_driver do
         edge.transmit(signal)
       end
     end
+    # (*1)
     signal.reservations.each{|r| fire(*r.fire_args)}
+    submit
+  end
+
+  on :'success.job.job.tengine.failed.tengined' do
+    # このイベントは壊れていたからfailedなのかもしれない。多重送信によ
+    # りfailedなのかもしれない。あまりへんな仮定を置かない方が良い。
+    e = event
+    f = e.properties           or next
+    g = f["original_event"]    or next
+    h = g["properties"]        or next
+    i = h["root_jobnet_id"]    or next
+    j = h["target_jobnet_id"]  or next
+    k = h["target_job_id"]     or next
+    l = Tengine::Job::RootJobnetActual.find(i) or next
+
+    # 上記(*1)のポイントでtenginedが落ちた時のことを考えると、後続のエッ
+    # ジはもうtransmitしているが送信すべきイベントが欠けている状態であ
+    # るので、この場合このジョブがおかしくなっているというよりむしろジョ
+    # ブネット全体がおかしくなっているというべきである。
+    l.update_with_lock do
+      m = l.find_descendant(j)  || l
+      n = m.find_descendant(k)
+      o = n.parent || n
+      o.phase_key = :stuck
+    end
   end
 
   on :'error.job.job.tengine' do
@@ -46,6 +103,29 @@ driver :jobnet_control_driver do
       # target_jobnet.jobnet_fail(signal)
     end
     signal.reservations.each{|r| fire(*r.fire_args)}
+    submit
+  end
+
+  on :'error.job.job.tengine.failed.tengined' do
+    # このイベントは壊れていたからfailedなのかもしれない。多重送信によ
+    # りfailedなのかもしれない。あまりへんな仮定を置かない方が良い。
+    e = event
+    f = e.properties           or next
+    g = f["original_event"]    or next
+    h = g["properties"]        or next
+    i = h["root_jobnet_id"]    or next
+    j = h["target_jobnet_id"]  or next
+    k = h["target_job_id"]     or next
+    l = Tengine::Job::RootJobnetActual.find(i) or next
+
+    # 同上で、この場合このジョブがおかしくなっているというよりむしろジョ
+    # ブネット全体がおかしくなっているというべきである。
+    l.update_with_lock do
+      m = l.find_descendant(j)  || l
+      n = m.find_descendant(k)
+      o = n.parent || n
+      o.phase_key = :stuck
+    end
   end
 
   on :'success.jobnet.job.tengine' do
@@ -71,8 +151,26 @@ driver :jobnet_control_driver do
         end
       end
     end
-    signal.execution.save! if event[:root_jobnet_id] == event[:target_jobnet_id]
+    signal.execution.safely(safemode(Tengine::Job::Execution.collection)).save! if event[:root_jobnet_id] == event[:target_jobnet_id]
     signal.reservations.each{|r| fire(*r.fire_args)}
+    submit
+  end
+
+  on :'success.jobnet.job.tengine.failed.tengined' do
+    # このイベントは壊れていたからfailedなのかもしれない。多重送信によ
+    # りfailedなのかもしれない。あまりへんな仮定を置かない方が良い。
+    e = event
+    f = e.properties           or next
+    g = f["original_event"]    or next
+    h = g["properties"]        or next
+    i = h["root_jobnet_id"]    or next
+    j = h["target_jobnet_id"]  or next
+    k = Tengine::Job::RootJobnetActual.find(i) or next
+
+    k.update_with_lock do
+      l = k.find_descendant(j) || k
+      l.phase_key = :stuck
+    end
   end
 
   on :'error.jobnet.job.tengine' do
@@ -98,10 +196,27 @@ driver :jobnet_control_driver do
       #   target_parent.end_vertex.transmit(signal)
       # end
     end
-    signal.execution.save! if event[:root_jobnet_id] == event[:target_jobnet_id]
+    signal.execution.safely(safemode(Tengine::Job::Execution.collection)).save! if event[:root_jobnet_id] == event[:target_jobnet_id]
     signal.reservations.each{|r| fire(*r.fire_args)}
+    submit
   end
 
+  on :'error.jobnet.job.tengine.failed.tengined' do
+    # このイベントは壊れていたからfailedなのかもしれない。多重送信によ
+    # りfailedなのかもしれない。あまりへんな仮定を置かない方が良い。
+    e = event
+    f = e.properties           or next
+    g = f["original_event"]    or next
+    h = g["properties"]        or next
+    i = h["root_jobnet_id"]    or next
+    j = h["target_jobnet_id"]  or next
+    k = Tengine::Job::RootJobnetActual.find(i) or next
+
+    k.update_with_lock do
+      l = k.find_descendant(j) || k
+      l.phase_key = :stuck
+    end
+  end
 
   on :'stop.jobnet.job.tengine' do
     signal = Tengine::Job::Signal.new(event)
@@ -112,6 +227,23 @@ driver :jobnet_control_driver do
       target_jobnet.stop(signal)
     end
     signal.reservations.each{|r| fire(*r.fire_args) }
+    submit
   end
 
+  on :'stop.jobnet.job.tengine.failed.tengined' do
+    # このイベントは壊れていたからfailedなのかもしれない。多重送信によ
+    # りfailedなのかもしれない。あまりへんな仮定を置かない方が良い。
+    e = event
+    f = e.properties           or next
+    g = f["original_event"]    or next
+    h = g["properties"]        or next
+    i = h["root_jobnet_id"]    or next
+    j = h["target_jobnet_id"]  or next
+    k = Tengine::Job::RootJobnetActual.find(i) or next
+
+    k.update_with_lock do
+      l = k.find_descendant(j) || k
+      l.phase_key = :stuck
+    end
+  end
 end

data/lib/tengine/job/drivers/schedule_driver.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+include Tengine::Core::SafeUpdatable
 
 # スケジュール管理ドライバ
 driver :schedule_driver do
@@ -9,22 +10,71 @@ driver :schedule_driver do
     status = Tengine::Core::Schedule::SCHEDULED
     if exec.actual_base_timeout_alert && !exec.actual_base_timeout_alert.zero?
       t1 = Time.now + (exec.actual_base_timeout_alert * 60.0)
-      Tengine::Core::Schedule.create event_type_name: "alert.execution.job.tengine", scheduled_at: t1, source_name: name, status: status , properties: event.properties
+      Tengine::Core::Schedule.safely(
+        safemode(Tengine::Core::Schedule.collection)
+      ).create(
+        event_type_name: "alert.execution.job.tengine", scheduled_at: t1, source_name: name, status: status , properties: event.properties
+      )
     end
     if exec.actual_base_timeout_termination && !exec.actual_base_timeout_termination.zero?
       t2 = Time.now + (exec.actual_base_timeout_termination * 60.0)
-      Tengine::Core::Schedule.create event_type_name: "stop.execution.job.tengine", scheduled_at: t2, source_name: name, status: status, properties: event.properties.merge(stop_reason: 'timeout')
+      Tengine::Core::Schedule.safely(
+        safemode(Tengine::Core::Schedule.collection)
+      ).create(
+        event_type_name: "stop.execution.job.tengine", scheduled_at: t2, source_name: name, status: status, properties: event.properties.merge(stop_reason: 'timeout')
+      )
     end
+    submit
+  end
+
+  on :'start.execution.job.tengine.failed.tengined' do
+    e = event
+    f = e.properties                     or next
+    g = f["original_event"]              or next
+    h = g["properties"]                  or next
+    i = h["execution_id"]                or next
+
+    orig = Tengine::Core::EventWrapper.new(Tengine::Core::Event.new(g)) # this object shall noe be persisted
+    exec = Tengine::Job::Signal.new(orig).execution
+    name = exec.name_as_resource
+    status = Tengine::Core::Schedule::SCHEDULED
+    if exec.actual_base_timeout_alert && !exec.actual_base_timeout_alert.zero? && Tengine::Core::Schedule.where(event_type_name: "alert.execution.job.tengine", source_name: name).count.zero?
+      t1 = Time.now + (exec.actual_base_timeout_alert * 60.0)
+      Tengine::Core::Schedule.safely(
+        safemode(Tengine::Core::Schedule.collection)
+      ).create(
+        event_type_name: "alert.execution.job.tengine", scheduled_at: t1, source_name: name, status: status , properties: orig.properties
+      )
+    end
+    if exec.actual_base_timeout_termination && !exec.actual_base_timeout_termination.zero? && Tengine::Core::Schedule.where(event_type_name: "stop.execution.job.tengine", source_name: name).count.zero?
+      t2 = Time.now + (exec.actual_base_timeout_termination * 60.0)
+      Tengine::Core::Schedule.safely(
+        safemode(Tengine::Core::Schedule.collection)
+      ).create(
+        event_type_name: "stop.execution.job.tengine", scheduled_at: t2, source_name: name, status: status, properties: orig.properties.merge(stop_reason: 'timeout')
+      )
+    end
+    submit
   end
 
   on :'success.execution.job.tengine' do
     name = Tengine::Job::Signal.new(event).execution.name_as_resource
-    Tengine::Core::Schedule.where(source_name: name, status: Tengine::Core::Schedule::SCHEDULED).update_all(status: Tengine::Core::Schedule::INVALID)
+    Tengine::Core::Schedule.safely(
+      safemode(Tengine::Core::Schedule.collection)
+    ).where(
+      source_name: name, status: Tengine::Core::Schedule::SCHEDULED
+    ).update_all(status: Tengine::Core::Schedule::INVALID)
+    submit
   end
 
   on :'error.execution.job.tengine' do
     name = Tengine::Job::Signal.new(event).execution.name_as_resource
-    Tengine::Core::Schedule.where(source_name: name, status: Tengine::Core::Schedule::SCHEDULED).update_all(status: Tengine::Core::Schedule::INVALID)
+    Tengine::Core::Schedule.safely(
+      safemode(Tengine::Core::Schedule.collection)
+    ).where(
+      source_name: name, status: Tengine::Core::Schedule::SCHEDULED
+    ).update_all(status: Tengine::Core::Schedule::INVALID)
+    submit
   end
 
 end

data/lib/tengine/job/end.rb

@@ -16,7 +16,7 @@ class Tengine::Job::End < Tengine::Job::Vertex
     if parent_finally && (parent.phase_key != :dying)
       parent_finally.transmit(signal)
     else
-      parent.finish(signal)
+      parent.finish(signal) unless parent.phase_key == :stuck
     end
   end
 

data/lib/tengine/job/jobnet.rb

@@ -8,6 +8,7 @@ require 'selectable_attr'
 class Tengine::Job::Jobnet < Tengine::Job::Job
   include Tengine::Core::SelectableAttr
   include Tengine::Job::ElementSelectorNotation
+  include Tengine::Core::SafeUpdatable
 
   autoload :Builder, "tengine/job/jobnet/builder"
   autoload :StateTransition, 'tengine/job/jobnet/state_transition'

data/lib/tengine/job/root_jobnet_actual.rb

@@ -18,7 +18,7 @@ class Tengine::Job::RootJobnetActual < Tengine::Job::JobnetActual
     result = Tengine::Job::Execution.new(options)
     result.target_actual_ids ||= []
     result.target_actual_ids += args.flatten
-    result.save!
+    result.safely(safemode(Tengine::Job::Execution.collection)).save!
     sender.wait_for_connection do
       sender.fire(:'start.execution.job.tengine', :properties => {
           :execution_id => result.id.to_s

data/lib/tengine/job/root_jobnet_template.rb

@@ -22,10 +22,12 @@ class Tengine::Job::RootJobnetTemplate < Tengine::Job::JobnetTemplate
   def execute(options = {})
     event_sender = options.delete(:sender) || Tengine::Event.default_sender
     actual = generate
-    actual.save!
-    result = Tengine::Job::Execution.create!(
-      (options || {}).update(:root_jobnet_id => actual.id)
-      )
+    actual.safely(safemode(actual.class.collection)).save!
+    result = Tengine::Job::Execution.safely(
+                safemode(Tengine::Job::Execution.collection)
+             ).create!(
+               (options || {}).update(:root_jobnet_id => actual.id)
+             )
     event_sender.fire(:"start.execution.job.tengine", :properties => {
         :execution_id => result.id,
         :root_jobnet_id => actual.id,

data/spec/spec_helper.rb

@@ -42,6 +42,8 @@ Tengine::Core.stderr_logger = Tengine.logger
 
 Tengine::Core::Kernel.event_exception_reporter = :raise_all
 
+Tengine::Core::Config::Core::Tengined.default_cache_drivers = true
+
 RSpec.configure do |config|
   config.include Factory::Syntax::Methods
 

data/spec/tengine/job/drivers/job_controll_driver_spec.rb

@@ -227,6 +227,27 @@ describe 'job_control_driver' do
       end
     end
 
+    it "stuckからのfinished.process.job.tengine" do
+      @jobnet.reload
+      j11 = @jobnet.find_descendant_by_name_path("/rjn0001/j11")
+      j11.phase_key = :stuck
+      j11.previous_edges.first.phase_key = :transmitted
+      @ctx[:root].save!
+      tengine.receive(:"finished.process.job.tengine",
+         :properties => {
+           :execution_id => @execution.id.to_s,
+           :root_jobnet_id => @jobnet.id.to_s,
+           :root_jobnet_name_path => @jobnet.name_path,
+           :target_jobnet_id => @jobnet.id.to_s,
+           :target_jobnet_name_path => @jobnet.name_path,
+           :target_job_id => @ctx[:j11].id.to_s,
+           :target_job_name_path => @ctx[:j11].name_path,
+           :exit_status => 0
+         })
+      @jobnet.reload
+      @ctx.vertex(:j11).phase_key.should == :stuck
+    end
+
     it "強制停止" do
       @pid = "123"
       @jobnet.reload
@@ -620,4 +641,100 @@ describe 'job_control_driver' do
     end
   end
 
+  context "start.job.job.tengine.failed.tengined" do
+    it "stuckにする" do
+      Tengine::Core::Schedule.delete_all
+      Tengine::Job::Vertex.delete_all
+      builder = Rjn0001SimpleJobnetBuilder.new
+      @root = builder.create_actual
+      @ctx = builder.context
+      @execution = Tengine::Job::Execution.create!({
+          :root_jobnet_id => @root.id,
+        })
+      @root.phase_key = :initialized
+      @root.save!
+      EM.run_block do
+        tengine.receive("start.job.job.tengine.failed.tengined", :properties => {
+          :original_event => {
+            :event_type_name => "start.job.job.tengine",
+            :properties => {
+              :execution_id => @execution.id.to_s,
+              :root_jobnet_id => @root.id.to_s,
+              :root_jobnet_name_path => @root.name_path,
+              :target_jobnet_id => @root.id.to_s,
+              :target_jobnet_name_path => @root.name_path,
+              :target_job_id => @root.children[1].id.to_s,
+            }}})
+      end
+      @root.reload
+      @root.children[1].phase_key.should == :stuck
+      @root.phase_key.should_not == :stuck # initialized
+    end
+
+    it "broken event" do
+      Tengine::Core::Schedule.delete_all
+      Tengine::Job::Vertex.delete_all
+      builder = Rjn0001SimpleJobnetBuilder.new
+      @root = builder.create_actual
+      @ctx = builder.context
+      @execution = Tengine::Job::Execution.create!({
+          :root_jobnet_id => @root.id,
+        })
+      @root.phase_key = :initialized
+      @root.save!
+      EM.run_block do
+        tengine.receive("start.job.job.tengine.failed.tengined", :properties => {
+          :original_event => {
+            :event_type_name => "start.job.job.tengine",
+            :properties => {
+              :execution_id => @execution.id.to_s,
+              :root_jobnet_id => @root.id.to_s,
+              :root_jobnet_name_path => @root.name_path,
+              :target_jobnet_id => @root.id.to_s,
+              :target_jobnet_name_path => @root.name_path,
+            }}})
+      end
+      @root.reload
+      @root.children[1].phase_key.should == :initialized
+      @root.phase_key.should_not == :stuck # initialized
+    end
+  end
+
+  %w[
+     stop.job.job.tengine.failed.tengined
+     finished.process.job.tengine.failed.tengined
+     expired.job.heartbeat.tengine.failed.tengined
+     restart.job.job.tengine.failed.tengined
+  ].each do |i|
+    describe i do
+      it "stuckにする" do
+        Tengine::Core::Schedule.delete_all
+        Tengine::Job::Vertex.delete_all
+        builder = Rjn0001SimpleJobnetBuilder.new
+        @root = builder.create_actual
+        @ctx = builder.context
+        @execution = Tengine::Job::Execution.create!({
+          :root_jobnet_id => @root.id,
+        })
+        @root.phase_key = :running
+        @root.save!
+        EM.run_block do
+          tengine.receive(i, :properties => {
+            :original_event => {
+              :event_type_name => "start.job.job.tengine",
+              :properties => {
+                :execution_id => @execution.id.to_s,
+                :root_jobnet_id => @root.id.to_s,
+                :root_jobnet_name_path => @root.name_path,
+                :target_jobnet_id => @root.id.to_s,
+                :target_jobnet_name_path => @root.name_path,
+                :target_job_id => @root.children[1].id.to_s,
+              }}})
+        end
+        @root.reload
+        @root.children[1].phase_key.should == :stuck
+        @root.phase_key.should_not == :stuck # running
+      end
+    end
+  end
 end

data/spec/tengine/job/drivers/job_execution_driver_spec.rb

@@ -48,6 +48,38 @@ describe 'job_execution_driver' do
       @root.phase_key.should == :ready
     end
 
+    it "start.execution.job.tengine.failed.tengined, double save" do
+      @execution.phase_key = :initialized
+      @execution.save!
+      @root.phase_key = :initialized
+      @root.save!
+      tengine.receive("start.execution.job.tengine.failed.tengined", :properties => {
+          :original_event => {
+            :event_type_name => "start.execution.job.tengine",
+            :properties => {
+              :execution_id => @execution.id.to_s,
+              :root_jobnet_id => @root.id.to_s,
+              :root_jobnet_name_path => @root.name_path,
+              :target_jobnet_id => @root.id.to_s,
+              :target_jobnet_name_path => @root.name_path,
+            }}})
+      @execution.reload
+      @execution.phase_key.should == :stuck
+    end
+
+    it "start.execution.job.tengine.failed.tengined, broken event" do
+      @execution.phase_key = :initialized
+      @execution.save!
+      @root.phase_key = :initialized
+      @root.save!
+      tengine.receive("start.execution.job.tengine.failed.tengined", :properties => {
+          :original_event => {
+            :event_type_name => "start.execution.job.tengine",
+            :properties => {}}})
+      @execution.reload
+      @execution.phase_key.should_not == :stuck
+    end
+
     %w[user_stop timeout].each do |stop_reason|
       context stop_reason do
         it "強制停止イベントを受け取ったら" do
@@ -83,6 +115,24 @@ describe 'job_execution_driver' do
       end
     end
     # jobnet_control_driverでexecution起動後の処理を行っています
-  end
 
+    it "stop.execution.job.tengine.failed.tengined" do
+      @execution.phase_key = :running
+      @execution.save!
+      @root.phase_key = :running
+      @root.save!
+      tengine.receive("stop.execution.job.tengine.failed.tengined", :properties => {
+          :original_event => {
+            :event_type_name => "STOP.execution.job.tengine",
+            :properties => {
+              :execution_id => @execution.id.to_s,
+              :root_jobnet_id => @root.id.to_s,
+              :root_jobnet_name_path => @root.name_path,
+              :target_jobnet_id => @root.id.to_s,
+              :target_jobnet_name_path => @root.name_path,
+            }}})
+      @execution.reload
+      @execution.phase_key.should == :stuck
+    end
+  end
 end

data/spec/tengine/job/drivers/jobnet_control_driver_spec.rb

@@ -177,6 +177,16 @@ describe 'job_control_driver' do
         @root.phase_key.should == :error
         @root.finished_at.utc.iso8601.should == @now.utc.iso8601
       end
+
+      it "上位のジョブネットがstuckしていた場合" do
+        @root.phase_key = :stuck
+        @root.save!
+        tengine.receive("success.job.job.tengine", :properties => {
+            :target_job_id => @ctx[:j12].id.to_s
+          }.update(@base_props))
+        @root.reload
+        @root.phase_key.should == :stuck
+      end
     end
 
   end
@@ -443,4 +453,156 @@ describe 'job_control_driver' do
 
   end
 
+  context "start.jobnet.job.tengine.failed.tengined" do
+    it "stuckにする" do
+      Tengine::Core::Schedule.delete_all
+      Tengine::Job::Vertex.delete_all
+      builder = Rjn0001SimpleJobnetBuilder.new
+      @root = builder.create_actual
+      @ctx = builder.context
+      @execution = Tengine::Job::Execution.create!({
+          :root_jobnet_id => @root.id,
+        })
+      @root.phase_key = :initialized
+      @root.save!
+      EM.run_block do
+        tengine.receive("start.jobnet.job.tengine.failed.tengined", :properties => {
+          :original_event => {
+            :event_type_name => "start.jobnet.job.tengine",
+            :properties => {
+              :execution_id => @execution.id.to_s,
+              :root_jobnet_id => @root.id.to_s,
+              :root_jobnet_name_path => @root.name_path,
+              :target_jobnet_id => @root.id.to_s,
+              :target_jobnet_name_path => @root.name_path,
+            }}})
+      end
+      @root.reload
+      @root.phase_key.should == :stuck
+    end
+
+    it "broken event" do
+      Tengine::Core::Schedule.delete_all
+      Tengine::Job::Vertex.delete_all
+      builder = Rjn0001SimpleJobnetBuilder.new
+      @root = builder.create_actual
+      @ctx = builder.context
+      @execution = Tengine::Job::Execution.create!({
+          :root_jobnet_id => @root.id,
+        })
+      @root.phase_key = :initialized
+      @root.save!
+      EM.run_block do
+        tengine.receive("start.jobnet.job.tengine.failed.tengined", :properties => {
+          :original_event => {
+            :event_type_name => "start.jobnet.job.tengine",
+            :properties => {
+              :execution_id => @execution.id.to_s,
+              :root_jobnet_id => @root.id.to_s,
+              :root_jobnet_name_path => @root.name_path,
+              :target_job_id => @root.id.to_s,
+              :target_jobnet_name_path => @root.name_path,
+            }}})
+      end
+      @root.reload
+      @root.children[1].phase_key.should == :initialized
+      @root.phase_key.should_not == :stuck # initialized
+    end
+  end
+
+  context "success.job.job.tengine.failed.tengined" do
+    it "stuckにする(ただし上位のジョブネットを)" do
+      Tengine::Core::Schedule.delete_all
+      Tengine::Job::Vertex.delete_all
+      builder = Rjn0001SimpleJobnetBuilder.new
+      @root = builder.create_actual
+      @ctx = builder.context
+      @execution = Tengine::Job::Execution.create!({
+          :root_jobnet_id => @root.id,
+        })
+      @root.phase_key = :initialized
+      @root.save!
+      EM.run_block do
+        tengine.receive("success.job.job.tengine.failed.tengined", :properties => {
+          :original_event => {
+            :event_type_name => "success.job.job.tengine",
+            :properties => {
+              :execution_id => @execution.id.to_s,
+              :root_jobnet_id => @root.id.to_s,
+              :root_jobnet_name_path => @root.name_path,
+              :target_jobnet_id => @root.id.to_s,
+              :target_jobnet_name_path => @root.name_path,
+              :target_job_id => @root.children[1].id.to_s,
+            }}})
+      end
+      @root.reload
+      @root.phase_key.should == :stuck
+    end
+  end
+
+  context "error.job.job.tengine.failed.tengined" do
+    it "stuckにする(ただし上位のジョブネットを)" do
+      Tengine::Core::Schedule.delete_all
+      Tengine::Job::Vertex.delete_all
+      builder = Rjn0001SimpleJobnetBuilder.new
+      @root = builder.create_actual
+      @ctx = builder.context
+      @execution = Tengine::Job::Execution.create!({
+          :root_jobnet_id => @root.id,
+        })
+      @root.phase_key = :initialized
+      @root.save!
+      EM.run_block do
+        tengine.receive("error.job.job.tengine.failed.tengined", :properties => {
+          :original_event => {
+            :event_type_name => "error.job.job.tengine",
+            :properties => {
+              :execution_id => @execution.id.to_s,
+              :root_jobnet_id => @root.id.to_s,
+              :root_jobnet_name_path => @root.name_path,
+              :target_jobnet_id => @root.id.to_s,
+              :target_jobnet_name_path => @root.name_path,
+              :target_job_id => @root.children[1].id.to_s,
+            }}})
+      end
+      @root.reload
+      @root.phase_key.should == :stuck
+    end
+  end
+
+  
+  %w[
+    success.jobnet.job.tengine.failed.tengined
+    error.jobnet.job.tengine.failed.tengined
+    stop.jobnet.job.tengine.failed.tengined
+  ].each do |i|
+    describe i do
+      it "stuckにする" do
+        Tengine::Core::Schedule.delete_all
+        Tengine::Job::Vertex.delete_all
+        builder = Rjn0001SimpleJobnetBuilder.new
+        @root = builder.create_actual
+        @ctx = builder.context
+        @execution = Tengine::Job::Execution.create!({
+          :root_jobnet_id => @root.id,
+        })
+        @root.phase_key = :initialized
+        @root.save!
+        EM.run_block do
+          tengine.receive(i, :properties => {
+            :original_event => {
+              :event_type_name => "start.jobnet.job.tengine",
+              :properties => {
+                :execution_id => @execution.id.to_s,
+                :root_jobnet_id => @root.id.to_s,
+                :root_jobnet_name_path => @root.name_path,
+                :target_jobnet_id => @root.id.to_s,
+                :target_jobnet_name_path => @root.name_path,
+              }}})
+        end
+        @root.reload
+        @root.phase_key.should == :stuck
+      end
+    end
+  end
 end

data/spec/tengine/job/drivers/schedule_driver_spec.rb

@@ -99,6 +99,45 @@ describe 'schedule_driver' do
           'stop_reason'=>'timeout'
         }
       end
+
+      it "tenginedクラッシュからの復帰" do
+        Tengine::Core::Schedule.delete_all
+        @execution.phase_key = :initialized
+        @execution.actual_base_timeout_alert = 32768
+        @execution.actual_base_timeout_termination = 65536
+        @execution.save!
+        @root.phase_key = :initialized
+        @root.save!
+        EM.run_block do
+          tengine.receive("start.execution.job.tengine.failed.tengined", :properties => {
+            :original_event => {
+              :event_type_name => "start.execution.job.tengine",
+              :properties => {
+                :execution_id => @execution.id.to_s,
+                :root_jobnet_id => @root.id.to_s,
+                :target_jobnet_id => @root.id.to_s,
+              }}})
+        end
+        Tengine::Core::Schedule.count(:conditions => {:status => Tengine::Core::Schedule::SCHEDULED}).should == 2
+        s1 = Tengine::Core::Schedule.first(:conditions => {:event_type_name => "alert.execution.job.tengine"})
+        s2 = Tengine::Core::Schedule.first(:conditions => {:event_type_name => "stop.execution.job.tengine"})
+        [s1, s2].each do |i|
+          i.source_name.should == @execution.name_as_resource
+          i.scheduled_at.should >= Time.now
+        end
+        s1.properties.should == {
+          'execution_id' => @execution.id.to_s,
+          'root_jobnet_id' => @root.id.to_s,
+          'target_jobnet_id' => @root.id.to_s,
+        }
+        s2.properties.should == {
+          'execution_id' => @execution.id.to_s,
+          'root_jobnet_id' => @root.id.to_s,
+          'target_jobnet_id' => @root.id.to_s,
+          'stop_reason'=>'timeout'
+        }
+        
+      end
     end
 
     shared_examples "terminated" do

data/tengine_job.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "tengine_job"
-  s.version = "0.6.9"
+  s.version = "0.6.10"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["taigou", "totty", "g-morita", "shyouhei", "akm", "guemon"]
-  s.date = "2012-02-09"
+  s.date = "2012-02-15"
   s.description = "tengine_job provides jobnet management"
   s.email = "tengine@nautilus-technologies.com"
   s.extra_rdoc_files = [
@@ -152,7 +152,7 @@ Gem::Specification.new do |s|
   s.homepage = "http://github.com/tengine/tengine_job"
   s.licenses = ["MPL2.0/LGPLv3"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.12"
+  s.rubygems_version = "1.8.15"
   s.summary = "tengine_job provides jobnet management"
 
   if s.respond_to? :specification_version then

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tengine_job
 version: !ruby/object:Gem::Version
-  version: 0.6.9
+  version: 0.6.10
   prerelease: 
 platform: ruby
 authors:
@@ -14,11 +14,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-09 00:00:00.000000000Z
+date: 2012-02-15 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tengine_support
-  requirement: &2162921480 !ruby/object:Gem::Requirement
+  requirement: &70277087175160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -26,10 +26,10 @@ dependencies:
         version: 0.3.12
   type: :runtime
   prerelease: false
-  version_requirements: *2162921480
+  version_requirements: *70277087175160
 - !ruby/object:Gem::Dependency
   name: tengine_core
-  requirement: &2162920200 !ruby/object:Gem::Requirement
+  requirement: &70277087174680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -37,10 +37,10 @@ dependencies:
         version: 0.5.25
   type: :runtime
   prerelease: false
-  version_requirements: *2162920200
+  version_requirements: *70277087174680
 - !ruby/object:Gem::Dependency
   name: tengine_resource
-  requirement: &2162919180 !ruby/object:Gem::Requirement
+  requirement: &70277087174200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -48,10 +48,10 @@ dependencies:
         version: 0.5.4
   type: :runtime
   prerelease: false
-  version_requirements: *2162919180
+  version_requirements: *70277087174200
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2162918160 !ruby/object:Gem::Requirement
+  requirement: &70277087173700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -59,10 +59,10 @@ dependencies:
         version: 2.6.0
   type: :development
   prerelease: false
-  version_requirements: *2162918160
+  version_requirements: *70277087173700
 - !ruby/object:Gem::Dependency
   name: factory_girl
-  requirement: &2162917280 !ruby/object:Gem::Requirement
+  requirement: &70277087173200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -70,10 +70,10 @@ dependencies:
         version: 2.1.2
   type: :development
   prerelease: false
-  version_requirements: *2162917280
+  version_requirements: *70277087173200
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &2162916400 !ruby/object:Gem::Requirement
+  requirement: &70277087172720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -81,10 +81,10 @@ dependencies:
         version: 0.7.2
   type: :development
   prerelease: false
-  version_requirements: *2162916400
+  version_requirements: *70277087172720
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &2162915520 !ruby/object:Gem::Requirement
+  requirement: &70277087172240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -92,10 +92,10 @@ dependencies:
         version: 1.0.18
   type: :development
   prerelease: false
-  version_requirements: *2162915520
+  version_requirements: *70277087172240
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &2162914720 !ruby/object:Gem::Requirement
+  requirement: &70277087171740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -103,10 +103,10 @@ dependencies:
         version: 1.6.4
   type: :development
   prerelease: false
-  version_requirements: *2162914720
+  version_requirements: *70277087171740
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &2162913580 !ruby/object:Gem::Requirement
+  requirement: &70277087171260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -114,10 +114,10 @@ dependencies:
         version: 0.5.3
   type: :development
   prerelease: false
-  version_requirements: *2162913580
+  version_requirements: *70277087171260
 - !ruby/object:Gem::Dependency
   name: ZenTest
-  requirement: &2162912640 !ruby/object:Gem::Requirement
+  requirement: &70277087170780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -125,7 +125,7 @@ dependencies:
         version: 4.6.2
   type: :development
   prerelease: false
-  version_requirements: *2162912640
+  version_requirements: *70277087170780
 description: tengine_job provides jobnet management
 email: tengine@nautilus-technologies.com
 executables: []
@@ -278,9 +278,6 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 2171567257402705046
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -289,8 +286,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.12
+rubygems_version: 1.8.16
 signing_key: 
 specification_version: 3
 summary: tengine_job provides jobnet management
 test_files: []
+has_rdoc: