tengine_job 1.1.0 → 1.2.0

data/lib/tengine/job/connectable.rb

@@ -1,43 +0,0 @@
-# -*- coding: utf-8 -*-
-require 'tengine/job'
-
-require 'tengine_resource'
-
-module Tengine::Job::Connectable
-  extend ActiveSupport::Concern
-
-  included do
-    field :server_name    , :type => String # 接続先となるサーバ名。Tengine::Resource::Server#name を指定します
-    field :credential_name, :type => String # 接続時に必要な認証情報。Tengine::Resource::Credential#name を指定します
-
-    include Tengine::Job::MmCompatibility::Connectable
-
-    def actual_credential_name
-      credential_name || (parent ? parent.actual_credential_name : nil)
-    end
-
-    def actual_server_name
-      server_name || (parent ? parent.actual_server_name : nil)
-    end
-
-    def actual_credential
-      key = actual_credential_name
-      return nil if key.blank?
-      result = Tengine::Resource::Credential.where({:name => key}).first
-      # TODO 使用する例外クラスはこれで良いのか検討
-      raise Mongoid::Errors::DocumentNotFound.new(Tengine::Resource::Credential, key, []) unless result
-      result
-    end
-
-    def actual_server
-      key = actual_server_name
-      return nil if key.blank?
-      result = Tengine::Resource::Server.where({:name => key}).first
-      # TODO 使用する例外クラスはこれで良いのか検討
-      raise Mongoid::Errors::DocumentNotFound.new(Tengine::Resource::Server, key, []) unless result
-      result
-    end
-
-
-  end
-end

data/lib/tengine/job/drivers/jobnet_control_driver.rb

@@ -1,249 +0,0 @@
-# -*- coding: utf-8 -*-
-include Tengine::Core::SafeUpdatable
-
-[
- :'start.jobnet.job.tengine',
- :'success.job.job.tengine',
- :'error.job.job.tengine',
- :'success.jobnet.job.tengine',
- :'error.jobnet.job.tengine',
- :'stop.jobnet.job.tengine',
-].each do |i|
-  ack_policy :after_all_handler_submit, i
-end
-
-
-# ジョブネット制御ドライバ
-driver :jobnet_control_driver do
-
-  on :'start.jobnet.job.tengine' do
-    signal = Tengine::Job::Signal.new(event)
-    root_jobnet = Tengine::Job::RootJobnetActual.find(event[:root_jobnet_id])
-    root_jobnet.update_with_lock do
-      signal.reset
-      target_jobnet = root_jobnet.find_descendant(event[:target_jobnet_id]) || root_jobnet
-      signal.with_paths_backup do
-        target_jobnet.activate(signal)
-      end
-    end
-    signal.execution.with(safe: safemode(Tengine::Job::Execution.collection)).save! if event[:root_jobnet_id] == event[:target_jobnet_id]
-    signal.reservations.each{|r| fire(*r.fire_args)}
-    submit
-  end
-
-  on :'start.jobnet.job.tengine.failed.tengined' do
-    # このイベントは壊れていたからfailedなのかもしれない。多重送信によ
-    # りfailedなのかもしれない。あまりへんな仮定を置かない方が良い。
-    e = event
-    f = e.properties           or next
-    g = f["original_event"]    or next
-    h = g["properties"]        or next
-    i = h["root_jobnet_id"]    or next
-    j = h["target_jobnet_id"]  or next
-    k = Tengine::Job::RootJobnetActual.find(i) or next
-
-    k.update_with_lock do
-      l = k.find_descendant(j) || k
-      l.phase_key = :stuck
-    end
-  end
-
-  on :'success.job.job.tengine' do
-    signal = Tengine::Job::Signal.new(event)
-    root_jobnet = Tengine::Job::RootJobnetActual.find(event[:root_jobnet_id])
-    root_jobnet.update_with_lock do
-      signal.reset
-      target_job = root_jobnet.find_descendant(event[:target_job_id])
-      signal.with_paths_backup do
-        edge = target_job.next_edges.first
-        edge.transmit(signal)
-      end
-    end
-    # (*1)
-    signal.reservations.each{|r| fire(*r.fire_args)}
-    submit
-  end
-
-  on :'success.job.job.tengine.failed.tengined' do
-    # このイベントは壊れていたからfailedなのかもしれない。多重送信によ
-    # りfailedなのかもしれない。あまりへんな仮定を置かない方が良い。
-    e = event
-    f = e.properties           or next
-    g = f["original_event"]    or next
-    h = g["properties"]        or next
-    i = h["root_jobnet_id"]    or next
-    j = h["target_jobnet_id"]  or next
-    k = h["target_job_id"]     or next
-    l = Tengine::Job::RootJobnetActual.find(i) or next
-
-    # 上記(*1)のポイントでtenginedが落ちた時のことを考えると、後続のエッ
-    # ジはもうtransmitしているが送信すべきイベントが欠けている状態であ
-    # るので、この場合このジョブがおかしくなっているというよりむしろジョ
-    # ブネット全体がおかしくなっているというべきである。
-    l.update_with_lock do
-      m = l.find_descendant(j)  || l
-      n = m.find_descendant(k)
-      o = n.parent || n
-      o.phase_key = :stuck
-    end
-  end
-
-  on :'error.job.job.tengine' do
-    signal = Tengine::Job::Signal.new(event)
-    root_jobnet = Tengine::Job::RootJobnetActual.find(event[:root_jobnet_id])
-    root_jobnet.update_with_lock do
-      signal.reset
-      target_job = root_jobnet.vertex(event[:target_job_id])
-      signal.with_paths_backup do
-        edge = target_job.next_edges.first
-        edge.close_followings
-        edge.transmit(signal)
-      end
-      # target_jobnet = target_job.parent
-      # target_jobnet.jobnet_fail(signal)
-    end
-    signal.reservations.each{|r| fire(*r.fire_args)}
-    submit
-  end
-
-  on :'error.job.job.tengine.failed.tengined' do
-    # このイベントは壊れていたからfailedなのかもしれない。多重送信によ
-    # りfailedなのかもしれない。あまりへんな仮定を置かない方が良い。
-    e = event
-    f = e.properties           or next
-    g = f["original_event"]    or next
-    h = g["properties"]        or next
-    i = h["root_jobnet_id"]    or next
-    j = h["target_jobnet_id"]  or next
-    k = h["target_job_id"]     or next
-    l = Tengine::Job::RootJobnetActual.find(i) or next
-
-    # 同上で、この場合このジョブがおかしくなっているというよりむしろジョ
-    # ブネット全体がおかしくなっているというべきである。
-    l.update_with_lock do
-      m = l.find_descendant(j)  || l
-      n = m.find_descendant(k)
-      o = n.parent || n
-      o.phase_key = :stuck
-    end
-  end
-
-  on :'success.jobnet.job.tengine' do
-    signal = Tengine::Job::Signal.new(event)
-    root_jobnet = Tengine::Job::RootJobnetActual.find(event[:root_jobnet_id])
-    root_jobnet.update_with_lock do
-      signal.reset
-      target_jobnet = root_jobnet.vertex(event[:target_jobnet_id])
-      signal.with_paths_backup do
-        case target_jobnet.jobnet_type_key
-        when :finally then
-          parent = target_jobnet.parent
-          edge = parent.end_vertex.prev_edges.first
-          (edge.closed? || edge.closing?) ?
-            parent.fail(signal) :
-            parent.succeed(signal)
-        else
-          if edge = (target_jobnet.next_edges || []).first
-            edge.transmit(signal)
-          else
-            (target_jobnet.parent || signal.execution).succeed(signal)
-          end
-        end
-      end
-    end
-    signal.execution.with(sage: safemode(Tengine::Job::Execution.collection)).save! if event[:root_jobnet_id] == event[:target_jobnet_id]
-    signal.reservations.each{|r| fire(*r.fire_args)}
-    submit
-  end
-
-  on :'success.jobnet.job.tengine.failed.tengined' do
-    # このイベントは壊れていたからfailedなのかもしれない。多重送信によ
-    # りfailedなのかもしれない。あまりへんな仮定を置かない方が良い。
-    e = event
-    f = e.properties           or next
-    g = f["original_event"]    or next
-    h = g["properties"]        or next
-    i = h["root_jobnet_id"]    or next
-    j = h["target_jobnet_id"]  or next
-    k = Tengine::Job::RootJobnetActual.find(i) or next
-
-    k.update_with_lock do
-      l = k.find_descendant(j) || k
-      l.phase_key = :stuck
-    end
-  end
-
-  on :'error.jobnet.job.tengine' do
-    signal = Tengine::Job::Signal.new(event)
-    root_jobnet = Tengine::Job::RootJobnetActual.find(event[:root_jobnet_id])
-    root_jobnet.update_with_lock do
-      signal.reset
-      target_jobnet = root_jobnet.find_descendant(event[:target_jobnet_id]) || root_jobnet
-      signal.with_paths_backup do
-        case target_jobnet.jobnet_type_key
-        when :finally then
-          target_jobnet.parent.fail(signal)
-        else
-          if edge = (target_jobnet.next_edges || []).first
-            edge.close_followings
-            edge.transmit(signal)
-          else
-            (target_jobnet.parent || signal.execution).fail(signal)
-          end
-        end
-      end
-      # if target_parent = target_jobnet.parent
-      #   target_parent.end_vertex.transmit(signal)
-      # end
-    end
-    signal.execution.with(safe: safemode(Tengine::Job::Execution.collection)).save! if event[:root_jobnet_id] == event[:target_jobnet_id]
-    signal.reservations.each{|r| fire(*r.fire_args)}
-    submit
-  end
-
-  on :'error.jobnet.job.tengine.failed.tengined' do
-    # このイベントは壊れていたからfailedなのかもしれない。多重送信によ
-    # りfailedなのかもしれない。あまりへんな仮定を置かない方が良い。
-    e = event
-    f = e.properties           or next
-    g = f["original_event"]    or next
-    h = g["properties"]        or next
-    i = h["root_jobnet_id"]    or next
-    j = h["target_jobnet_id"]  or next
-    k = Tengine::Job::RootJobnetActual.find(i) or next
-
-    k.update_with_lock do
-      l = k.find_descendant(j) || k
-      l.phase_key = :stuck
-    end
-  end
-
-  on :'stop.jobnet.job.tengine' do
-    signal = Tengine::Job::Signal.new(event)
-    root_jobnet = Tengine::Job::RootJobnetActual.find(event[:root_jobnet_id])
-    root_jobnet.update_with_lock do
-      signal.reset
-      target_jobnet = root_jobnet.find_descendant(event[:target_jobnet_id]) || root_jobnet
-      target_jobnet.stop(signal)
-    end
-    signal.reservations.each{|r| fire(*r.fire_args) }
-    submit
-  end
-
-  on :'stop.jobnet.job.tengine.failed.tengined' do
-    # このイベントは壊れていたからfailedなのかもしれない。多重送信によ
-    # りfailedなのかもしれない。あまりへんな仮定を置かない方が良い。
-    e = event
-    f = e.properties           or next
-    g = f["original_event"]    or next
-    h = g["properties"]        or next
-    i = h["root_jobnet_id"]    or next
-    j = h["target_jobnet_id"]  or next
-    k = Tengine::Job::RootJobnetActual.find(i) or next
-
-    k.update_with_lock do
-      l = k.find_descendant(j) || k
-      l.phase_key = :stuck
-    end
-  end
-end

data/lib/tengine/job/end.rb

@@ -1,32 +0,0 @@
-# -*- coding: utf-8 -*-
-require 'tengine/job'
-
-# ジョブネットの終端を表すVertex。特に状態は持たない。
-class Tengine::Job::End < Tengine::Job::Vertex
-
-  # https://cacoo.com/diagrams/hdLgrzYsTBBpV3Wj#D26C1
-  def transmit(signal)
-    activate(signal)
-  end
-
-  def activate(signal)
-    complete_origin_edge(signal, :except_closed => true)
-    parent = self.parent # Endのparentであるジョブネット
-    parent_finally = parent.finally_vertex
-    if parent_finally && (parent.phase_key != :dying)
-      parent_finally.transmit(signal)
-    else
-      parent.finish(signal) unless parent.phase_key == :stuck
-    end
-  end
-
-  def reset(signal)
-    parent = self.parent # Endのparentであるジョブネット
-    if signal.execution.in_scope?(parent)
-      if f = parent.finally_vertex
-        f.reset(signal)
-      end
-    end
-  end
-
-end

data/lib/tengine/job/expansion.rb

@@ -1,37 +0,0 @@
-# -*- coding: utf-8 -*-
-require 'tengine/job'
-
-# ルートジョブネットを他のジョブネット内に展開するための特殊なテンプレート用Vertex。
-class Tengine::Job::Expansion < Tengine::Job::Job
-  def actual_class
-    Tengine::Job::JobnetActual
-  end
-  def root_jobnet_template
-    unless @root_jobnet_template
-      cond = {:dsl_version => root.dsl_version, :name => name}
-      @root_jobnet_template = Tengine::Job::RootJobnetTemplate.where(cond).first
-    end
-    @root_jobnet_template
-  end
-
-  IGNORED_FIELD_NAMES = (Tengine::Job::Vertex::IGNORED_FIELD_NAMES + %w[name dsl_version jobnet_type_cd version updated_at created_at children edges]).freeze
-
-  def generating_attrs
-    result = super
-    attrs = root_jobnet_template.attributes.dup
-    if template = root_jobnet_template
-      attrs[:template_id] = template.id
-    end
-    attrs.delete_if{|attr, value| IGNORED_FIELD_NAMES.include?(attr)}
-    result.update(attrs)
-    result
-  end
-  def generating_children; root_jobnet_template.children; end
-  def generating_edges; root_jobnet_template.edges; end
-
-  def generate(klass = actual_class)
-    result = super
-    result.was_expansion = true
-    result
-  end
-end

data/lib/tengine/job/fork.rb

@@ -1,6 +0,0 @@
-# -*- coding: utf-8 -*-
-require 'tengine/job'
-
-# 一つのVertexから複数のVertexへSignalを通知する分岐のVertex。
-class Tengine::Job::Fork < Tengine::Job::Junction
-end

data/lib/tengine/job/jobnet.rb

@@ -1,184 +0,0 @@
-# -*- coding: utf-8 -*-
-require 'tengine/job'
-
-require 'selectable_attr'
-
-# ジョブの始端から終端までを持ち、VertexとEdgeを組み合わせてジョブネットを構成することができるVertex。
-# 自身もジョブネットを構成するVertexの一部として扱われる。
-class Tengine::Job::Jobnet < Tengine::Job::Job
-  include Tengine::Core::SelectableAttr
-  include Tengine::Job::ElementSelectorNotation
-  include Tengine::Core::SafeUpdatable
-
-  autoload :Builder, "tengine/job/jobnet/builder"
-  autoload :StateTransition, 'tengine/job/jobnet/state_transition'
-  autoload :JobStateTransition, 'tengine/job/jobnet/job_state_transition'
-  autoload :JobnetStateTransition, 'tengine/job/jobnet/jobnet_state_transition'
-
-  field :script        , :type => String # 実行されるスクリプト(本来Tengine::Job::Scriptが保持しますが、子要素を保持してかつスクリプトを実行するhadoop_job_runもある)
-  field :description   , :type => String # ジョブネットの説明
-  field :jobnet_type_cd, :type => Integer, :default => 1 # ジョブネットの種類。後述の定義を参照してください。
-
-  selectable_attr :jobnet_type_cd do
-    entry 1, :normal        , "normal"
-    entry 2, :finally       , "finally", :alternative => true
-    # entry 3, :recover       , "recover", :alternative => true
-    entry 4, :hadoop_job_run, "hadoop job run"
-    entry 5, :hadoop_job    , "hadoop job"    , :chained_box => true
-    entry 6, :map_phase     , "map phase"     , :chained_box => true
-    entry 7, :reduce_phase  , "reduce phase"  , :chained_box => true
-  end
-  def chained_box?; jobnet_type_entry[:chained_box]; end
-
-  embeds_many :edges, :class_name => "Tengine::Job::Edge", :inverse_of => :owner , :validate => false
-
-  before_validation do |r|
-    r.edges.each do |edge|
-      unless edge.valid?
-        edge.errors.each do |f, error|
-          r.errors.add(:base, "#{edge.name_for_message} #{f.to_s.humanize} #{error}")
-        end
-      end
-    end
-  end
-
-  class << self
-    def by_name(name)
-      where({:name => name}).first
-    end
-  end
-
-  def script_executable?
-    !script.blank?
-  end
-
-  def start_vertex
-    self.children.detect{|child| child.is_a?(Tengine::Job::Start)}
-  end
-
-  def end_vertex
-    self.children.detect{|child| child.is_a?(Tengine::Job::End)}
-  end
-
-  def finally_vertex
-    self.children.detect{|child| child.is_a?(Tengine::Job::Jobnet) && (child.jobnet_type_key == :finally)}
-  end
-  alias_method :finally_jobnet, :finally_vertex
-
-  def with_start
-    self.children << Tengine::Job::Start.new
-    self
-  end
-
-  def prepare_end
-    if self.children.last.is_a?(Tengine::Job::End)
-      _end = self.children.last
-      yield(_end) if block_given?
-    else
-      _end = Tengine::Job::End.new
-      yield(_end) if block_given?
-      self.children << _end
-    end
-    _end
-  end
-
-  def child_by_name(str)
-    case str
-    when '..'      then parent
-    when '.'       then self
-    when 'start'   then start_vertex
-    when 'end'     then end_vertex
-    when 'finally' then finally_vertex
-    else
-      self.children.detect{|c| c.is_a?(Tengine::Job::Job) && (c.name == str)}
-    end
-  end
-
-  def build_edges(auto_sequence, boot_job_names, redirections)
-    if self.children.length == 1 # 最初に追加したStartだけなら。
-      self.children.delete_all
-      return
-    end
-    if auto_sequence || boot_job_names.empty?
-      prepare_end
-      build_sequencial_edges
-    else
-      Builder.new(self, boot_job_names, redirections).process
-    end
-  end
-
-  def build_sequencial_edges
-    self.edges.clear
-    current = nil
-    self.children.each do |child|
-      next if child.is_a?(Tengine::Job::Jobnet) && !!child.jobnet_type_entry[:alternative]
-      if current
-        edge = self.new_edge(current, child)
-        yield(edge) if block_given?
-      end
-      current = child
-    end
-  end
-
-  def new_edge(origin, destination)
-    origin_id = origin.is_a?(Tengine::Job::Vertex) ? origin.id : origin
-    destination_id = destination.is_a?(Tengine::Job::Vertex) ? destination.id : destination
-    edges.new(:origin_id => origin_id, :destination_id => destination_id)
-  end
-
-  def find_descendant_edge(edge_id)
-    edge_id = String(edge_id)
-    visitor = Tengine::Job::Vertex::AnyVisitor.new do |vertex|
-      if vertex.respond_to?(:edges)
-        vertex.edges.detect{|edge| edge.id.to_s == edge_id}
-      else
-        nil
-      end
-    end
-    visitor.visit(self)
-  end
-  alias_method :edge, :find_descendant_edge
-
-  def find_descendant(vertex_id)
-    vertex_id = String(vertex_id)
-    return nil if vertex_id == self.id.to_s
-    vertex(vertex_id)
-  end
-
-  def vertex(vertex_id)
-    vertex_id = String(vertex_id)
-    return self if vertex_id == self.id.to_s
-    visitor = Tengine::Job::Vertex::AnyVisitor.new{|v| vertex_id == v.id.to_s ? v : nil }
-    visitor.visit(self)
-  end
-
-  def find_descendant_by_name_path(name_path)
-    return nil if name_path == self.name_path
-    vertex_by_name_path(name_path)
-  end
-
-  def vertex_by_name_path(name_path)
-    Tengine::Job::NamePath.absolute?(name_path) ?
-      root.vertex_by_absolute_name_path(name_path) :
-      vertex_by_relative_name_path(name_path)
-  end
-
-  def vertex_by_absolute_name_path(name_path)
-    return self if name_path.to_s == self.name_path
-    visitor = Tengine::Job::Vertex::AnyVisitor.new do |vertex|
-      if name_path == (vertex.respond_to?(:name_path) ? vertex.name_path : nil)
-        vertex
-      else
-        nil
-      end
-    end
-    visitor.visit(self)
-  end
-
-  def vertex_by_relative_name_path(name_path)
-    head, tail = *name_path.split(Tengine::Job::NamePath::SEPARATOR, 2)
-    child = child_by_name(head)
-    tail ? child.vertex_by_relative_name_path(tail) : child
-  end
-
-end