tengine_job 0.6.9

data/lib/tengine/job/end.rb

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+require 'tengine/job'
+
+# ジョブネットの終端を表すVertex。特に状態は持たない。
+class Tengine::Job::End < Tengine::Job::Vertex
+
+  # https://cacoo.com/diagrams/hdLgrzYsTBBpV3Wj#D26C1
+  def transmit(signal)
+    activate(signal)
+  end
+
+  def activate(signal)
+    complete_origin_edge(signal, :except_closed => true)
+    parent = self.parent # Endのparentであるジョブネット
+    parent_finally = parent.finally_vertex
+    if parent_finally && (parent.phase_key != :dying)
+      parent_finally.transmit(signal)
+    else
+      parent.finish(signal)
+    end
+  end
+
+  def reset(signal)
+    parent = self.parent # Endのparentであるジョブネット
+    if signal.execution.in_scope?(parent)
+      if f = parent.finally_vertex
+        f.reset(signal)
+      end
+    end
+  end
+
+end

data/lib/tengine/job/executable.rb

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+require 'tengine/job'
+require 'selectable_attr'
+
+# ジョブ／ジョブネットを実行する際の情報に関するモジュール
+# Tengine::Job::JobnetActual, Tengine::Job::JobnetTemplateがこのモジュールをincludeします
+module Tengine::Job::Executable
+  extend ActiveSupport::Concern
+
+  class PhaseError < StandardError
+  end
+
+  included do
+    field :phase_cd   , :type => Integer, :default => 20 # 進行状況。とりうる値は以下を参照してください。詳しくは「tengine_jobパッケージ設計書」の「ジョブ／ジョブネット状態遷移」を参照してください
+    field :started_at , :type => Time     # 開始時刻。以前はDateTimeでしたが、実績ベースの予定終了時刻の計算のためにTimeにしました
+    field :finished_at, :type => Time     # 終了時刻。強制終了時にも設定されます。
+
+    include Tengine::Core::SelectableAttr
+    selectable_attr :phase_cd do
+      entry 20, :initialized, 'initialized'
+      entry 30, :ready      , "ready"
+      entry 50, :starting   , "starting"
+      entry 60, :running    , "running"
+      entry 70, :dying      , "dying"
+      entry 40, :success    , "success"
+      entry 80, :error      , "error"
+      entry 90, :stuck      , "stuck"
+    end
+
+    def phase_key=(phase_key)
+      element_type = nil
+      case self.class
+      when Tengine::Job::Execution then element_type = "execution"
+      when Tengine::Job::RootJobnetActual then element_type = "root_jobnet"
+      when Tengine::Job::JobnetActual then element_type = self.script_executable? ? "job" :
+        self.jobnet_type_key == :normal ?  "jobnet" : self.jobnet_type_name
+      end
+      Tengine.logger.debug("#{element_type} phase changed. <#{ self.id.to_s}> #{self.phase_name} -> #{ self.class.phase_name_by_key(phase_key)}")
+      if is_a?(Tengine::Job::JobnetActual)
+        children.each{|child| child.phase_key = phase_key if child.respond_to?(:chained_box?) && child.chained_box?}
+      end
+      self.write_attribute(:phase_cd, self.class.phase_id_by_key(phase_key))
+    end
+
+
+    unless defined?(HUMAN_PHASE_KEYS_HASH)
+      HUMAN_PHASE_KEYS_HASH = {
+        'user_stop' => { :dying => :stopping_by_user, :error => :stopped_by_user }.freeze,
+        'timeout' => { :dying => :stopping_by_timeout, :error => :stopped_by_timeout }.freeze,
+      }.freeze
+
+      HUMAN_PHASE_KEYS_ADDITIONAL = HUMAN_PHASE_KEYS_HASH.values.map{|hash| hash.values}.flatten.freeze
+      # HUMAN_PHASE_KEYS = (phase_keys + HUMAN_PHASE_KEYS_ADDITIONAL).freeze
+    end
+
+    # 可読可能なphase_keyを返します。
+    # 具体的にはphase_keyが:dying、:errorの場合は、stop_reasonを考慮した値を返します。
+    def human_phase_key
+      case phase_key
+      when :dying, :error then
+        hash = HUMAN_PHASE_KEYS_HASH[self.stop_reason]
+        hash ? hash[phase_key] : phase_key
+      else phase_key
+      end
+    end
+
+    # human_phase_keyの表示用の文字列を返します。
+    def human_phase_name
+      I18n.t(human_phase_key, :scope => "selectable_attrs.tengine/job/jobnet_actual.human_phase_name")
+    end
+
+  end
+
+end

data/lib/tengine/job/execution.rb

@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+require 'tengine/core'
+
+class Tengine::Job::Execution
+  include Mongoid::Document
+  include Mongoid::Timestamps
+  include Tengine::Job::Executable
+  include Tengine::Core::CollectionAccessible
+
+  field :target_actual_ids, :type => Array
+  array_text_accessor :target_actual_ids
+
+  field :preparation_command, :type => String
+  field :actual_base_timeout_alert, :type => Integer
+  field :actual_base_timeout_termination, :type => Integer
+  field :estimated_time, :type => Integer
+  field :keeping_stdout, :type => Boolean
+  field :keeping_stderr, :type => Boolean
+
+  # 1.0.0で想定している実行は 再実行でない実行時にスポット実行は想定していないが
+  # そういう利用も考えられるので、対応できるように属性はわけておきます
+  field :retry, :type => Boolean, :default => false # 再実行かどうか
+  field :spot , :type => Boolean, :default => false # スポット実行かどうか
+
+  belongs_to :root_jobnet, :class_name => "Tengine::Job::RootJobnetActual", :index => true, :inverse_of => :executions
+
+  attr_accessor :signal # runを実行して、ackを返す際に一時的にsignalを記録しておく属性です。それ以外には使用しないでください。
+
+  # 実開始日時から求める予定終了時刻
+  def actual_estimated_end
+    return nil unless started_at
+    (started_at + (estimated_time || 0)).utc
+  end
+
+  def name_as_resource
+    root_jobnet.name_as_resource.sub(/^job:/, 'execution:')
+  end
+
+  def target_actuals
+    r = self.root_jobnet
+    if target_actual_ids.nil? || target_actual_ids.empty?
+      [r]
+    else
+      target_actual_ids.map do |target_actual_id|
+        r.vertex(target_actual_id)
+      end
+    end
+  end
+
+  def scope_root
+    unless @scope_root
+      actual = target_actuals.first
+      @scope_root = spot ? actual : actual.parent || actual
+      unless @scope_root
+        raise "@scope_root must not be nil"
+      end
+    end
+    @scope_root
+  end
+
+  def in_scope?(vertex)
+    return false if vertex.nil?
+    return true if target_actual_ids.nil? || target_actual_ids.empty?
+    (vertex.id == scope_root.id) || vertex.ancestors.map(&:id).include?(scope_root.id)
+  end
+
+  def transmit(signal)
+    case phase_key
+    when :initialized then
+      if self.retry
+        target_actuals.each do |target|
+          target.reset(signal)
+        end
+      end
+      self.phase_key = :ready
+      activate(signal)
+    else
+      raise "Unsupported phase_key for transmit: #{phase_key.inspect}"
+    end
+  end
+
+  def activate(signal)
+    case phase_key
+    when :ready then
+      self.phase_key = :starting
+      if self.retry
+        target_actuals.each do |target|
+          target.transmit(signal)
+        end
+      else
+        root_jobnet.transmit(signal)
+      end
+    else
+      raise "Unsupported phase_key for activate: #{phase_key.inspect}"
+    end
+  end
+
+  def ack(signal)
+    case phase_key
+    when :ready then
+      raise Tengine::Job::Executable::PhaseError, "ack not available on #{phase_key.inspect}"
+    when :starting then
+      self.phase_key = :running
+    end
+  end
+
+  def succeed(signal)
+    case phase_key
+    when :initialized, :ready, :error then
+      raise Tengine::Job::Executable::PhaseError, "succeed not available on #{phase_key.inspect}"
+    when :starting, :running, :dying, :stuck then
+      self.phase_key = :success
+      signal.fire(self, :"success.execution.job.tengine")
+    end
+  end
+
+  def fail(signal)
+    case phase_key
+    when :initialized, :ready, :success then
+      raise Tengine::Job::Executable::PhaseError, "fail not available on #{phase_key.inspect}"
+    when :starting, :running, :dying, :stuck then
+      self.phase_key = :error
+      signal.fire(self, :"error.execution.job.tengine")
+    end
+  end
+
+#   def fire_stop(signal)
+#     return if self.phase_key == :initialized
+#     signal.fire(self, :"stop.execution.job.tengine", {
+#         :execution_id => self.id,
+#         :root_jobnet_id => root_jobnet.id,
+#         :target_jobnet_id => root_jobnet.id,
+#       })
+#   end
+
+  def stop(signal)
+    self.phase_key = :dying
+    root_jobnet.fire_stop(signal)
+  end
+
+end

data/lib/tengine/job/expansion.rb

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+require 'tengine/job'
+
+# ルートジョブネットを他のジョブネット内に展開するための特殊なテンプレート用Vertex。
+class Tengine::Job::Expansion < Tengine::Job::Job
+  def actual_class
+    Tengine::Job::JobnetActual
+  end
+  def root_jobnet_template
+    unless @root_jobnet_template
+      cond = {:dsl_version => root.dsl_version, :name => name}
+      @root_jobnet_template = Tengine::Job::RootJobnetTemplate.first(:conditions => cond)
+    end
+    @root_jobnet_template
+  end
+
+  IGNORED_FIELD_NAMES = (Tengine::Job::Vertex::IGNORED_FIELD_NAMES + %w[name dsl_version jobnet_type_cd version updated_at created_at children edges]).freeze
+
+  def generating_attrs
+    result = super
+    attrs = root_jobnet_template.attributes.dup
+    if template = root_jobnet_template
+      attrs[:template_id] = template.id
+    end
+    attrs.delete_if{|attr, value| IGNORED_FIELD_NAMES.include?(attr)}
+    result.update(attrs)
+    result
+  end
+  def generating_children; root_jobnet_template.children; end
+  def generating_edges; root_jobnet_template.edges; end
+
+  def generate(klass = actual_class)
+    result = super
+    result.was_expansion = true
+    result
+  end
+end

data/lib/tengine/job/fork.rb

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+require 'tengine/job'
+
+# 一つのVertexから複数のVertexへSignalを通知する分岐のVertex。
+class Tengine::Job::Fork < Tengine::Job::Junction
+end

data/lib/tengine/job/job.rb

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+require 'tengine/job'
+
+# 処理を意味するVertex。実際に実行を行うTengine::Job::Scriptやジョブネットである
+# Tengine::Job::Jobnetの継承元である。
+class Tengine::Job::Job < Tengine::Job::Vertex
+  include Tengine::Job::Connectable
+  include Tengine::Job::Killing
+
+  field :name, :type => String # ジョブの名称。
+
+  validates :name, :presence => true
+
+  # リソース識別子を返します
+  def name_as_resource
+    @name_as_resource ||= "job:#{Tengine::Event.host_name}/#{Process.pid.to_s}/#{root.id.to_s}/#{id.to_s}"
+  end
+
+  def short_inspect
+    "#<%%%-30s id: %s name: %s>" % [self.class.name, self.id.to_s, name]
+  end
+
+end

data/lib/tengine/job/jobnet.rb

@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+require 'tengine/job'
+
+require 'selectable_attr'
+
+# ジョブの始端から終端までを持ち、VertexとEdgeを組み合わせてジョブネットを構成することができるVertex。
+# 自身もジョブネットを構成するVertexの一部として扱われる。
+class Tengine::Job::Jobnet < Tengine::Job::Job
+  include Tengine::Core::SelectableAttr
+  include Tengine::Job::ElementSelectorNotation
+
+  autoload :Builder, "tengine/job/jobnet/builder"
+  autoload :StateTransition, 'tengine/job/jobnet/state_transition'
+  autoload :JobStateTransition, 'tengine/job/jobnet/job_state_transition'
+  autoload :JobnetStateTransition, 'tengine/job/jobnet/jobnet_state_transition'
+
+  field :script        , :type => String # 実行されるスクリプト(本来Tengine::Job::Scriptが保持しますが、子要素を保持してかつスクリプトを実行するhadoop_job_runもある)
+  field :description   , :type => String # ジョブネットの説明
+  field :jobnet_type_cd, :type => Integer, :default => 1 # ジョブネットの種類。後述の定義を参照してください。
+
+  selectable_attr :jobnet_type_cd do
+    entry 1, :normal        , "normal"
+    entry 2, :finally       , "finally", :alternative => true
+    # entry 3, :recover       , "recover", :alternative => true
+    entry 4, :hadoop_job_run, "hadoop job run"
+    entry 5, :hadoop_job    , "hadoop job"    , :chained_box => true
+    entry 6, :map_phase     , "map phase"     , :chained_box => true
+    entry 7, :reduce_phase  , "reduce phase"  , :chained_box => true
+  end
+  def chained_box?; jobnet_type_entry[:chained_box]; end
+
+  embeds_many :edges, :class_name => "Tengine::Job::Edge", :inverse_of => :owner
+
+  class << self
+    def by_name(name)
+      first(:conditions => {:name => name})
+    end
+  end
+
+  def script_executable?
+    !script.blank?
+  end
+
+  def start_vertex
+    self.children.detect{|child| child.is_a?(Tengine::Job::Start)}
+  end
+
+  def end_vertex
+    self.children.detect{|child| child.is_a?(Tengine::Job::End)}
+  end
+
+  def finally_vertex
+    self.children.detect{|child| child.is_a?(Tengine::Job::Jobnet) && (child.jobnet_type_key == :finally)}
+  end
+  alias_method :finally_jobnet, :finally_vertex
+
+  def with_start
+    self.children << Tengine::Job::Start.new
+    self
+  end
+
+  def prepare_end
+    if self.children.last.is_a?(Tengine::Job::End)
+      _end = self.children.last
+      yield(_end) if block_given?
+    else
+      _end = Tengine::Job::End.new
+      yield(_end) if block_given?
+      self.children << _end
+    end
+    _end
+  end
+
+  def child_by_name(str)
+    case str
+    when '..'      then parent
+    when '.'       then self
+    when 'start'   then start_vertex
+    when 'end'     then end_vertex
+    when 'finally' then finally_vertex
+    else
+      self.children.detect{|c| c.is_a?(Tengine::Job::Job) && (c.name == str)}
+    end
+  end
+
+  def build_edges(auto_sequence, boot_job_names, redirections)
+    if self.children.length == 1 # 最初に追加したStartだけなら。
+      self.children.delete_all
+      return
+    end
+    if auto_sequence || boot_job_names.empty?
+      prepare_end
+      build_sequencial_edges
+    else
+      Builder.new(self, boot_job_names, redirections).process
+    end
+  end
+
+  def build_sequencial_edges
+    self.edges.clear
+    current = nil
+    self.children.each do |child|
+      next if child.is_a?(Tengine::Job::Jobnet) && !!child.jobnet_type_entry[:alternative]
+      if current
+        edge = self.new_edge(current, child)
+        yield(edge) if block_given?
+      end
+      current = child
+    end
+  end
+
+  def new_edge(origin, destination)
+    origin_id = origin.is_a?(Tengine::Job::Vertex) ? origin.id : origin
+    destination_id = destination.is_a?(Tengine::Job::Vertex) ? destination.id : destination
+    edges.new(:origin_id => origin_id, :destination_id => destination_id)
+  end
+
+  def find_descendant_edge(edge_id)
+    edge_id = String(edge_id)
+    visitor = Tengine::Job::Vertex::AnyVisitor.new do |vertex|
+      if vertex.respond_to?(:edges)
+        vertex.edges.detect{|edge| edge.id.to_s == edge_id}
+      else
+        nil
+      end
+    end
+    visitor.visit(self)
+  end
+  alias_method :edge, :find_descendant_edge
+
+  def find_descendant(vertex_id)
+    vertex_id = String(vertex_id)
+    return nil if vertex_id == self.id.to_s
+    vertex(vertex_id)
+  end
+
+  def vertex(vertex_id)
+    vertex_id = String(vertex_id)
+    return self if vertex_id == self.id.to_s
+    visitor = Tengine::Job::Vertex::AnyVisitor.new{|v| vertex_id == v.id.to_s ? v : nil }
+    visitor.visit(self)
+  end
+
+  def find_descendant_by_name_path(name_path)
+    return nil if name_path == self.name_path
+    vertex_by_name_path(name_path)
+  end
+
+  def vertex_by_name_path(name_path)
+    Tengine::Job::NamePath.absolute?(name_path) ?
+      root.vertex_by_absolute_name_path(name_path) :
+      vertex_by_relative_name_path(name_path)
+  end
+
+  def vertex_by_absolute_name_path(name_path)
+    return self if name_path.to_s == self.name_path
+    visitor = Tengine::Job::Vertex::AnyVisitor.new do |vertex|
+      if name_path == (vertex.respond_to?(:name_path) ? vertex.name_path : nil)
+        vertex
+      else
+        nil
+      end
+    end
+    visitor.visit(self)
+  end
+
+  def vertex_by_relative_name_path(name_path)
+    head, tail = *name_path.split(Tengine::Job::NamePath::SEPARATOR, 2)
+    child = child_by_name(head)
+    tail ? child.vertex_by_relative_name_path(tail) : child
+  end
+
+end