tengine_job 0.6.9

data/lib/tengine/job/drivers/schedule_driver.rb

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+
+# スケジュール管理ドライバ
+driver :schedule_driver do
+
+  on :'start.execution.job.tengine' do
+    exec = Tengine::Job::Signal.new(event).execution
+    name = exec.name_as_resource
+    status = Tengine::Core::Schedule::SCHEDULED
+    if exec.actual_base_timeout_alert && !exec.actual_base_timeout_alert.zero?
+      t1 = Time.now + (exec.actual_base_timeout_alert * 60.0)
+      Tengine::Core::Schedule.create event_type_name: "alert.execution.job.tengine", scheduled_at: t1, source_name: name, status: status , properties: event.properties
+    end
+    if exec.actual_base_timeout_termination && !exec.actual_base_timeout_termination.zero?
+      t2 = Time.now + (exec.actual_base_timeout_termination * 60.0)
+      Tengine::Core::Schedule.create event_type_name: "stop.execution.job.tengine", scheduled_at: t2, source_name: name, status: status, properties: event.properties.merge(stop_reason: 'timeout')
+    end
+  end
+
+  on :'success.execution.job.tengine' do
+    name = Tengine::Job::Signal.new(event).execution.name_as_resource
+    Tengine::Core::Schedule.where(source_name: name, status: Tengine::Core::Schedule::SCHEDULED).update_all(status: Tengine::Core::Schedule::INVALID)
+  end
+
+  on :'error.execution.job.tengine' do
+    name = Tengine::Job::Signal.new(event).execution.name_as_resource
+    Tengine::Core::Schedule.where(source_name: name, status: Tengine::Core::Schedule::SCHEDULED).update_all(status: Tengine::Core::Schedule::INVALID)
+  end
+
+end

data/lib/tengine/job/dsl_binder.rb

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+require 'tengine/job'
+
+# ジョブDSLをロードする際に使用される語彙に関するメソッドを定義するモジュール
+module Tengine::Job::DslBinder
+  include Tengine::Job::DslEvaluator
+
+  def jobnet(name, *args, &block)
+    Tengine::Job::RootJobnetTemplate.by_name(name)
+  end
+
+end

data/lib/tengine/job/dsl_evaluator.rb

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+require 'tengine/job'
+
+# Tengine::Job::DslLoader と Tengine::Job::DslBinder がincludeしているモジュールです。
+# それぞれに共通のメソッドを定義します。
+module Tengine::Job::DslEvaluator
+  private
+  def __stack_instance_variable__(ivar_name, value)
+    backup = instance_variable_get(ivar_name)
+    instance_variable_set(ivar_name, value)
+    begin
+      return yield if block_given?
+    ensure
+      instance_variable_set(ivar_name, backup)
+    end
+  end
+
+end

data/lib/tengine/job/dsl_loader.rb

@@ -0,0 +1,180 @@
+# -*- coding: utf-8 -*-
+require 'tengine/job'
+
+# ジョブDSLをロードする際に使用される語彙に関するメソッドを定義するモジュール
+module Tengine::Job::DslLoader
+  include Tengine::Job::DslEvaluator
+
+  class << self
+    def loading_template_block_store
+      @loading_template_block_store ||= {}
+    end
+
+    def template_block_store
+      @template_block_store ||= {}
+    end
+
+    def template_block_store_key(job, name)
+      "#{job.root.id.to_s}/#{job.id.to_s}##{name}"
+    end
+
+    def update_loaded_blocks(loaded_root)
+      if loaded_root
+        loading_template_block_store.each do |unsaved_job, (name, block)|
+          loaded_job = loaded_root.vertex_by_name_path(unsaved_job.name_path)
+          key = template_block_store_key(loaded_job, name)
+          template_block_store[key] = block
+        end
+      else
+        loading_template_block_store.each do |saved_job, (name, block)|
+          key = template_block_store_key(saved_job, name)
+          template_block_store[key] = block
+        end
+      end
+      loading_template_block_store.clear
+    end
+  end
+
+
+  def jobnet(name, *args, &block)
+    options = args.extract_options!
+    options = {
+      :name => name,
+      :description => args.first || name,
+    }.update(options)
+    auto_sequence = options.delete(:auto_sequence)
+    result = __with_redirection__(options) do
+      if @jobnet.nil?
+        klass = Tengine::Job::RootJobnetTemplate
+        options[:dsl_version] = config.dsl_version
+        path, lineno = *block.source_location
+        options[:dsl_filepath] = config.relative_path_from_dsl_dir(path)
+        options[:dsl_lineno] = lineno.to_i
+      else
+        klass = Tengine::Job::JobnetTemplate
+      end
+      klass.new(options)
+    end
+    result.with_start
+    @jobnet.children << result if @jobnet
+    if result.parent.nil?
+      if duplicated = result.find_duplication
+        if (duplicated.dsl_filepath != result.dsl_filepath) ||
+            (duplicated.dsl_lineno != result.dsl_lineno)
+          raise Tengine::Job::DslError, "2 jobnet named #{name.inspect} found at #{duplicated.dsl_filepath}:#{duplicated.dsl_lineno} and #{result.dsl_filepath}:#{result.dsl_lineno}"
+        end
+      end
+    end
+
+    __stack_instance_variable__(:@auto_sequence,  auto_sequence || @auto_sequence) do
+      __stack_instance_variable__(:@boot_job_names,  []) do
+        __stack_instance_variable__(:@redirections,  []) do
+          __stack_instance_variable__(:@jobnet, result, &block)
+          result.build_edges(@auto_sequence, @boot_job_names, @redirections)
+        end
+      end
+    end
+    if result.parent.nil?
+      loaded = result.find_duplication
+      result.save! unless loaded
+      Tengine::Job::DslLoader.update_loaded_blocks(loaded)
+      loaded || result
+    else
+      result
+    end
+  end
+
+  def auto_sequence
+    @auto_sequence = true
+  end
+
+  def boot_jobs(*boot_job_names)
+    @auto_sequence = false
+    @boot_job_names = boot_job_names
+  end
+
+  def job(name, *args)
+    script, description, options = __parse_job_args__(name, args)
+    options = {
+      :name => name,
+      :description => description,
+      :script => script
+    }.update(options)
+    preparation = options.delete(:preparation)
+    result = __with_redirection__(options) do
+      Tengine::Job::JobnetTemplate.new(options)
+    end
+    @jobnet.children << result
+    if preparation
+      Tengine::Job::DslLoader.loading_template_block_store[result] = [:preparation, preparation]
+    end
+    result
+  end
+
+  def hadoop_job_run(name, *args, &block)
+    script, description, options = __parse_job_args__(name, args)
+    options[:script] = script
+    options[:jobnet_type_key] = :hadoop_job_run
+    jobnet(name, description, options, &block)
+  end
+
+  def hadoop_job(name, options = {})
+    result = __with_redirection__(options) do
+      Tengine::Job::JobnetTemplate.new(:name => name, :jobnet_type_key => :hadoop_job)
+    end
+    result.children << start  = Tengine::Job::Start.new
+    result.children << fork   = Tengine::Job::Fork.new
+    result.children << map    = Tengine::Job::JobnetTemplate.new(:name => "Map"   , :jobnet_type_key => :map_phase   )
+    result.children << reduce = Tengine::Job::JobnetTemplate.new(:name => "Reduce", :jobnet_type_key => :reduce_phase)
+    result.children << join   = Tengine::Job::Join.new
+    result.children << _end   = Tengine::Job::End.new
+    result.edges.new(:origin_id => start.id , :destination_id => fork.id  )
+    result.edges.new(:origin_id => fork.id  , :destination_id => map.id   )
+    result.edges.new(:origin_id => fork.id  , :destination_id => reduce.id)
+    result.edges.new(:origin_id => map.id   , :destination_id => join.id  )
+    result.edges.new(:origin_id => reduce.id, :destination_id => join.id  )
+    result.edges.new(:origin_id => join.id  , :destination_id => _end.id  )
+    @jobnet.children << result
+    result
+  end
+
+  def finally(&block)
+    jobnet("finally", :jobnet_type_key => :finally, &block)
+  end
+
+  def expansion(root_jobnet_name, options = {})
+    options = {
+      :name => root_jobnet_name,
+    }.update(options)
+    result = __with_redirection__(options) do
+      Tengine::Job::Expansion.new(options)
+    end
+    @jobnet.children << result
+    result
+  end
+
+  private
+  def __parse_job_args__(name, args)
+    options = args.extract_options!
+    description_or_script, script = *args
+    if script
+      description = description_or_script
+    else
+      script = description_or_script
+      description = name
+    end
+    return script, description, options
+  end
+
+  def __with_redirection__(options)
+    destination_names = Array(options.delete(:to) || options.delete(:redirect_to))
+    result = yield
+    destination_names.each do |dest_name|
+      @redirections << [result.name, dest_name]
+    end
+    result
+  end
+
+
+
+end

data/lib/tengine/job/edge.rb

@@ -0,0 +1,150 @@
+# -*- coding: utf-8 -*-
+require 'tengine/job'
+require 'selectable_attr'
+
+# Vertexとともにジョブネットを構成するグラフの「辺」を表すモデル
+# Tengine::Job::Jobnetにembeddedされます。
+class Tengine::Job::Edge
+  include Mongoid::Document
+  include Mongoid::Timestamps
+  include Tengine::Core::SelectableAttr
+  include Tengine::Job::Signal::Transmittable
+
+  class StatusError < StandardError
+  end
+
+  embedded_in :owner, :class_name => "Tengine::Job::Jobnet", :inverse_of => :edges
+
+  field :phase_cd     , :type => Integer, :default => 0 # ステータス。とりうる値は後述を参照してください。詳しくはtengine_jobパッケージ設計書の「edge状態遷移」を参照してください。
+  field :origin_id     , :type => BSON::ObjectId # 辺の遷移元となるvertexのid
+  field :destination_id, :type => BSON::ObjectId # 辺の遷移先となるvertexのid
+
+  validates :origin_id, :presence => true
+  validates :destination_id, :presence => true
+
+  selectable_attr :phase_cd do
+    entry  0, :active      , "active"      , :alive => true
+    entry 10, :transmitting, "transmitting", :alive => true
+    entry 20, :transmitted , "transmitted" , :alive => false
+    entry 30, :suspended   , "suspended"   , :alive => true
+    entry 31, :keeping     , "keeping"     , :alive => true
+    entry 40, :closing     , "closing"     , :alive => false
+    entry 50, :closed      , "closed"      , :alive => false
+  end
+
+  def alive?; !!phase_entry[:alive]; end
+  def alive_or_closing?; alive? || closing?; end
+  def alive_or_closing_or_closed?; alive? || closing? || closed?; end
+
+  phase_keys.each do |phase_key|
+    class_eval(<<-END_OF_METHOD)
+      def #{phase_key}?; phase_key == #{phase_key.inspect}; end
+    END_OF_METHOD
+  end
+
+  def origin
+    owner.children.detect{|c| c.id == origin_id}
+  end
+
+  def destination
+    owner.children.detect{|c| c.id == destination_id}
+  end
+
+  def name_for_message
+    "edge(#{id.to_s}) from #{origin ? origin.name_path : 'no origin'} to #{destination ? destination.name_path : 'no destination'}"
+  end
+
+  def inspect
+    "#<#{self.class.name} #{name_for_message}>"
+  end
+
+  # https://cacoo.com/diagrams/hdLgrzYsTBBpV3Wj#3E9EA
+  def transmit(signal)
+    case phase_key
+    when :active then
+      d = destination
+      if signal.execution.in_scope?(d)
+        self.phase_key = :transmitting
+        signal.leave(self)
+      else
+        Tengine.logger.info("#{d.name_path} will not be executed, becauase it is out of execution scope.")
+      end
+    when :suspended then
+      self.phase_key = :keeping
+    when :closing then
+      self.phase_key = :closed
+      signal.paths << self
+      signal.with_paths_backup do
+        if destination.is_a?(Tengine::Job::Job)
+          destination.next_edges.first.transmit(signal)
+        else
+          signal.leave(self)
+        end
+      end
+    end
+  end
+
+  def complete(signal)
+    case phase_key
+    when :transmitting then
+      self.phase_key = :transmitted
+    when :active, :closed then
+      # IG
+    when :suspended, :keeping then
+      # N/A
+      raise Tengine::Job::Edge::StatusError, "#{self.class.name}#complete not available on #{phase_key.inspect} at #{self.inspect}"
+    end
+  end
+
+  def reset(signal)
+    # 全てのステータスから遷移する
+    if d = destination
+      if signal.execution.in_scope?(d)
+        self.phase_key = :active
+        d.reset(signal)
+      end
+    else
+      raise "destination not found: #{destination_id.inspect} from #{origin.inspect}"
+    end
+  end
+
+  def close(signal)
+    case phase_key
+    when :active, :suspended, :keeping, :transmitting then
+      self.phase_key = :closing
+    end
+  end
+
+  def close_followings
+    accept_visitor(Tengine::Job::Edge::Closer.new)
+  end
+
+  def accept_visitor(visitor)
+    visitor.visit(self)
+  end
+
+  def phase_key=(phase_key)
+    Tengine.logger.debug("edge phase changed. <#{self.id.to_s}> #{self.phase_name} -> #{Tengine::Job::Edge.phase_name_by_key(phase_key)}")
+    self.write_attribute(:phase_cd, Tengine::Job::Edge.phase_id_by_key(phase_key))
+  end
+
+  class Closer
+    def visit(obj)
+      if obj.is_a?(Tengine::Job::End)
+        if parent = obj.parent
+          (parent.next_edges || []).each{|edge| edge.accept_visitor(self)}
+        end
+      elsif obj.is_a?(Tengine::Job::Vertex)
+        obj.next_edges.each{|edge| edge.accept_visitor(self)}
+      elsif obj.is_a?(Tengine::Job::Edge)
+        obj.close(nil)
+        obj.destination.accept_visitor(self)
+      else
+        raise "Unsupported class #{obj.inspect}"
+      end
+    end
+
+  end
+
+
+end

data/lib/tengine/job/element_selector_notation.rb

@@ -0,0 +1,169 @@
+# -*- coding: utf-8 -*-
+require 'tengine/job'
+
+# # Tengine Jobnet Element Selector Notation
+# Tengineジョブネット要素セレクタ記法
+#
+# Tengineジョブネット要素セレクタ記法は、TenigneのジョブDSLで生成されるジョブネットから
+# 特定のVertexやEdgeを特定するための記述方法です。
+# 正式名称が長いので、ここでは略して「セレクタ記法」と呼ぶ
+#
+# ## 背景
+# Tengienジョブでは、ジョブやジョブネットなどのvertexとそれらを結ぶedgeが連携して動くこと
+# ジョブの実行を行うが、ジョブ、ジョブネット以外のvertexとedgeには名前がついていない。
+# しかし、ジョブネット内の要素群が連携して動作することを確認するためには、どのvertex、
+# あるいはどのedgeなのかという点について状態などを確認する必要がある。
+#
+# ## name_path
+# Tengineのジョブ／ジョブネットは親子関係によるツリー構造を持ち、
+# それぞれ兄弟間では重複しない名前が指定されるので、スラッシュによって区切ることによって
+# ファイルパスのような表記によって、どのジョブ／ジョブネットなのかを特定できる
+#
+# ## name_pathの問題点
+# ジョブとジョブネットについては名前があるのでname_pathだけで特定できるが、
+# それ以外のedgeやfork, joinについては特定することができない。
+#
+# ## それぞれの指定方法
+# ### ジョブ／ジョブネット
+# 1. name_pathをそのまま
+#  * 例: "/j100/j110/j111"    # jxxx は、ジョブあるいはジョブネットを想定しています
+# 2. "#{ジョブ／ジョブネット名}@#{ジョブネットのname_path}"
+#  * 例: "j111@/j100/j110"
+#
+#
+# ### startとend
+# 1. "#{start or end}!#{ジョブネットのセレクタ}"
+#  * 例: start!j110@/j100
+#  * 例: end!/j100/j110
+# 2. "#{start or end}!#{ジョブネットのname_path}"
+#  * 例: start@/j100/j110
+#  * 例: end@/j100/j110
+#
+# ### edge
+# #### ジョブ／ジョブネットの前後のedge
+# 1. "#{prev or next}!#{ジョブ／ジョブネットのセレクタ}"
+#  * 例: prev!j110@/j100
+#  * 例: next!/j100/j110
+#
+# #### ジョブ／ジョブネットの間のedge
+# 1. "#{前のジョブ名}~#{後のジョブ名}@#{ジョブネットのセレクタ}"
+#  * 例: j111~j112@/j100/j110
+#
+# #### fork-join間のedge
+# 1. "fork~join!#{前のジョブ名}~#{後のジョブ名}@#{ジョブネットのセレクタ}"
+#  * 例: fork~join!j112~j113@/j100/j110
+#
+# ### fork, join
+# 1. "#{fork or join}!#{前のジョブ名}~#{後のジョブ名}@#{ジョブネットのセレクタ}"
+#  * 例: fork!j112~j113@/j100/j110
+#  * 例: join!j112~j113@/j100/j110
+#
+#
+
+module Tengine::Job::ElementSelectorNotation
+
+  class NotFound < Tengine::Errors::NotFound
+    attr_reader :jobnet, :notation
+    def initialize(jobnet, notation)
+      @jobnet, @notation = jobnet, notation
+    end
+
+    def message
+      "Tengine Jobnet Element not found by selector #{notation.inspect} in #{@jobnet.name_path}"
+    end
+  end
+
+
+  NAME_PART = /[A-Za-z_][\w\-]*/.freeze
+  NAME_PATH_PART = /[A-Za-z_\/][\w\-\/]*/.freeze
+
+  # elementメソッドは指定されたnotationによって対象となる要素が見つからなかった場合はnilを返します。
+  # 例外をraiseさせたい場合は element!メソッドを使ってください。
+  def element(notation)
+    direction, current_path = *notation.split(/@/, 2)
+    return vertex_by_name_path(direction) if current_path.nil? && Tengine::Job::NamePath.absolute?(direction)
+    current = current_path ? vertex_by_name_path(current_path) : self
+    raise "#{current_path.inspect} not found" unless current
+    case direction
+    # when /^prev!(?:#{Tengine::Core::Validation::BASE_NAME.format})/
+    when /^(prev|next)!(#{NAME_PATH_PART})$/ then
+      job = $2 ? current.vertex_by_name_path($2) : self
+      job.send("#{$1}_edges").first
+    when /^(start|end|finally)!(#{NAME_PATH_PART})$/ then
+      job = $2 ? current.vertex_by_name_path($2) : self
+      job.child_by_name($1)
+    when /^(start|end|finally)$/ then
+      current.child_by_name($1)
+    when /^(#{NAME_PART})~(#{NAME_PART})$/ then
+      job1 = current.child_by_name($1)
+      job2 = current.child_by_name($2)
+      job1.next_edges.detect{|edge| edge.destination_id == job2.id}
+    when /^(fork|join)!(#{NAME_PART})~(#{NAME_PART})$/ then
+      klass = Tengine::Job.const_get($1.capitalize)
+      job1 = current.child_by_name($2)
+      job2 = current.child_by_name($3)
+      paths = PathFinder.new(job1, job2).process
+      paths.each do |path|
+        path.each do |element|
+          return element if element.is_a?(klass)
+        end
+      end
+    when /^fork~join!(#{NAME_PART})~(#{NAME_PART})$/ then
+      job1 = current.child_by_name($1)
+      job2 = current.child_by_name($2)
+      paths = PathFinder.new(job1, job2).process
+      paths.each do |path|
+        path.each do |element|
+          if element.is_a?(Tengine::Job::Edge)
+            if element.origin.is_a?(Tengine::Job::Fork) &&
+                element.destination.is_a?(Tengine::Job::Join)
+              return element
+            end
+          end
+        end
+      end
+    else
+      current.child_by_name(direction)
+    end
+  end
+
+  # element!メソッドは指定されたnotationによって対象となる要素が見つからなかった場合はnilを返します。
+  # 例外をraiseさせたい場合は element!メソッドを使ってください。
+  def element!(notation, *args)
+    result = element(notation, *args)
+    raise NotFound.new(self, notation) unless result
+    result
+  end
+
+  class PathFinder
+    def initialize(origin, dest)
+      @origin, @dest = origin, dest
+    end
+
+    def process
+      @routes = []
+      @current_route = []
+      @origin.accept_visitor(self)
+      @routes.select{|route| route.last == @dest}
+    end
+
+    def visit(element)
+      @current_route << element
+      if element.is_a?(Tengine::Job::Edge)
+        element.destination.accept_visitor(self)
+      else
+        @routes << @current_route.dup
+        return if element == @dest
+        element.next_edges.each do |edge|
+          bak = @current_route.dup
+          begin
+            edge.accept_visitor(self)
+          ensure
+            @current_route = bak
+          end
+        end
+      end
+    end
+  end
+
+end