tengine_job 1.1.0 → 1.2.0

data/lib/tengine/job/{name_path.rb → structure/name_path.rb}

@@ -1,6 +1,6 @@
-require 'tengine/job'
+require 'tengine/job/structure'
 
-module Tengine::Job::NamePath
+module Tengine::Job::Structure::NamePath
 
   SEPARATOR = '/'.freeze
   ABSOLUTE_PATH_REGEXP = /^\//.freeze

data/lib/tengine/job/structure/tree.rb

@@ -0,0 +1,20 @@
+require 'tengine/job/structure'
+
+module Tengine::Job::Structure::Tree
+  def root?
+    parent.nil?
+  end
+
+  def root
+    root? ? self : parent.root
+  end
+
+  def ancestors
+    if parent = self.parent
+      parent.ancestors + [parent]
+    else
+      []
+    end
+  end
+
+end

data/lib/tengine/job/structure/visitor.rb

@@ -0,0 +1,67 @@
+require 'tengine/job/structure'
+
+module Tengine::Job::Structure::Visitor
+  module Accepter
+  def accept_visitor(visitor)
+    visitor.visit(self)
+  end
+  end
+
+  class Any
+    def initialize(&block)
+      @block = block
+    end
+    def visit(vertex)
+      if result = @block.call(vertex)
+        return result
+      end
+      vertex.children.each do |child|
+        if result = child.accept_visitor(self)
+          return result
+        end
+      end
+      return nil
+    end
+  end
+
+  class All
+    def initialize(&block)
+      @block = block
+    end
+
+    def visit(vertex)
+      @block.call(vertex)
+      vertex.children.each do |child|
+        child.accept_visitor(self)
+      end
+    end
+  end
+
+  class AllWithEdge < All
+    def visit(obj)
+      if obj.respond_to?(:children)
+        super(obj)
+      else
+        @block.call(obj)
+      end
+      return unless obj.respond_to?(:edges)
+      obj.edges.each{|edge| edge.accept_visitor(self)}
+    end
+  end
+
+  class TraceEdge
+    def initialize(&block)
+      @block = block
+    end
+
+    def visit(obj)
+      @block.call(obj)
+      if obj.respond_to?(:destination)
+        obj.destination.accept_visitor(self)
+      elsif obj.respond_to?(:next_edges)
+        obj.next_edges.each{|edge| edge.accept_visitor(self) }
+      end
+    end
+  end
+
+end

data/lib/tengine/job/template.rb

@@ -0,0 +1,24 @@
+require 'tengine/job'
+
+module Tengine::Job::Template
+  autoload :Vertex              , "tengine/job/template/vertex"
+  autoload :Edge                , "tengine/job/template/edge"
+
+  autoload :NamedVertex         , "tengine/job/template/named_vertex"
+
+  autoload :Jobnet              , "tengine/job/template/jobnet"
+  autoload :Start               , "tengine/job/template/jobnet"
+  autoload :End                 , "tengine/job/template/jobnet"
+
+  autoload :Junction            , "tengine/job/template/junction"
+  autoload :Fork                , "tengine/job/template/junction"
+  autoload :Join                , "tengine/job/template/junction"
+
+  autoload :RootJobnet          , "tengine/job/template/root_jobnet"
+
+  autoload :SshJob              , "tengine/job/template/ssh_job"
+
+  autoload :Expansion           , "tengine/job/template/expansion"
+
+  autoload :Generator           , "tengine/job/template/generator"
+end

data/lib/tengine/job/template/edge.rb

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+require 'tengine/job/template'
+require 'selectable_attr'
+
+# Vertexとともにジョブネットを構成するグラフの「辺」を表すモデル
+# Tengine::Job::Template::Jobnetにembeddedされます。
+class Tengine::Job::Template::Edge
+  include Mongoid::Document
+  include Mongoid::Timestamps
+  include Tengine::Core::SelectableAttr
+  include Tengine::Job::Structure::Visitor::Accepter
+
+  embedded_in :owner, :class_name => "Tengine::Job::Template::Jobnet", :inverse_of => :edges
+
+  field :origin_id     , :type => Moped::BSON::ObjectId # 辺の遷移元となるvertexのid
+  field :destination_id, :type => Moped::BSON::ObjectId # 辺の遷移先となるvertexのid
+
+  validates :origin_id, :presence => true
+  validates :destination_id, :presence => true
+
+  def origin
+    owner.children.detect{|c| c.id == origin_id}
+  end
+
+  def destination
+    owner.children.detect{|c| c.id == destination_id}
+  end
+
+  def name_for_message
+    "edge(#{id.to_s}) from #{origin ? origin.name_path : 'no origin'} to #{destination ? destination.name_path : 'no destination'}"
+  end
+
+  def inspect
+    "#<#{self.class.name} #{name_for_message}>"
+  end
+
+end

data/lib/tengine/job/template/expansion.rb

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+require 'tengine/job/template'
+
+# ルートジョブネットを他のジョブネット内に展開するための特殊なテンプレート用Vertex。
+class Tengine::Job::Template::Expansion < Tengine::Job::Template::NamedVertex
+  def actual_class
+    Tengine::Job::Runtime::Jobnet
+  end
+
+  def root_jobnet_template
+    unless @root_jobnet_template
+      cond = {:dsl_version => root.dsl_version, :name => name}
+      @root_jobnet_template = Tengine::Job::Template::RootJobnet.where(cond).first
+    end
+    @root_jobnet_template
+  end
+
+  def actual_credential_name
+    @root_jobnet_template.actual_credential_name
+  end
+  def actual_server_name
+    @root_jobnet_template.actual_server_name
+  end
+end

data/lib/tengine/job/template/generator.rb

@@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+
+require 'tengine/job/template'
+
+class Tengine::Job::Template::Generator
+  def execute(template, options = {})
+    @inherited_attrs = {}
+    process(template, options)
+  end
+
+  CLASS_MAP = %w[RootJobnet Jobnet Start End Fork Join SshJob].each_with_object({}){|t, d|
+    c = "Tengine::Job::Template::#{t}"
+    d[c] = c.sub(/Template/, 'Runtime')
+  }.update({"Tengine::Job::Template::Expansion" => "Tengine::Job::Runtime::Jobnet"})
+
+  BASE_IGNORED_FIELD_NAMES = ["_type", "_id"].freeze
+  EXPANSION_IGNORED_FIELD_NAMES = (BASE_IGNORED_FIELD_NAMES +
+    %w[dsl_version jobnet_type_cd version updated_at created_at children edges]).freeze
+
+  def expansion?
+    @current.is_a?(Tengine::Job::Template::Expansion)
+  end
+
+  def expansion_root
+    cond = {:dsl_version => @current.root.dsl_version, :name => @current.name}
+    Tengine::Job::Template::RootJobnet.where(cond).first
+  end
+
+  def ignored_fields
+    expansion? ?
+      EXPANSION_IGNORED_FIELD_NAMES : BASE_IGNORED_FIELD_NAMES
+  end
+
+  def generating_attrs
+    field_names = @current.class.fields.keys - ignored_fields
+    attrs = field_names.each_with_object({}){|name, d| d[name] = @current.send(name) }
+    if expansion?
+      attrs[:was_expansion] = true
+      if (t = expansion_root)
+        attrs[:template_id] = t.id
+      end
+    else
+      if @current.is_a?(Tengine::Job::Template::Jobnet)
+        attrs[:template_id] = @current.id
+      end
+    end
+    attrs
+  end
+
+  def generating_children
+    expansion? ?
+      expansion_root.children : @current.children
+  end
+
+  def generating_edges
+    expansion? ?
+      expansion_root.edges :
+      @current.respond_to?(:edges) ? @current.edges : []
+  end
+
+  def process(template, options)
+    @inherited_attrs, inherited_attrs_backup = merge_inherited_attrs, @inherited_attrs
+    @current, current_backup = template, @current
+    begin
+      generate(options)
+    ensure
+      @current = current_backup
+      @inherited_attrs = inherited_attrs_backup
+    end
+  end
+
+  def merge_inherited_attrs
+    source = @current.nil? ? nil : expansion? ? expansion_root : @current
+    return {} if source.nil?
+    result = @inherited_attrs.dup
+    {
+      server_name: source.actual_server_name,
+      credential_name: source.actual_credential_name,
+      killing_signals: source.actual_killing_signals,
+      killing_signal_interval: source.actual_killing_signal_interval,
+    }.each do |key, value|
+      next if value.blank?
+      result[key] = value # sourceからの値が有効なら上書きする
+    end
+    result
+  end
+
+  def generate(options)
+    attrs = generating_attrs.update(child_index: options[:child_index], parent: options[:parent])
+    @inherited_attrs.each{|key, value| attrs[key] ||= value }
+    klass_name = CLASS_MAP[@current.class.name]
+    result = klass_name.constantize.new(attrs)
+    result.save!
+    src_to_generated = {}
+    generating_children.each.with_index do |child, index|
+      generated = process(child, child_index: index + 1, parent: result)
+      src_to_generated[child.id] = generated.id
+      result.children << generated
+    end
+    generating_edges.each do |edge|
+      generated = Tengine::Job::Runtime::Edge.new
+      generated.origin_id = src_to_generated[edge.origin_id]
+      generated.destination_id = src_to_generated[edge.destination_id]
+      result.edges << generated
+    end
+    yield(result) if block_given? # 派生クラスでsave!の前に処理を入れるためのyield
+    result.save!
+    result
+  end
+
+end

data/lib/tengine/job/template/jobnet.rb

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+require 'tengine/job/template'
+
+require 'selectable_attr'
+
+# ジョブの始端から終端までを持ち、VertexとEdgeを組み合わせてジョブネットを構成することができるVertex。
+# 自身もジョブネットを構成するVertexの一部として扱われる。
+class Tengine::Job::Template::Jobnet < Tengine::Job::Template::NamedVertex
+  include Tengine::Core::SelectableAttr
+  include Tengine::Core::SafeUpdatable
+
+  include Tengine::Job::Structure::JobnetBuilder
+  include Tengine::Job::Structure::JobnetFinder
+  include Tengine::Job::Structure::ElementSelectorNotation
+
+  include Tengine::Job::Template::SshJob::Settings
+
+  autoload :Builder, "tengine/job/jobnet/builder"
+  autoload :StateTransition, 'tengine/job/jobnet/state_transition'
+  autoload :JobStateTransition, 'tengine/job/jobnet/job_state_transition'
+  autoload :JobnetStateTransition, 'tengine/job/jobnet/jobnet_state_transition'
+
+  field :description   , :type => String # ジョブネットの説明
+
+  field :jobnet_type_cd, :type => Integer, :default => 1 # ジョブネットの種類。後述の定義を参照してください。
+
+  selectable_attr :jobnet_type_cd do
+    entry 1, :normal        , "normal"
+    entry 2, :finally       , "finally", :alternative => true
+    # entry 3, :recover       , "recover", :alternative => true
+    entry 4, :hadoop_job_run, "hadoop job run"
+    entry 5, :hadoop_job    , "hadoop job"    , :chained_box => true
+    entry 6, :map_phase     , "map phase"     , :chained_box => true
+    entry 7, :reduce_phase  , "reduce phase"  , :chained_box => true
+  end
+  def chained_box?; jobnet_type_entry[:chained_box]; end
+
+  embeds_many :edges, :class_name => "Tengine::Job::Template::Edge", :inverse_of => :owner , :validate => false
+  accepts_nested_attributes_for :edges
+
+  before_validation do |r|
+    r.edges.each do |edge|
+      unless edge.valid?
+        edge.errors.each do |f, error|
+          r.errors.add(:base, "#{edge.name_for_message} #{f.to_s.humanize} #{error}")
+        end
+      end
+    end
+  end
+
+
+  VERTEX_CLASSES = {
+    vertex: "Vertex",
+    start_vertex: "Start",
+    end_vertex: "End",
+    jobnet: "Jobnet",
+    fork: "Fork",
+    join: "Join",
+  }.freeze
+
+  VERTEX_CLASSES.each do |key, value|
+    instance_eval("def #{key}_class; Tengine::Job::Template::#{value}; end", __FILE__, __LINE__)
+  end
+
+  class << self
+    def by_name(name)
+      where({:name => name}).first
+    end
+
+  end
+end
+
+
+
+
+# ジョブネットの始端を表すVertex。特に状態は持たない。
+class Tengine::Job::Template::Start < Tengine::Job::Template::Vertex
+end
+
+
+# ジョブネットの終端を表すVertex。特に状態は持たない。
+class Tengine::Job::Template::End < Tengine::Job::Template::Vertex
+end

data/lib/tengine/job/template/junction.rb

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+require 'tengine/job/template'
+
+# ForkやJoinの継承元となるVertex。特に状態は持たない。
+class Tengine::Job::Template::Junction < Tengine::Job::Template::Vertex
+end
+
+# 一つのVertexから複数のVertexへSignalを通知する分岐のVertex。
+class Tengine::Job::Template::Fork < Tengine::Job::Template::Junction
+end
+
+# 複数のVertexの終了を待ちあわせて一つのVertexへSignalを通知する合流のVertex。
+class Tengine::Job::Template::Join < Tengine::Job::Template::Junction
+end

data/lib/tengine/job/{job.rb → template/named_vertex.rb}

@@ -1,11 +1,9 @@
 # -*- coding: utf-8 -*-
-require 'tengine/job'
+require 'tengine/job/template'
 
 # 処理を意味するVertex。実際に実行を行うTengine::Job::Scriptやジョブネットである
-# Tengine::Job::Jobnetの継承元である。
-class Tengine::Job::Job < Tengine::Job::Vertex
-  include Tengine::Job::Connectable
-  include Tengine::Job::Killing
+# Tengine::Job::Template::Jobnetの継承元である。
+class Tengine::Job::Template::NamedVertex < Tengine::Job::Template::Vertex
 
   field :name, :type => String # ジョブの名称。
 

data/lib/tengine/job/{root_jobnet_template.rb → template/root_jobnet.rb}

@@ -1,39 +1,20 @@
 # -*- coding: utf-8 -*-
-require 'tengine/job'
+require 'tengine/job/template'
 
 # DSLを評価して登録されるルートジョブネットを表すVertex
-class Tengine::Job::RootJobnetTemplate < Tengine::Job::JobnetTemplate
-  include Tengine::Job::Root
+class Tengine::Job::Template::RootJobnet < Tengine::Job::Template::Jobnet
   include Tengine::Core::FindByName
 
   field :dsl_filepath, :type => String  # ルートジョブネットを定義した際にロードされたDSLのファイル名(Tengine::Core::Config#dsl_dir_pathからの相対パス)
   field :dsl_lineno  , :type => Integer # ルートジョブネットを定義するjobnetメソッドの呼び出しの、ロードされたDSLのファイルでの行番号
   field :dsl_version , :type => String  # ルートジョブネットを定義した際のDSLのバージョン
 
-  def actual_class
-    Tengine::Job::RootJobnetActual
-  end
-  def generate(klass = actual_class)
-    result = super(klass)
-    result.template = self
-    result
-  end
+  belongs_to :category, inverse_of: nil, index: true, class_name: "Tengine::Job::Structure::Category"
 
-  def execute(options = {})
-    event_sender = options.delete(:sender) || Tengine::Event.default_sender
-    actual = generate
-    actual.with(safe: safemode(actual.class.collection)).save!
-    result = Tengine::Job::Execution.with(
-                safe: safemode(Tengine::Job::Execution.collection)
-             ).create!(
-               (options || {}).update(:root_jobnet_id => actual.id)
-             )
-    event_sender.fire(:"start.execution.job.tengine", :properties => {
-        :execution_id => result.id,
-        :root_jobnet_id => actual.id,
-        :target_jobnet_id => actual.id
-      })
-    result
+  def generate(options = {})
+    super(options) do |r|
+      r.template = self
+    end
   end
 
   def find_duplication
@@ -48,4 +29,9 @@ class Tengine::Job::RootJobnetTemplate < Tengine::Job::JobnetTemplate
       where({:name => name, :dsl_version => version}).first
     end
   end
+
+  Tengine::Job::Template::Jobnet::VERTEX_CLASSES.keys.each do |key|
+    instance_eval("def #{key}_class; Tengine::Job::Template::Jobnet.#{key}_class; end", __FILE__, __LINE__)
+  end
+
 end