tengine_job 0.6.9

data/examples/0006_retry_three_layer.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+typeset -i time="$1"
+export LOGFILE=tengine_job_test.log
+echo "`date` tengine_job_test $MM_ACTUAL_JOB_NAME_PATH start" >> $LOGFILE
+case "$MM_ACTUAL_JOB_NAME_PATH" in
+  "/jn0006/jn1/jn11/j111" ) 
+    sleep ${J111_SLEEP:=0} 
+    if [ "$J111_FAIL" = "true" ] ; then
+      exit 1
+    fi
+    ;;
+  "/jn0006/jn1/jn11/j112" ) 
+    sleep ${J112_SLEEP:=0} 
+    if [ "$J112_FAIL" = "true" ] ; then
+      exit 1
+    fi
+    ;;
+  "/jn0006/jn1/jn11/finally/jn11_f" ) 
+    sleep ${JN11_F_SLEEP:=0} 
+    if [ "$JN11_F_FAIL" = "true" ] ; then
+      exit 1
+    fi
+    ;;
+  "/jn0006/jn1/j12" ) 
+    sleep ${J12_SLEEP:=0} 
+    if [ "$JN12_FAIL" = "true" ] ; then
+      exit 1
+    fi
+    ;;
+  "/jn0006/jn1/finally/jn1_f" ) 
+    sleep ${JN1_F_SLEEP:=0} 
+    if [ "$JN1_F_FAIL" = "true" ] ; then
+      exit 1
+    fi
+    ;;
+  "/jn0006/jn2/j21" ) 
+    sleep ${J21_SLEEP:=0} 
+    if [ "$JN21_FAIL" = "true" ] ; then
+      exit 1
+    fi
+    ;;
+  "/jn0006/jn2/jn22/j221" ) 
+    sleep ${J221_SLEEP:=0} 
+    if [ "$J221_FAIL" = "true" ] ; then
+      exit 1
+    fi
+    ;;
+  "/jn0006/jn2/jn22/j222" ) 
+    sleep ${J222_SLEEP:=0} 
+    if [ "$J222_FAIL" = "true" ] ; then
+      exit 1
+    fi
+    ;;
+  "/jn0006/jn2/jn22/finally/jn22_f" ) 
+    sleep ${JN22_F_SLEEP:=0} 
+    if [ "$JN22_F_FAIL" = "true" ] ; then
+      exit 1
+    fi
+    ;;
+  "/jn0006/jn2/finally/jn2_f" ) 
+    sleep ${JN2_F_SLEEP:=0} 
+    if [ "$JN2_F_FAIL" = "true" ] ; then
+      exit 1
+    fi
+    ;;
+  "/jn0006/finally/finally/jn_f" ) 
+    sleep ${JN_F_SLEEP:=0} 
+    if [ "$JN_F_FAIL" = "true" ] ; then
+      exit 1
+    fi
+    ;;
+esac
+
+echo "`date` tengine_job_test $MM_ACTUAL_JOB_NAME_PATH finish" >> $LOGFILE

data/examples/0007_simple_jobnet.rb

@@ -0,0 +1,7 @@
+require 'tengine_job'
+
+jobnet("rjn0007", :server_name => "test_server1", :credential_name => "test_credential1") do
+  auto_sequence
+  job("j11", "$HOME/tengine_job_test.sh 30 j11")
+  job("j12", "$HOME/tengine_job_test.sh 30 j12")
+end

data/examples/0021_dynamic_env.rb

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+require 'tengine_job'
+
+jobnet("rjn0021_1", :server_name => "test_server1", :credential_name => "test_credential1") do
+  auto_sequence
+  job("j1", "env | sort", :preparation => proc{ "export FOO=BAR" })
+  job("j2", "env | sort", :preparation => proc{ "export SERVER_NAME=#{actual_server.name} && export DNS_NAME=#{actual_server.addresses['private_dns_name']}" })
+end
+
+jobnet("rjn0021_2") do # :server_name, :credential_nameの指定なし
+  auto_sequence
+  job("j1", "env | sort", :preparation => proc{ "export FOO=BAR" })
+  job("j2", "env | sort", :preparation => proc{ "export SERVER_NAME=#{actual_server.name} && export DNS_NAME=#{actual_server.addresses['private_dns_name']}" })
+end
+
+jobnet("rjn0021", :server_name => "test_server1", :credential_name => "test_credential1") do
+  auto_sequence
+  expansion("rjn0021_1")
+  expansion("rjn0021_2")
+end

data/examples/VERSION

@@ -0,0 +1 @@
+20111003160000

data/examples/tengine_job_test.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+typeset -i time="$1"
+export LOGFILE=tengine_job_test.log
+echo "`date` tengine_job_test $2 start" >> $LOGFILE
+if [ "$SLEEP" != ""  ] ; then
+  sleep ${SLEEP:=0}
+else
+  sleep `expr 1 + $time`
+fi
+echo "`date` tengine_job_test $2 finish" >> $LOGFILE

data/lib/tengine/job.rb

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+require 'tengine_job'
+
+require 'active_support/concern'
+
+module Tengine::Job
+  autoload :DslEvaluator        , "tengine/job/dsl_evaluator"
+  autoload :DslLoader           , "tengine/job/dsl_loader"
+  autoload :DslBinder           , "tengine/job/dsl_binder"
+
+  autoload :Signal              , 'tengine/job/signal'
+
+  autoload :Execution           , "tengine/job/execution"
+
+  autoload :Category            , "tengine/job/category"
+  autoload :Edge                , "tengine/job/edge"
+  autoload :Vertex              , "tengine/job/vertex"
+
+  autoload :Start               , "tengine/job/start"
+  autoload :End                 , "tengine/job/end"
+  autoload :Junction            , "tengine/job/junction"
+  autoload :Fork                , "tengine/job/fork"
+  autoload :Join                , "tengine/job/join"
+
+  autoload :Job                 , "tengine/job/job"
+
+  autoload :Jobnet              , "tengine/job/jobnet"
+  autoload :JobnetActual        , "tengine/job/jobnet_actual"
+  autoload :RootJobnetActual    , "tengine/job/root_jobnet_actual"
+  autoload :JobnetTemplate      , "tengine/job/jobnet_template"
+  autoload :RootJobnetTemplate  , "tengine/job/root_jobnet_template"
+
+  autoload :Expansion           , "tengine/job/expansion"
+
+  autoload :Root                , "tengine/job/root"
+  autoload :NamePath            , "tengine/job/name_path"
+  autoload :Executable          , "tengine/job/executable"
+  autoload :Stoppable           , "tengine/job/stoppable"
+  autoload :Killing             , "tengine/job/killing"
+  autoload :Connectable         , "tengine/job/connectable"
+  autoload :ScriptExecutable    , "tengine/job/script_executable"
+
+  autoload :MmCompatibility     , "tengine/job/mm_compatibility"
+  autoload :ElementSelectorNotation, "tengine/job/element_selector_notation"
+
+  class << self
+    # tengine_coreからそのプラグインへ通知を受けるための
+    def notify(sender, msg)
+Tengine::Core.stdout_logger.info("*" * 100)
+        Dir[File.expand_path("job/drivers/*.rb", File.dirname(__FILE__))].each do |f|
+Tengine::Core.stdout_logger.info("#{self.name}.notify  #{f}")
+        end
+      # if (msg == :before___evaluate__) # だと、最初にtengine/jobがrequireされる前に実行されるのでフックできません
+      if (msg == :after___evaluate__)
+        Tengine::Core::Driveable.module_eval{ include Tengine::Job::DslBinder }
+        Dir[File.expand_path("job/drivers/*.rb", File.dirname(__FILE__))].each do |f|
+          # Tengine::Core.stdout_logger.debug("#{self.name} now evaluating #{f}")
+          # sender.instance_eval(File.read(f), f)
+          load(f)
+        end
+      end
+      if (msg == :after_load_dsl) && sender.respond_to?(:config)
+        # RootJobnetTemplateのdsl_filepathからCategoryを生成します
+        Tengine::Job::Category.update_for(sender.config.dsl_dir_path)
+      end
+    end
+
+
+    # 自動テストで呼び出しをフックするためのメソッド
+    def test_harness(idx, msg)
+    end
+
+    # test_harnessメソッドに渡されるidxを初期化します
+    def test_harness_clear
+      @test_harness_index = 0
+      Tengine.logger.debug("#{self.name}.test_harness_clear")
+    end
+
+    # test_harness呼び出すメソッド。
+    # ライブラリを提供する側が使用します。
+    def test_harness_hook(msg)
+      @test_harness_index ||= 0
+      @test_harness_index += 1
+      Tengine.logger.debug("#{self.name}.test_harness(#{@test_harness_index}, #{msg.inspect})")
+      test_harness(@test_harness_index, msg)
+    end
+
+  end
+
+  # DSLの記述に問題があることを示す例外
+  class DslError < ::Tengine::DslError
+  end
+
+end

data/lib/tengine/job/category.rb

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+require 'tengine/job'
+
+require 'yaml'
+require 'tengine/support/yaml_with_erb'
+
+class Tengine::Job::Category
+  include Mongoid::Document
+  include Mongoid::Timestamps
+  include Tengine::Core::FindByName
+
+  field :name       , :type => String # カテゴリ名。ディレクトリ名を元に設定されるので、"/"などは使用不可。
+  field :caption    , :type => String # カテゴリの表示名。各ディレクトリ名に対応する表示名。通常dictionary.ymlに定義する。
+
+  with_options(:class_name => "Tengine::Job::Category") do |c|
+    c.belongs_to :parent, :inverse_of => :children, :index => true
+    c.has_many   :children, :inverse_of => :parent, :order => [:name, :asc]
+  end
+
+  class << self
+    def update_for(base_dir)
+      root_dir = File.basename(base_dir)
+      dic_dir_base = File.dirname(base_dir)
+      root_jobnets = Tengine::Job::RootJobnetTemplate.all
+      root_jobnets.each do |root_jobnet|
+        dirs = File.dirname(root_jobnet.dsl_filepath || "").split('/') - ['.', '..']
+        dirs.unshift(root_dir)
+        last_category = nil
+        dic_dir = dic_dir_base
+        dirs.each do |dir|
+          caption = nil
+          dic_path = File.expand_path("dictionary.yml", dic_dir)
+          if File.exist?(dic_path)
+            # TODO dictionary.yml が不正な形の場合の処理が必要
+            hash = YAML.load_file(dic_path)
+            caption = hash[dir]
+          end
+          category = Tengine::Job::Category.find_or_create_by(
+            :name => dir,
+            :caption => caption || dir,
+            :parent_id => last_category ? last_category.id : nil)
+          dic_dir = File.join(dic_dir, dir)
+          last_category = category
+        end
+        if last_category
+          root_jobnet.category_id = last_category.id
+          root_jobnet.save!
+        end
+      end
+
+    end
+  end
+
+end

data/lib/tengine/job/connectable.rb

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+require 'tengine/job'
+
+require 'tengine_resource'
+
+module Tengine::Job::Connectable
+  extend ActiveSupport::Concern
+
+  included do
+    field :server_name    , :type => String # 接続先となるサーバ名。Tengine::Resource::Server#name を指定します
+    field :credential_name, :type => String # 接続時に必要な認証情報。Tengine::Resource::Credential#name を指定します
+
+    include Tengine::Job::MmCompatibility::Connectable
+
+    def actual_credential_name
+      credential_name || (parent ? parent.actual_credential_name : nil)
+    end
+
+    def actual_server_name
+      server_name || (parent ? parent.actual_server_name : nil)
+    end
+
+    def actual_credential
+      key = actual_credential_name
+      return nil if key.blank?
+      result = Tengine::Resource::Credential.first(:conditions => {:name => key})
+      # TODO 使用する例外クラスはこれで良いのか検討
+      raise Mongoid::Errors::DocumentNotFound.new(Tengine::Resource::Credential, key) unless result
+      result
+    end
+
+    def actual_server
+      key = actual_server_name
+      return nil if key.blank?
+      result = Tengine::Resource::Server.first(:conditions => {:name => key})
+      # TODO 使用する例外クラスはこれで良いのか検討
+      raise Mongoid::Errors::DocumentNotFound.new(Tengine::Resource::Server, key) unless result
+      result
+    end
+
+
+  end
+end

data/lib/tengine/job/drivers/job_control_driver.rb

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+
+# ジョブ制御ドライバ
+driver :job_control_driver do
+
+  on :'start.job.job.tengine' do
+    signal = Tengine::Job::Signal.new(event)
+    # activate
+    root_jobnet = Tengine::Job::RootJobnetActual.find(event[:root_jobnet_id])
+    root_jobnet.update_with_lock do
+      signal.reset
+      target_jobnet = root_jobnet.find_descendant(event[:target_jobnet_id]) || root_jobnet
+      target_job = target_jobnet.find_descendant(event[:target_job_id])
+      signal.with_paths_backup do
+        target_job.activate(signal) # transmitは既にされているはず。
+      end
+    end
+    root_jobnet.reload
+    if signal.callback
+      block = signal.callback
+      signal.callback = nil
+      block.call
+    end
+    if signal.callback
+      root_jobnet.update_with_lock(&signal.callback)
+    end
+    signal.reservations.each{|r| fire(*r.fire_args)}
+  end
+
+  on :'stop.job.job.tengine' do
+    signal = Tengine::Job::Signal.new(event)
+    root_jobnet = Tengine::Job::RootJobnetActual.find(event[:root_jobnet_id])
+    root_jobnet.update_with_lock do
+      signal.reset
+      target_jobnet = root_jobnet.find_descendant(event[:target_jobnet_id]) || root_jobnet
+      target_job = target_jobnet.find_descendant(event[:target_job_id])
+      signal.with_paths_backup do
+        target_job.stop(signal)
+      end
+    end
+    root_jobnet.reload
+    if signal.callback
+      signal.callback.call
+    end
+    signal.reservations.each{|r| fire(*r.fire_args)}
+  end
+
+  on :'finished.process.job.tengine' do
+    signal = Tengine::Job::Signal.new(event)
+    root_jobnet = Tengine::Job::RootJobnetActual.find(event[:root_jobnet_id])
+    # finish
+    root_jobnet.update_with_lock do
+      signal.reset
+      job = root_jobnet.find_descendant(event[:target_job_id])
+      job.finish(signal)
+    end
+    signal.reservations.each{|r| fire(*r.fire_args)}
+  end
+
+  on :'expired.job.heartbeat.tengine' do
+    event.tap do |e|
+      Tengine::Job::RootJobnetActual.find(e[:root_jobnet_id]).tap do |r|
+        r.update_with_lock do
+          r.find_descendant(e[:target_job_id]).phase_key = :stuck
+        end
+      end
+    end
+  end
+
+  on :'restart.job.job.tengine' do
+    signal = Tengine::Job::Signal.new(event)
+    root_jobnet = Tengine::Job::RootJobnetActual.find(event[:root_jobnet_id])
+    root_jobnet.update_with_lock do
+      signal.reset
+      job = root_jobnet.find_descendant(event[:target_job_id])
+      job.reset(signal)
+      job.transmit(signal)
+    end
+    signal.reservations.each{|r| fire(*r.fire_args)}
+  end
+
+end

data/lib/tengine/job/drivers/job_execution_driver.rb

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+
+# ジョブ起動ドライバ
+driver :job_execution_driver do
+
+  on :'start.execution.job.tengine' do
+    signal = Tengine::Job::Signal.new(event)
+    execution = signal.execution
+    root_jobnet = execution.root_jobnet
+    root_jobnet.update_with_lock do
+      signal.reset
+      execution.transmit(signal)
+    end
+    execution.save!
+    signal.reservations.each{|r| fire(*r.fire_args)}
+  end
+
+  on :'stop.execution.job.tengine' do
+    signal = Tengine::Job::Signal.new(event)
+    execution = signal.execution
+    root_jobnet = execution.root_jobnet
+    root_jobnet.update_with_lock do
+      signal.reset
+      execution.stop(signal)
+    end
+    execution.save!
+    signal.reservations.each{|r| fire(*r.fire_args)}
+  end
+
+end

data/lib/tengine/job/drivers/jobnet_control_driver.rb

@@ -0,0 +1,117 @@
+# -*- coding: utf-8 -*-
+
+# ジョブネット制御ドライバ
+driver :jobnet_control_driver do
+
+  on :'start.jobnet.job.tengine' do
+    signal = Tengine::Job::Signal.new(event)
+    root_jobnet = Tengine::Job::RootJobnetActual.find(event[:root_jobnet_id])
+    root_jobnet.update_with_lock do
+      signal.reset
+      target_jobnet = root_jobnet.find_descendant(event[:target_jobnet_id]) || root_jobnet
+      signal.with_paths_backup do
+        target_jobnet.activate(signal)
+      end
+    end
+    signal.execution.save! if event[:root_jobnet_id] == event[:target_jobnet_id]
+    signal.reservations.each{|r| fire(*r.fire_args)}
+  end
+
+  on :'success.job.job.tengine' do
+    signal = Tengine::Job::Signal.new(event)
+    root_jobnet = Tengine::Job::RootJobnetActual.find(event[:root_jobnet_id])
+    root_jobnet.update_with_lock do
+      signal.reset
+      target_job = root_jobnet.find_descendant(event[:target_job_id])
+      signal.with_paths_backup do
+        edge = target_job.next_edges.first
+        edge.transmit(signal)
+      end
+    end
+    signal.reservations.each{|r| fire(*r.fire_args)}
+  end
+
+  on :'error.job.job.tengine' do
+    signal = Tengine::Job::Signal.new(event)
+    root_jobnet = Tengine::Job::RootJobnetActual.find(event[:root_jobnet_id])
+    root_jobnet.update_with_lock do
+      signal.reset
+      target_job = root_jobnet.vertex(event[:target_job_id])
+      signal.with_paths_backup do
+        edge = target_job.next_edges.first
+        edge.close_followings
+        edge.transmit(signal)
+      end
+      # target_jobnet = target_job.parent
+      # target_jobnet.jobnet_fail(signal)
+    end
+    signal.reservations.each{|r| fire(*r.fire_args)}
+  end
+
+  on :'success.jobnet.job.tengine' do
+    signal = Tengine::Job::Signal.new(event)
+    root_jobnet = Tengine::Job::RootJobnetActual.find(event[:root_jobnet_id])
+    root_jobnet.update_with_lock do
+      signal.reset
+      target_jobnet = root_jobnet.vertex(event[:target_jobnet_id])
+      signal.with_paths_backup do
+        case target_jobnet.jobnet_type_key
+        when :finally then
+          parent = target_jobnet.parent
+          edge = parent.end_vertex.prev_edges.first
+          (edge.closed? || edge.closing?) ?
+            parent.fail(signal) :
+            parent.succeed(signal)
+        else
+          if edge = (target_jobnet.next_edges || []).first
+            edge.transmit(signal)
+          else
+            (target_jobnet.parent || signal.execution).succeed(signal)
+          end
+        end
+      end
+    end
+    signal.execution.save! if event[:root_jobnet_id] == event[:target_jobnet_id]
+    signal.reservations.each{|r| fire(*r.fire_args)}
+  end
+
+  on :'error.jobnet.job.tengine' do
+    signal = Tengine::Job::Signal.new(event)
+    root_jobnet = Tengine::Job::RootJobnetActual.find(event[:root_jobnet_id])
+    root_jobnet.update_with_lock do
+      signal.reset
+      target_jobnet = root_jobnet.find_descendant(event[:target_jobnet_id]) || root_jobnet
+      signal.with_paths_backup do
+        case target_jobnet.jobnet_type_key
+        when :finally then
+          target_jobnet.parent.fail(signal)
+        else
+          if edge = (target_jobnet.next_edges || []).first
+            edge.close_followings
+            edge.transmit(signal)
+          else
+            (target_jobnet.parent || signal.execution).fail(signal)
+          end
+        end
+      end
+      # if target_parent = target_jobnet.parent
+      #   target_parent.end_vertex.transmit(signal)
+      # end
+    end
+    signal.execution.save! if event[:root_jobnet_id] == event[:target_jobnet_id]
+    signal.reservations.each{|r| fire(*r.fire_args)}
+  end
+
+
+  on :'stop.jobnet.job.tengine' do
+    signal = Tengine::Job::Signal.new(event)
+    root_jobnet = Tengine::Job::RootJobnetActual.find(event[:root_jobnet_id])
+    root_jobnet.update_with_lock do
+      signal.reset
+      target_jobnet = root_jobnet.find_descendant(event[:target_jobnet_id]) || root_jobnet
+      target_jobnet.stop(signal)
+    end
+    signal.reservations.each{|r| fire(*r.fire_args) }
+  end
+
+end