tengine_job 0.6.9

data/spec/tengine/job/dsls/0013_hadoop_job_run.rb

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+
+require 'tengine_job'
+
+# [jobnet0013]
+# start --- job1 --- [hadoop_job_run1] --- job2 --- end
+#
+# [hadoop_job_run1]
+# start --- [hadoop_job1] --- [hadoop_job2] --- end
+#
+# [hadoop_job1]
+#          |--- map ------|
+# start ---F              J--- end
+#          |--- reduce ---|
+#
+# [hadoop_job2]
+#          |--- map ------|
+# start ---F              J--- end
+#          |--- reduce ---|
+#
+jobnet("jobnet0013", "ジョブネット0013", :vm_instance_name => "i-11111111", :credential_name => "goku-ssh-pk1") do
+  auto_sequence
+  job("job1", "ジョブ1", "import_hdfs.sh")
+  hadoop_job_run("hadoop_job_run1", "Hadoopジョブ1", "hadoop_job_run.sh") do
+    hadoop_job("hadoop_job1")
+    hadoop_job("hadoop_job2")
+  end
+  job("job2", "ジョブ2", "export_hdfs.sh")
+end

data/spec/tengine/job/dsls/0014_join_and_join.rb

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+
+require 'tengine_job'
+
+# [jobnet0014]
+#          |--- job1 ---|
+#          |            J--- job4 ---|
+# start ---F--- job2 ---|            |
+#          |                         J--- job5 --- end
+#          |--- job3 ----------------|
+#
+jobnet("jobnet0014", :instance_name => "i-11111111", :user_group_credential_name => "goku-ssh-pk1") do
+  boot_jobs("job1", "job2", "job3")
+  job("job1", "echo 'job1'", :to => "job4")
+  job("job2", "echo 'job2'", :to => "job4")
+  job("job3", "echo 'job3'", :to => "job5")
+  job("job4", "echo 'job4'", :to => "job5")
+  job("job5", "echo 'job5'")
+end

data/spec/tengine/job/dsls/0015_fork_and_fork.rb

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+
+require 'tengine_job'
+
+# [jobnet0015]
+#                   |--- job2--------------------|
+#                   |                            |
+# start --- job1 ---F              |--- job4 --- J--- end
+#                   |--- job3 ---- F             |
+#                                  |--- job5 ----|
+jobnet("jobnet0015", :instance_name => "i-11111111", :credential_name => "goku-ssh-pk1") do
+  boot_jobs("job1")
+  job("job1", "echo 'job1'", :to => ["job2", "job3"])
+  job("job2", "echo 'job2'")
+  job("job3", "echo 'job3'", :to => ["job4", "job5"])
+  job("job4", "echo 'job4'")
+  job("job5", "echo 'job5'")
+end

data/spec/tengine/job/dsls/0016_complex_fork_and_join.rb

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+
+require 'tengine_job'
+
+# [jobnet0016]
+#                                |--- job4 --- job5 ---|
+#          |--- job1 --- job3 ---F                     |
+# start ---F                     |=== J --- job6 ----- J--- end
+#          |--- job2 ------------F                     |
+#                                |--------- job7 ------|
+jobnet("jobnet0016", :instance_name => "i-11111111", :credential_name => "goku-ssh-pk1") do
+  boot_jobs("job1", "job2")
+  job("job1", "echo 'job1'", :to => ["job3"])
+  job("job2", "echo 'job2'", :to => ["job6", "job7"])
+  job("job3", "echo 'job3'", :to => ["job4", "job6"])
+  job("job4", "echo 'job4'", :to => "job5")
+  job("job5", "echo 'job5'")
+  job("job6", "echo 'job6'")
+  job("job7", "echo 'job7'")
+end

data/spec/tengine/job/dsls/0017_finally.rb

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+
+require 'tengine_job'
+
+jobnet("jobnet0017", "ジョブネット0017", :instance_name => "i-11111111", :credential_name => "goku-ssh-pk1") do
+  auto_sequence
+  job("job1", "ジョブ1", "job1.sh")
+  job("job2", "ジョブ2", "job2.sh")
+  job("job3", "ジョブ3", "job3.sh")
+  finally do
+    auto_sequence
+    job("finally_job1", "finallyジョブ1", "finally_job1.sh")
+    job("finally_job2", "finallyジョブ2", "finally_job2.sh")
+  end
+end

data/spec/tengine/job/dsls/0018_expansion.rb

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+
+require 'tengine_job'
+
+jobnet("jobnet0018_01") do
+  boot_jobs("job101", "job102", "job103")
+  job("job101", "echo 'success101'")
+  job("job102", "echo 'success102'")
+  job("job103", "echo 'success103'")
+end
+
+jobnet("jobnet0018_02") do
+  auto_sequence
+  job("job101", "echo 'success101'")
+  job("job102", "echo 'success102'")
+  job("job103", "echo 'success103'")
+end
+
+jobnet("jobnet0018") do
+  auto_sequence
+  expansion("jobnet0018_01")
+  expansion("jobnet0018_02")
+end

data/spec/tengine/job/dsls/0019_execute_job_on_event.rb

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+require 'tengine_job'
+
+jobnet("jobnet0019") do
+  auto_sequence
+  job("j11", "tengine_job_test 5 j11'")
+  job("j12", "tengine_job_test 5 j12'")
+  job("j13", "tengine_job_test 5 j13'")
+end
+
+driver :driver_for_jobnet_0019 do
+  on :foo do
+    jobnet("jobnet0019").execute(:sender => self)
+  end
+end

data/spec/tengine/job/dsls/0020_duplicated_jobnet_name.rb

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+require 'tengine_job'
+
+# 同じジョブネット名を2回指定しています
+jobnet("jobnet0020", :instance_name => "test_server1", :credential_name => "test_credential1") do
+  auto_sequence
+  job("job1", "$HOME/tengine_job_test.sh 0 job1")
+  job("job2", "$HOME/tengine_job_test.sh 0 job2")
+end
+
+jobnet("jobnet0020", :instance_name => "test_server1", :credential_name => "test_credential1") do
+  auto_sequence
+  job("job3", "$HOME/tengine_job_test.sh 0 job3")
+  job("job4", "$HOME/tengine_job_test.sh 0 job4")
+end

data/spec/tengine/job/dsls/1060_test_dir1/1060_test_dir2/0013_hadoop_job_run.rb

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+
+require 'tengine_job'
+
+# [jobnet0013]
+# start --- job1 --- [hadoop_job_run1] --- job2 --- end
+#
+# [hadoop_job_run1]
+# start --- [hadoop_job1] --- [hadoop_job2] --- end
+#
+# [hadoop_job1]
+#          |--- map ------|
+# start ---F              J--- end
+#          |--- reduce ---|
+#
+# [hadoop_job2]
+#          |--- map ------|
+# start ---F              J--- end
+#          |--- reduce ---|
+#
+jobnet("jobnet0013", "ジョブネット0013", :vm_instance_name => "i-11111111", :credential_name => "goku-ssh-pk1") do
+  auto_sequence
+  job("job1", "ジョブ1", "import_hdfs.sh")
+  hadoop_job_run("hadoop_job_run1", "Hadoopジョブ1", "hadoop_job_run.sh") do
+    hadoop_job("hadoop_job1")
+    hadoop_job("hadoop_job2")
+  end
+  job("job2", "ジョブ2", "export_hdfs.sh")
+end

data/spec/tengine/job/dsls/2003_expansion/expansion_5.rb

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+
+require 'tengine_job'
+
+jobnet('job_reference_test_6_2', '機能テスト6-2', :vm_instance_name => "test_server1", :credential_name => "test_credential1") do
+  boot_jobs('job6-2-1')
+  job('job6-2-1', '~/mm_server/spec/models/job/mm_job.sh 30 job_function_test_6-2-1', :to => ['job6-2-2', 'job6-2-3'])
+  job('job6-2-2', '~/mm_server/spec/models/job/mm_job.sh 30 job_function_test_6-2-2', :to => 'job6-2-4')
+  job('job6-2-3', '~/mm_server/spec/models/job/mm_job.sh 30 job_function_test_6-2-3', :to => 'job6-2-4')
+  job('job6-2-4', '~/mm_server/spec/models/job/mm_job.sh 30 job_function_test_6-2-4', :to => 'job6-2-3')
+end

data/spec/tengine/job/dsls/VERSION

@@ -0,0 +1 @@
+20111003160000

data/spec/tengine/job/dynamic_env_spec.rb

@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+
+describe 'hadoop_job_run' do
+  before(:all) do
+    Tengine.plugins.add(Tengine::Job::DslLoader)
+  end
+
+  def load_dsl(filename)
+    config = {
+      :action => "load",
+      :tengined => { :load_path => File.expand_path("../../../examples/#{filename}", File.dirname(__FILE__)) },
+    }
+    @version = File.read(File.expand_path("../../../examples/VERSION", File.dirname(__FILE__))).strip
+    @bootstrap = Tengine::Core::Bootstrap.new(config)
+    @bootstrap.boot
+    Tengine::Core::Setting.stub(:dsl_version).and_return(@version)
+  end
+
+  describe "基本的なジョブDSL" do
+    context "0021_dynamic_env.rb" do
+      before do
+        Tengine::Job::JobnetTemplate.delete_all
+        Tengine::Resource::PhysicalServer.delete_all
+        TestServerFixture.test_server1
+        TestCredentialFixture.test_credential1
+        load_dsl("0021_dynamic_env.rb")
+      end
+
+      shared_examples_for "実行時に環境変数を設定できる" do
+        it "j1" do
+          mock_event = mock(:event)
+          signal = Tengine::Job::Signal.new(mock_event)
+          mock_ssh = mock(:ssh)
+          Net::SSH.should_receive(:start).and_yield(mock_ssh)
+          mock_channel = mock(:channel)
+          mock_ssh.should_receive(:open_channel).and_yield(mock_channel)
+          mock_channel.should_receive(:exec).with(an_instance_of(String)) do |cmd|
+            cmd.should =~ /export FOO=BAR/
+          end
+          @root.element("j1").run(@execution)
+        end
+
+        it "j2" do
+          mock_event = mock(:event)
+          signal = Tengine::Job::Signal.new(mock_event)
+          mock_ssh = mock(:ssh)
+          Net::SSH.should_receive(:start).and_yield(mock_ssh)
+          mock_channel = mock(:channel)
+          mock_ssh.should_receive(:open_channel).and_yield(mock_channel)
+          mock_channel.should_receive(:exec).with(an_instance_of(String)) do |cmd|
+            cmd.should =~ /export SERVER_NAME=test_server1 && export DNS_NAME=localhost/
+          end
+          @root.element("j2").run(@execution)
+        end
+      end
+
+      context "rjn0021_1" do
+        before do
+          @template = Tengine::Job::RootJobnetTemplate.find_by_name!("rjn0021_1")
+          mock_sender = mock(:sender)
+          mock_sender.should_receive(:fire).with(any_args)
+          @execution = @template.execute(:sender => mock_sender)
+          @root = @execution.root_jobnet
+        end
+        it_should_behave_like "実行時に環境変数を設定できる"
+      end
+
+      context "/rjn0021/rjn0021_1" do
+        before do
+          @template = Tengine::Job::RootJobnetTemplate.find_by_name!("rjn0021")
+          mock_sender = mock(:sender)
+          mock_sender.should_receive(:fire).with(any_args)
+          @execution = @template.execute(:sender => mock_sender)
+          @root = @execution.root_jobnet.vertex_by_name_path("/rjn0021/rjn0021_1")
+        end
+        it_should_behave_like "実行時に環境変数を設定できる"
+      end
+
+      context "/rjn0021/rjn0021_2" do
+        before do
+          @template = Tengine::Job::RootJobnetTemplate.find_by_name!("rjn0021")
+          mock_sender = mock(:sender)
+          mock_sender.should_receive(:fire).with(any_args)
+          @execution = @template.execute(:sender => mock_sender)
+          @root = @execution.root_jobnet.vertex_by_name_path("/rjn0021/rjn0021_2")
+        end
+        it_should_behave_like "実行時に環境変数を設定できる"
+      end
+
+    end
+
+  end
+
+end

data/spec/tengine/job/edge_spec.rb

@@ -0,0 +1,241 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+
+describe Tengine::Job::Edge do
+  before do
+    @now = Time.now.utc
+    @event = mock(:event, :occurred_at => @now)
+    @execution = mock(:execution,
+      :id => "execution_id",
+      :estimated_time => 600,
+      :actual_estimated_end => Time.utc(2011,10,27,19,8),
+      :preparation_command => nil)
+    @execution.stub!(:retry).and_return(false) # 再実行ではないという設定
+    @execution.stub!(:in_scope?).with(any_args).and_return(true)
+    @signal = Tengine::Job::Signal.new(@event)
+    @signal.stub!(:execution).and_return(@execution)
+  end
+
+  describe :transmit do
+    context "シンプルなケース" do
+      # in [rjn0001]
+      # (S1) --e1-->(j11)--e2-->(j12)--e3-->(E1)
+      before do
+        builder = Rjn0001SimpleJobnetBuilder.new
+        builder.create_actual
+        @ctx = builder.context
+      end
+
+      it "e1をtransmitするとtransmitting、j11はactivateされてreadyになり、イベントが通知されるのを待つ" do
+        @ctx[:e1].phase_key = :active
+        @ctx[:j11].phase_key = :initialized
+        @ctx[:root].save!
+        @ctx[:e1].transmit(@signal)
+        @ctx[:e1].phase_key.should == :transmitting
+        @ctx[:j11].phase_key.should == :ready
+        @signal.reservations.length.should == 1
+        reservation = @signal.reservations.first
+        reservation.event_type_name.should == :"start.job.job.tengine"
+        reservation.options[:properties][:target_jobnet_id].should == @ctx[:root].id.to_s
+        reservation.options[:properties][:target_job_id].should == @ctx[:j11].id.to_s
+        reservation.options[:source_name].should =~ %r<^job:.+/\d+/#{@ctx[:root].id.to_s}/#{@ctx[:j11].id.to_s}$>
+      end
+
+      it "j11がactivateされると、e1はcompleteされてtransmittedになり、j11はstartingになる" do
+        @ctx[:root].phase_key = :starting
+        @ctx[:e1].phase_key = :transmitting
+        @ctx[:j11].phase_key = :ready
+        @ctx[:root].save!
+        @ctx[:j11].should_receive(:execute)
+        @execution.should_receive(:signal=).with(@signal)
+        @execution.stub(:retry).and_return(false)
+        @ctx[:j11].activate(@signal)
+        @signal.callback.should_not be_nil
+        @signal.callback.call # 2回に分けてphaseを更新するのでcallbackすることを期待しています
+        @ctx[:e1].phase_key.should == :transmitted
+        @ctx[:j11].phase_key.should == :starting
+        @signal.reservations.length.should == 0
+      end
+    end
+
+    context "分岐するケース" do
+      # in [rjn0002]
+      #              |--e2-->(j11)--e4-->|
+      # (S1)--e1-->[F1]                [J1]--e6-->(E1)
+      #              |--e3-->(j12)--e5-->|
+      before do
+        builder = Rjn0002SimpleParallelJobnetBuilder.new
+        builder.create_actual
+        @ctx = builder.context
+      end
+
+      it "e1をtransmitするとe1はtransmitted,e2とe3はtransmittingでj11とj12はreadyになる" do
+        [:e1, :e2, :e3].each{|name| @ctx[name].phase_key = :active}
+        @ctx[:j11].phase_key = :initialized
+        @ctx[:j12].phase_key = :initialized
+        @ctx[:root].save!
+        @ctx[:e1].transmit(@signal)
+        @ctx[:root].save!
+        @ctx[:root].reload
+        @ctx.edge(:e1).phase_key.should == :transmitted
+        @ctx.edge(:e2).phase_key.should == :transmitting
+        @ctx.edge(:e3).phase_key.should == :transmitting
+        @ctx.vertex(:j11).phase_key.should == :ready
+        @ctx.vertex(:j12).phase_key.should == :ready
+        @signal.reservations.length.should == 2
+        @signal.reservations.first.tap do |r|
+          r.event_type_name.should == :"start.job.job.tengine"
+          r.options[:properties][:target_jobnet_id].should == @ctx[:root].id.to_s
+          r.options[:properties][:target_job_id].should == @ctx[:j11].id.to_s
+          r.options[:source_name].should =~ %r<^job:.+/\d+/#{@ctx[:root].id.to_s}/#{@ctx[:j11].id.to_s}$>
+        end
+        @signal.reservations.last.tap do |r|
+          r.event_type_name.should == :"start.job.job.tengine"
+          r.options[:properties][:target_jobnet_id].should == @ctx[:root].id.to_s
+          r.options[:properties][:target_job_id].should == @ctx[:j12].id.to_s
+          r.options[:source_name].should =~ %r<^job:.+/\d+/#{@ctx[:root].id.to_s}/#{@ctx[:j12].id.to_s}$>
+        end
+      end
+
+      it "e4をtransmitするとtransmittedになるけどe6は変わらず" do
+        [:e4, :e5, :e6].each{|name| @ctx[name].phase_key = :active}
+        @ctx[:root].save!
+        @ctx[:e4].transmit(@signal)
+        @ctx[:root].save!
+        @ctx[:root].reload
+        @ctx[:e4].phase_key.should == :transmitted
+        @ctx[:e5].phase_key.should == :active
+        @ctx[:e6].phase_key.should == :active
+        @signal.reservations.should be_empty
+      end
+
+      it "e4をtransmitした後、e5をtransmitするとe6もtransmittedになる" do
+        @ctx[:root].phase_key = :running
+        @ctx[:e4].phase_key = :transmitted
+        @ctx[:e5].phase_key = :active
+        @ctx[:e6].phase_key = :active
+        @ctx[:root].save!
+        @ctx[:e5].transmit(@signal)
+        @ctx[:root].save!
+        @ctx[:root].reload
+        @ctx.edge(:e4).phase_key.should == :transmitted
+        @ctx.edge(:e5).phase_key.should == :transmitted
+        @ctx.edge(:e6).phase_key.should == :transmitted
+        @ctx.vertex(:J1).activatable?.should == true
+        @signal.reservations.first.tap do |r|
+          r.event_type_name.should == :"success.jobnet.job.tengine"
+          r.options[:properties][:target_jobnet_id].should == @ctx[:root].id.to_s
+        end
+      end
+    end
+
+    context "forkとjoinが直接組み合わされるケース" do
+      # in [rjn0003]
+      #                                                |--e7-->(j14)--e11-->(j16)--e14--->|
+      #              |--e2-->(j11)--e4-->(j13)--e6-->[F2]                                 |
+      # (S1)--e1-->[F1]                                |--e8-->[J1]--e12-->(j17)--e15-->[J2]--e16-->(E2)
+      #              |                                 |--e9-->[J1]                       |
+      #              |--e3-->(j12)------e5---------->[F3]                                 |
+      #                                                |--e10---->(j15)---e13------------>|
+      before do
+        builder = Rjn0003ForkJoinJobnetBuilder.new
+        builder.create_actual
+        @ctx = builder.context
+      end
+
+      it "e6.transmitしてもe12には伝搬しない" do
+        [:e5, :e6, :e7, :e8, :e9, :e10, :e12].each{|name| @ctx[name].phase_key = :active}
+        [:j14, :j15, :j17].each{|name| @ctx[name].phase_key = :initialized}
+        @ctx[:root].save!
+        @ctx[:e6].transmit(@signal)
+        @ctx[:e6].phase_key.should == :transmitted
+        @ctx[:e7].phase_key.should == :transmitting
+        @ctx[:e8].phase_key.should == :transmitted
+        @ctx[:e12].phase_key.should == :active
+        @ctx[:j14].phase_key.should == :ready
+        @signal.reservations.length.should == 1
+        @signal.reservations.first.tap do |r|
+          r.event_type_name.should == :"start.job.job.tengine"
+          r.options[:properties][:target_jobnet_id].should == @ctx[:root].id.to_s
+          r.options[:properties][:target_job_id].should == @ctx[:j14].id.to_s
+          r.options[:source_name].should =~ %r<^job:.+/\d+/#{@ctx[:root].id.to_s}/#{@ctx[:j14].id.to_s}$>
+        end
+      end
+
+      it "e5とe6の両方をtransmitするとe12に伝搬する" do
+        [:e5, :e9, :e10, :e12].each{|name| @ctx[name].phase_key = :active}
+        @ctx[:e6].phase_key = :transmitted
+        @ctx[:e7].phase_key = :transmitting
+        @ctx[:e8].phase_key = :transmitted
+        [:j14].each{|name| @ctx[name].phase_key = :ready}
+        [:j15, :j17].each{|name| @ctx[name].phase_key = :initialized}
+        @ctx[:root].save!
+        @signal.reservations.clear
+        @ctx[:e5].transmit(@signal)
+        @ctx[:e6].phase_key.should == :transmitted
+        @ctx[:e7].phase_key.should == :transmitting
+        @ctx[:e8].phase_key.should == :transmitted
+        @ctx[:e9].phase_key.should == :transmitted
+        @ctx[:e10].phase_key.should == :transmitting
+        @ctx[:e12].phase_key.should == :transmitting
+        @signal.reservations.length.should == 2
+        @signal.reservations.first.tap do |r|
+          r.event_type_name.should == :"start.job.job.tengine"
+          r.options[:properties][:target_jobnet_id].should == @ctx[:root].id.to_s
+          r.options[:properties][:target_job_id].should == @ctx[:j17].id.to_s
+        end
+        @signal.reservations.last.tap do |r|
+          r.event_type_name.should == :"start.job.job.tengine"
+          r.options[:properties][:target_jobnet_id].should == @ctx[:root].id.to_s
+          r.options[:properties][:target_job_id].should == @ctx[:j15].id.to_s
+        end
+      end
+
+    end
+
+  end
+
+  context "<BUG>再実行したジョブが準備中のままになってしまう" do
+    # in [jn0004]
+    #                         |--e3-->(j2)--e5-->|
+    # (S1)--e1-->(j1)--e2-->[F1]                [J1]--e7-->(j4)--e8-->(E1)
+    #                         |--e4-->(j3)--e6-->|
+    #
+    # in [jn0004/finally]
+    # (S2) --e9-->(jn0004_f)-e10-->(E2)
+    before do
+      builder = Rjn0004ParallelJobnetWithFinally.new
+      builder.create_actual
+      @ctx = builder.context
+    end
+
+    context "j1が失敗するとe2以降のedgeはclosedになる。" do
+      before do
+        [:e1].each{|name| @ctx[name].phase_key = :transmitted}
+        [:e2, :e3, :e4, :e5, :e6, :e7, :e8].each{|name| @ctx[name].phase_key = :closed}
+        [:root, :j1].each{|name| @ctx[name].phase_key = :error}
+        [:j2].each{|name| @ctx[name].phase_key = :ready}
+        [:j3, :j4].each{|name| @ctx[name].phase_key = :initialized}
+        @ctx[:root].save!
+      end
+
+      it "j2以降を再実行しようとしてclosedのe3に対してcompleteしても問題なく動けばOK" do
+        @now = Time.now.utc
+        @event = mock(:event, :occurred_at => @now)
+        @execution = mock(:execution,
+          :id => "execution_id",
+          :estimated_time => 600,
+          :retry => true,
+          :spot => true,
+          :target_actual_ids => [@ctx[:j2].id.to_s],
+          :actual_estimated_end => Time.utc(2011,10,27,19,8),
+          :preparation_command => "export J2_FAIL=true")
+        @execution.should_receive(:ack).with(an_instance_of(Tengine::Job::Signal))
+        @signal = Tengine::Job::Signal.new(@event)
+        @signal.stub!(:execution).and_return(@execution)
+        @ctx.vertex(:j2).activate(@signal)
+      end
+    end
+  end
+
+end