tengine_job 0.6.9

data/spec/fixtures/rjn_0011_nested_fork_jobnet_builder.rb

@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+
+require File.expand_path('test_credential_fixture', File.dirname(__FILE__))
+require File.expand_path('test_server_fixture', File.dirname(__FILE__))
+
+# 以下のジョブネットについてテンプレートジョブネットや
+# 実行用ジョブネットを扱うフィクスチャ生成のためのクラスです。
+#
+# in [rjn0011]
+# (S1)--e1-->[j1100]--e2-->(j1200)--e3-->[j1300]--e4-->(E1)
+#
+# in [j1100]
+# (S2)--e5-->(j1110)--e6-->[j1120]--e7-->[j1130]--e8-->(j1140)--e9-->(E2)
+#
+# in [j1120]
+# (S3)--e10-->(j1121)--e11-->(E3)
+#
+# in [j1130]
+# (S4)--e12-->(j1131)--e13-->(E4)
+#
+# in [j1300]
+# (S5)--e14-->(j1310)--e15-->(E5)
+#
+class Rjn0011NestedForkJobnetBuilder < JobnetFixtureBuilder
+  include TestCredentialFixture
+  include TestServerFixture
+
+  DSL = <<-EOS
+    jobnet("rjn0011", :server_name => "test_server1", :credential_name => "test_credential1") do
+      auto_sequence
+      jobnet("j1100") do
+        job("j1110", "job_test j1110")
+        jobnet("j1120") do
+          job("j1121", "job_test j1121")
+        end
+        jobnet("j1130") do
+          job("j1131", "job_test j1131")
+        end
+        job("j1140", "job_test j1140")
+      end
+      job("j1200", "job_test j1200")
+      jobnet("j1300") do
+        job("j1310", "job_test j1310")
+      end
+    end
+  EOS
+
+  def create(options = {})
+    root = new_root_jobnet("rjn0011", :server_name => test_server1.name, :credential_name => test_credential1.name)
+    root.children << new_start
+    root.children << new_jobnet("j1100")
+    root.children << new_script("j1200", :script => "job_test j1200")
+    root.children << new_jobnet("j1300")
+    root.children << new_end
+    root.edges << new_edge(:S1   , :j1100)
+    root.edges << new_edge(:j1100, :j1200)
+    root.edges << new_edge(:j1200, :j1300)
+    root.edges << new_edge(:j1300, :E1  )
+    self[:j1100].tap do |j1100|
+      j1100.children << new_start
+      j1100.children << new_script("j1110", :script => "job_test j1110")
+      j1100.children << new_jobnet("j1120")
+      j1100.children << new_jobnet("j1130")
+      j1100.children << new_script("j1140", :script => "job_test j1140")
+      j1100.children << new_end
+      j1100.edges << new_edge(:S2   , :j1110)
+      j1100.edges << new_edge(:j1110, :j1120)
+      j1100.edges << new_edge(:j1120, :j1130)
+      j1100.edges << new_edge(:j1130, :j1140)
+      j1100.edges << new_edge(:j1140, :E2   )
+    end
+    self[:j1120].tap do |j1120|
+      j1120.children << new_start
+      j1120.children << new_script("j1121", :script => "job_test j1121")
+      j1120.children << new_end
+      j1120.edges << new_edge(:S3   , :j1121)
+      j1120.edges << new_edge(:j1121, :E3   )
+    end
+    self[:j1130].tap do |j1130|
+      j1130.children << new_start
+      j1130.children << new_script("j1131", :script => "job_test j1131")
+      j1130.children << new_end
+      j1130.edges << new_edge(:S4   , :j1131)
+      j1130.edges << new_edge(:j1131, :E4   )
+    end
+    self[:j1300].tap do |j1300|
+      j1300.children << new_start
+      j1300.children << new_script("j1310", :script => "job_test j1310")
+      j1300.children << new_end
+      j1300.edges << new_edge(:S5   , :j1310)
+      j1300.edges << new_edge(:j1310, :E5   )
+    end
+    root.save!
+    root
+  end
+end

data/spec/fixtures/rjn_0012_nested_and_finally_builder.rb

@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*-
+# 以下のジョブネットについてテンプレートジョブネットや
+# 実行用ジョブネットを扱うフィクスチャ生成のためのクラスです。
+#
+# in [rjn0012]
+# (S1)--e1-->[j1000]--e2-->[j2000]--e3-->(E1)
+#
+# in [j1000]
+# (S2)--e4-->[j1100]--e5-->[j1200]--e6-->(E2)
+#
+# in [j1100]
+# (S3)--e7-->(j1110)--e8-->(E3)
+#
+# in [j1200]
+# (S4)--e9-->(j1210)--e10-->(E4)
+#
+# in [j1000:finally (=j1f00)]
+# (S5)--e11-->[j1f10]--e12-->(E5)
+#
+# in [j1f10]
+# (S6)--e13-->(j1f11)--e14-->(E6)
+#
+# in [j1000:finally:finally (=j1ff0)]
+# (S7)--e15-->(j1ff1)--e16-->(E7)
+#
+# in [j2000]
+# (S8)--e17-->(j2100)--e18-->(E8)
+#
+# in [jf000:finally (=jf000)]
+# (S9)--e19-->(jf100)--e20-->(E9)
+#
+
+class Rjn0012NestedAndFinallyBuilder < JobnetFixtureBuilder
+  DSL = <<-EOS
+    jobnet("rjn0012") do
+      auto_sequence
+      jobnet("j1000") do
+        jobnet("j1100") do
+          job("j1110", "job_test j1110")
+        end
+        jobnet("j1200") do
+          job("j1210", "job_test j1210")
+        end
+        finally do
+          jobnet("j1f10") do
+            job("j1f11", "job_test j1f11")
+          end
+          finally do
+            job("j1ff1", "job_test j1ff1")
+          end
+        end
+      end
+
+      jobnet("j2000") do
+        job("j2100", "job_test j2100")
+      end
+
+      finally do
+        job("jf100", "job_test jf100")
+      end
+    end
+  EOS
+
+  def create(options = {})
+    root = new_root_jobnet("rjn0012")
+    root.children << new_start
+    root.children << new_jobnet("j1000")
+    root.children << new_jobnet("j2000")
+    root.children << new_finally
+    root.children << new_end
+    root.edges << new_edge(:S1   , :j1000)
+    root.edges << new_edge(:j1000, :j2000)
+    root.edges << new_edge(:j2000, :E1  )
+
+    self[:j1000].tap do |j1000|
+      j1000.children << new_start
+      j1000.children << new_jobnet("j1100")
+      j1000.children << new_jobnet("j1200")
+      j1000.children << new_finally
+      j1000.children << new_end
+      j1000.edges << new_edge(:S2   , :j1100)
+      j1000.edges << new_edge(:j1100, :j1200)
+      j1000.edges << new_edge(:j1200, :E2   )
+
+      self[:j1100].tap do |j1100|
+        j1100.children << new_start
+        j1100.children << new_script("j1110", :script => "job_test j1110")
+        j1100.children << new_end
+        j1100.edges << new_edge(:S3   , :j1110)
+        j1100.edges << new_edge(:j1110, :E3   )
+      end
+      self[:j1200].tap do |j1200|
+        j1200.children << new_start
+        j1200.children << new_script("j1210", :script => "job_test j1210")
+        j1200.children << new_end
+        j1200.edges << new_edge(:S4   , :j1210)
+        j1200.edges << new_edge(:j1210, :E4   )
+      end
+
+      self[:j1f00] = j1000.finally_vertex
+      self[:j1f00].tap do |j1f00|
+        j1f00.children << new_start
+        j1f00.children << new_jobnet("j1f10")
+        j1f00.children << new_finally
+        j1f00.children << new_end
+        j1f00.edges << new_edge(:S5   , :j1f10)
+        j1f00.edges << new_edge(:j1f10, :E5   )
+        self[:j1f10].tap do |j1f10|
+          j1f10.children << new_start
+          j1f10.children << new_script("j1f11", :script => "job_test j1f11")
+          j1f10.children << new_end
+          j1f10.edges << new_edge(:S6   , :j1f11)
+          j1f10.edges << new_edge(:j1f11, :E6   )
+        end
+        self[:j1ff0] = j1f00.finally_vertex
+        self[:j1ff0].tap do |j1ff0|
+          j1ff0.children << new_start
+          j1ff0.children << new_jobnet("j1ff1")
+          j1ff0.children << new_end
+          j1ff0.edges << new_edge(:S7   , :j1ff1)
+          j1ff0.edges << new_edge(:j1ff1, :E7   )
+        end
+      end
+    end
+
+    self[:j2000].tap do |j2000|
+      j2000.children << new_start
+      j2000.children << new_script("j2100", :script => "job_test j2100")
+      j2000.children << new_end
+      j2000.edges << new_edge(:S8   , :j2100)
+      j2000.edges << new_edge(:j2100, :E8   )
+    end
+
+    self[:jf000] = root.finally_vertex
+    self[:jf000].tap do |jf000|
+      jf000.children << new_start
+      jf000.children << new_script("jf100")
+      jf000.children << new_end
+      jf000.edges << new_edge(:S9   , :jf100)
+      jf000.edges << new_edge(:jf100, :E9   )
+    end
+
+    unless root.valid?
+      v = Tengine::Job::Vertex::AllVisitorWithEdge.new do |obj|
+        if obj.respond_to?(:errors) && !obj.errors.empty?
+          puts obj.errors.inspect
+          true
+        end
+      end
+      root.accept_visitor(v)
+    end
+
+    root.save!
+
+    root
+  end
+end

data/spec/fixtures/rjn_1004_hadoop_job_in_jobnet_fixture.rb

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+
+require File.expand_path('test_credential_fixture', File.dirname(__FILE__))
+require File.expand_path('test_server_fixture', File.dirname(__FILE__))
+
+# 以下のジョブネットについてテンプレートジョブネットや
+# 実行用ジョブネットを扱うフィクスチャ生成のためのクラスです。
+#
+# [rjn1004]
+# (S1)--e1-->(job1)--e2-->[hadoop_job_run1]--e3-->(job2)--e4-->(E1)
+#
+# [hadoop_job_run1]
+# (S2)--e5-->[hadoop_job1]--e6-->[hadoop_job2]--e7-->(E2)
+#
+# [hadoop_job1]
+#              |--e9--->(Map)------e11-->|
+# (S3)--e8-->[F1]                      [J1]--e13-->(E3)
+#              |--e10-->(Reduce) --e12-->|
+#
+# [hadoop_job2]
+#               |--e15-->(Map)------e17-->|
+# (S4)--e14-->[F2]                      [J2]--e19-->(E4)
+#               |--e16-->(Reduce) --e18-->|
+
+class Rjn1004HadoopJobInJobnetFixture < JobnetFixtureBuilder
+  include TestCredentialFixture
+  include TestServerFixture
+
+  DSL = <<-EOS
+    require 'tengine_job'
+
+    jobnet("jobnet1004", :instance_name => "test_server1", :credential_name => "test_credential1") do
+      auto_sequence
+      job("job1", "$HOME/import_hdfs.sh")
+      hadoop_job_run("hadoop_job_run1", "Hadoopジョブ1", "$HOME/hadoop_job_run.sh") do
+        hadoop_job("hadoop_job1")
+        hadoop_job("hadoop_job2")
+      end
+      job("job2", "$HOME/export_hdfs.sh")
+    end
+  EOS
+
+  def create(options = {})
+    root = new_root_jobnet("rjn1004", {
+        :server_name => test_server1.name, 
+        :credential_name => test_credential1.name
+      }.update(options || { }))
+    root.children << new_start
+    root.children << new_script("job1", :script => "$HOME/import_hdfs.sh")
+    root.children << new_jobnet("hadoop_job_run1", :jobnet_type_key => :hadoop_job_run,
+      :description => "Hadoopジョブ1", :script => "$HOME/hadoop_job_run.sh")
+    root.children << new_script("job2", :script => "$HOME/export_hdfs.sh")
+    root.children << new_end
+    root.build_sequencial_edges
+    self[:e1] = root.edges[0]
+    self[:e2] = root.edges[1]
+    self[:e3] = root.edges[2]
+    self[:e4] = root.edges[3]
+
+    self[:hadoop_job_run1].tap do |hadoop_job_run1|
+      hadoop_job_run1.children << new_start
+      hadoop_job_run1.children << new_jobnet("hadoop_job1", :jobnet_type_key => :hadoop_job)
+      hadoop_job_run1.children << new_jobnet("hadoop_job2", :jobnet_type_key => :hadoop_job)
+      hadoop_job_run1.children << new_end
+      hadoop_job_run1.build_sequencial_edges
+      self[:e5] = hadoop_job_run1.edges[0]
+      self[:e6] = hadoop_job_run1.edges[1]
+      self[:e7] = hadoop_job_run1.edges[2]
+
+
+      self[:hadoop_job1].tap do |hadoop_job1|
+        hadoop_job1.children << new_start
+        hadoop_job1.children << new_fork
+        hadoop_job1.children << new_jobnet("Map", :jobnet_type_key => :map_phase)
+        hadoop_job1.children << new_jobnet("Reduce", :jobnet_type_key => :reduce_phase)
+        hadoop_job1.children << new_join
+        hadoop_job1.children << new_end
+        hadoop_job1.edges << new_edge(:S3, :F1)
+        hadoop_job1.edges << new_edge(:F1, :Map   )
+        hadoop_job1.edges << new_edge(:F1, :Reduce)
+        hadoop_job1.edges << new_edge(:Map   , :J1)
+        hadoop_job1.edges << new_edge(:Reduce, :J1)
+        hadoop_job1.edges << new_edge(:J1, :E3)
+      end
+
+      self[:hadoop_job2].tap do |hadoop_job1|
+        hadoop_job1.children << new_start
+        hadoop_job1.children << new_fork
+        hadoop_job1.children << new_jobnet("Map", :jobnet_type_key => :map_phase)
+        hadoop_job1.children << new_jobnet("Reduce", :jobnet_type_key => :reduce_phase)
+        hadoop_job1.children << new_join
+        hadoop_job1.children << new_end
+        hadoop_job1.edges << new_edge(:S4, :F2)
+        hadoop_job1.edges << new_edge(:F2, :Map   )
+        hadoop_job1.edges << new_edge(:F2, :Reduce)
+        hadoop_job1.edges << new_edge(:Map   , :J2)
+        hadoop_job1.edges << new_edge(:Reduce, :J2)
+        hadoop_job1.edges << new_edge(:J2, :E4)
+      end
+    end
+
+    root.save!
+    root
+  end
+end

data/spec/fixtures/test_credential_fixture.rb

@@ -0,0 +1,12 @@
+module TestCredentialFixture
+  module_function
+
+  def test_credential1
+    Tengine::Resource::Credential.find_or_create_by(
+      :name => "test_credential1",
+      :auth_type_key => :ssh_password,
+      :auth_values => { :username => "goku", :password => "dragonball"}
+      )
+  end
+
+end

data/spec/fixtures/test_server_fixture.rb

@@ -0,0 +1,28 @@
+module TestServerFixture
+  module_function
+
+  def test_server1
+    Tengine::Resource::PhysicalServer.find_or_create_by(
+      :name => "test_server1",
+      :provided_id => "test_server1",
+      :properties => {},
+      :addresses => {
+        :private_dns_name => "localhost",
+        # :private_ip_address => "127.0.0.1"
+      }
+      )
+  end
+
+  def test_server2
+    Tengine::Resource::PhysicalServer.find_or_create_by(
+      :name => "test_server2",
+      :provided_id => "test_server2",
+      :properties => {},
+      :addresses => {
+        :private_dns_name => "test_server2",
+        :private_ip_address => "192.168.1.2"
+      }
+      )
+  end
+
+end

data/spec/mongoid.yml

@@ -0,0 +1,35 @@
+defaults: &defaults
+  # mongoid defaults for configurable settings
+  # autocreate_indexes: false
+  # allow_dynamic_fields: true
+  # include_root_in_json: false
+  # parameterize_keys: true
+  persist_in_safe_mode: true
+  # raise_not_found_error: true
+  # reconnect_time: 3
+  logger: false
+
+development:
+  <<: *defaults
+  host: localhost
+  database: tengine_job_development
+
+test:
+  <<: *defaults
+  host: localhost
+  database: tengine_job_test
+  autocreate_indexes: true
+
+# set these environment variables on your prod server
+production:
+  <<: *defaults
+  host: <%= ENV['MONGOID_HOST'] %>
+  port: <%= ENV['MONGOID_PORT'] %>
+  username: <%= ENV['MONGOID_USERNAME'] %>
+  password: <%= ENV['MONGOID_PASSWORD'] %>
+  database: <%= ENV['MONGOID_DATABASE'] %>
+  # slaves:
+  #   - host: slave1.local
+  #     port: 27018
+  #   - host: slave2.local
+  #     port: 27019

data/spec/spec_helper.rb

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+ENV["RACK_ENV"] ||= "test" # Mongoid.load!で参照しています
+
+require 'simplecov'
+SimpleCov.start if ENV["COVERAGE"]
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rspec'
+require 'factory_girl'
+
+require 'tengine_job'
+require 'mongoid'
+Mongoid.load!(File.expand_path('mongoid.yml', File.dirname(__FILE__)))
+Mongoid.database.connection.drop_database(Mongoid.database.name)
+
+gem_names = ["tengine_core", "tengine_resource"]
+gem_names.each{|f| require f}
+
+base_dirs = gem_names.map{|gem_name| Gem.loaded_specs[gem_name].gem_dir}
+# base_dirs += [File.expand_path("..", File.dirname(__FILE__))]
+base_dirs += [File.expand_path(".")]
+base_dirs.each do |dir_path|
+  # fixtures/以下のファイルがsupport以下のファイルに依存していることがあるので、
+  # あえて２回検索しています
+  Dir["#{dir_path}/spec/support/**/*.rb"].each {|f| require f}
+  Dir["#{dir_path}/spec/fixtures/**/*.rb"].each {|f| require f}
+end
+
+
+Tengine::Core::MethodTraceable.disabled = true
+require 'logger'
+log_path = File.expand_path("../tmp/log/test.log", File.dirname(__FILE__))
+Tengine.logger = Logger.new(log_path)
+Tengine.logger.level = Logger::DEBUG
+Tengine::Core.stdout_logger = Tengine.logger
+Tengine::Core.stderr_logger = Tengine.logger
+# Tengine::Core.stdout_logger = Logger.new(log_path)
+# Tengine::Core.stdout_logger.level = Logger::DEBUG
+# Tengine::Core.stderr_logger = Logger.new(log_path)
+# Tengine::Core.stderr_logger.level = Logger::DEBUG
+
+Tengine::Core::Kernel.event_exception_reporter = :raise_all
+
+RSpec.configure do |config|
+  config.include Factory::Syntax::Methods
+
+  config.before(:all) do
+    unless Tengine::Core::Setting.first(:conditions => {:name => "dsl_version"})
+      Tengine::Core::Setting.create!(:name => "dsl_version", :value => "1234567890")
+    end
+  end
+
+end
+
+Dir["#{File.expand_path('factories', File.dirname(__FILE__))}/**/*.rb"].each {|f| require f}