tengine_job 1.1.0 → 1.2.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    OGIwMDc3MjQxYzUzMjdiYmEzMmMxMzViZTY4MmI2MmRhNWE2NWRmZA==
+  data.tar.gz: !binary |-
+    YTJlZmE4NjFkNTViMWQ1YmJiYWMwY2NkNjAxMzIxMjRlMTQxYThiMw==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    MDM2ZGI3ZjJkMDBhMzFkYzQwNGVkZTgzYjgyMTJkNjgzYzY2NDQxN2I4MTZj
+    MGNiMzYzNTk4ZDEzYTE1ODgzZDBkM2I4OGU0NDU5YzQ4NzY2MjU4OTYzMDkw
+    YjM4ZjY0MWM5OTg5MWM4ZmM0YWUwY2E0NTEyYTE2YTFhY2Y3NTc=
+  data.tar.gz: !binary |-
+    MjQ5MDA3NjM0Zjg3ZDZkMzBhZDFkZmZkMmYyYjc1M2U1OGU3MmFkNjY2YjI4
+    MmRkYTdiZmQ4N2UzOTJiNTE0YzZlY2FhMjA3ZGQ5MzJmY2I0NzU1M2M0YmI4
+    YmQ4ZTlhYTNiNGMyNjgwMzA0NTI4N2Q5MmYzMmRjNGI5YmRmOTA=

data/Gemfile.lock

@@ -1,51 +1,69 @@
 PATH
   remote: .
   specs:
-    tengine_job (1.1.0)
-      tengine_core (~> 1.1.0)
-      tengine_resource (~> 1.1.0)
-      tengine_support (~> 1.1.0)
+    tengine_job (1.2.0)
+      tengine_core (~> 1.2.0)
+      tengine_resource (~> 1.2.0)
+      tengine_support (~> 1.2.0)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    ZenTest (4.8.2)
-    activemodel (3.2.8)
-      activesupport (= 3.2.8)
+    ZenTest (4.9.1)
+    activemodel (3.2.13)
+      activesupport (= 3.2.13)
       builder (~> 3.0.0)
-    activesupport (3.2.8)
-      i18n (~> 0.6)
+    activesupport (3.2.13)
+      i18n (= 0.6.1)
       multi_json (~> 1.0)
-    amq-client (0.8.7)
-      amq-protocol (>= 0.8.4)
+    amq-client (0.9.12)
+      amq-protocol (>= 1.2.0)
       eventmachine
-    amq-protocol (0.8.4)
-    amqp (0.8.4)
-      amq-client (~> 0.8.7)
-      amq-protocol (~> 0.8.4)
+    amq-protocol (1.2.0)
+    amqp (0.9.10)
+      amq-client (~> 0.9.12)
+      amq-protocol (~> 1.2.0)
       eventmachine
     autotest (4.4.6)
       ZenTest (>= 4.4.1)
     builder (3.0.4)
+    coderay (1.0.9)
+    columnize (0.3.6)
     daemons (1.1.9)
+    debugger (1.5.0)
+      columnize (>= 0.3.1)
+      debugger-linecache (~> 1.2.0)
+      debugger-ruby_core_source (~> 1.2.0)
+    debugger-linecache (1.2.0)
+    debugger-ruby_core_source (1.2.0)
     diff-lcs (1.1.3)
-    eventmachine (1.0.0)
+    eventmachine (1.0.3)
     factory_girl (3.3.0)
       activesupport (>= 3.0.0)
     i18n (0.6.1)
     macaddr (1.6.1)
       systemu (~> 2.5.0)
-    mongoid (3.0.15)
-      activemodel (~> 3.1)
-      moped (~> 1.1)
+    method_source (0.8.1)
+    mongoid (3.1.3)
+      activemodel (~> 3.2)
+      moped (~> 1.4.2)
       origin (~> 1.0)
       tzinfo (~> 0.3.22)
-    moped (1.3.1)
-    multi_json (1.3.6)
+    moped (1.4.5)
+    multi_json (1.7.2)
     net-ssh (2.5.2)
-    origin (1.0.11)
-    rake (0.9.2.2)
-    rdiscount (1.6.8)
+    origin (1.1.0)
+    pry (0.9.12.1)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    pry-debugger (0.2.2)
+      debugger (~> 1.3)
+      pry (~> 0.9.10)
+    pry-doc (0.4.5)
+      pry (>= 0.9)
+      yard (>= 0.8)
+    rake (10.0.4)
     right_aws (2.1.0)
       right_http_connection (>= 1.2.5)
     right_http_connection (1.3.0)
@@ -59,49 +77,61 @@ GEM
     rspec-mocks (2.10.1)
     selectable_attr (0.3.17)
       i18n
-    simplecov (0.6.4)
+    simplecov (0.7.1)
       multi_json (~> 1.0)
-      simplecov-html (~> 0.5.3)
-    simplecov-html (0.5.3)
+      simplecov-html (~> 0.7.1)
+    simplecov-html (0.7.1)
+    slop (3.4.4)
     systemu (2.5.2)
-    tengine_core (1.1.0)
+    tengine_core (1.2.0)
       activemodel (>= 3.1.0)
       activesupport (>= 3.1.0)
       daemons (~> 1.1.4)
-      mongoid (~> 3.0.0)
+      mongoid (>= 3.0.22)
       selectable_attr (~> 0.3.15)
-      tengine_event (~> 1.1.0)
-      tengine_support (~> 1.1.0)
-    tengine_event (1.1.0)
+      tengine_event (~> 1.2.0)
+      tengine_support (~> 1.2.0)
+    tengine_event (1.2.0)
       activesupport (>= 3.0.0)
-      amqp (~> 0.8.0)
-      tengine_support (~> 1.1.0)
+      amqp (~> 0.9.10)
+      tengine_support (~> 1.2.0)
       uuid (~> 2.3.4)
-    tengine_resource (1.1.0)
+    tengine_job_agent (1.2.0)
+      tengine_event (~> 1.2.0)
+      tengine_support (~> 1.2.0)
+    tengine_resource (1.2.0)
       net-ssh (~> 2.5.2)
-      tengine_core (~> 1.1.0)
-      tengine_support (~> 1.1.0)
-    tengine_resource_ec2 (1.1.0)
+      tengine_core (~> 1.2.0)
+      tengine_support (~> 1.2.0)
+      text-table (~> 1.2.2)
+      thor (>= 0.14.6)
+    tengine_resource_ec2 (1.2.0)
       right_aws (~> 2.1.0)
-      tengine_resource (~> 1.1.0)
-    tengine_support (1.1.0)
+      tengine_resource (~> 1.2.0)
+    tengine_support (1.2.0)
       activesupport (>= 3.0.0)
-    tzinfo (0.3.35)
-    uuid (2.3.5)
+    text-table (1.2.3)
+    thor (0.18.1)
+    tzinfo (0.3.37)
+    uuid (2.3.7)
       macaddr (~> 1.0)
-    yard (0.8.2.1)
+    yard (0.8.6.1)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  ZenTest (~> 4.9.0)
   autotest
   bundler
   factory_girl (~> 3.3.0)
-  rake (~> 0.9.2.2)
-  rdiscount
+  pry
+  pry-debugger
+  pry-doc
+  rake
   rspec (~> 2.10.0)
-  simplecov (~> 0.6.4)
+  simplecov
   tengine_job!
-  tengine_resource_ec2 (~> 1.1.0)
-  yard (~> 0.8.1)
+  tengine_job_agent (~> 1.2.0)
+  tengine_resource_ec2 (~> 1.2.0)
+  yard

data/bin/tengine_job

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+$LOAD_PATH << File.expand_path('../lib', File.dirname(__FILE__))
+
+require 'thor'
+require 'tengine_job'
+
+class TengineJob < Thor
+
+  private
+  def setup
+    config = Tengine::Core::Config::Core.new.tap do |c|
+      if path = (options[:config] || options["config"])
+        c.load_file(path)
+      else
+        c.load({:db => Tengine::Core::Config::DB::DEFAULT_SETTINGS})
+      end
+    end
+    require 'mongoid'
+    Mongoid.configure do |c|
+      c.send :load_configuration, config[:db]
+    end
+  end
+
+  public
+  desc "list", "list RootJobnetTemplate names"
+  method_option :config, type: :string, aliases: '-f', :desc => "config to connect MongoDB and RabbitmQ as same as tengined's config"
+  def list
+    setup
+    require 'text-table'
+    res = [%w[name type category server credential created_at updated_at]]
+    Tengine::Job::Template::RootJobnet.all.order_by({name: 1}).each do |t|
+      res << [
+        t.name,
+        t.jobnet_type_name,
+        t.category ? t.category.name : nil,
+        t.server_name,
+        t.credential_name,
+        t.created_at.iso8601,
+        t.updated_at.iso8601,
+      ]
+    end
+    $stdout.puts res.to_table(:first_row_is_head => true)
+    res
+  end
+
+  desc "kick [RootJobnetTemplate name]", "start job"
+  method_option :config, type: :string, aliases: '-f', :desc => "config to connect MongoDB and RabbitmQ as same as tengined's config"
+  def kick(name, *args)
+    setup
+    EM.run {
+      t = Tengine::Job::Template::RootJobnet.where(name: name).first
+      r = t.generate.execute
+      $stdout.puts "#{name} was executed successfully.\n#{r.inspect}"
+    }
+  end
+
+  desc "remove [RootJobnetTemplate name]", "remove job"
+  method_option :config, type: :string, aliases: '-f', :desc => "config to connect MongoDB and RabbitmQ as same as tengined's config"
+  def remove(name, *args)
+    setup
+    t = Tengine::Job::Template::RootJobnet.where(name: name).first
+    t.destroy
+    $stdout.puts "#{name} was removed successfully."
+  end
+
+end
+
+
+TengineJob.start

data/examples/0004_retry_one_layer.rb

@@ -11,14 +11,17 @@ require 'tengine_job'
 #                    {                        }
 #                    {[S2]-->[jn0004_f]-->[E2]}
 #                    {________________________}
-#                     
-jobnet("jn0004", :instance_name => "test_server1", :credential_name => "test_credential1") do
+#
+
+base_dir = File.expand_path("..", __FILE__)
+
+jobnet("jn0004", :server_name => "localhost", :credential_name => "ssh_pk") do
   boot_jobs("j1")
-  job("j1", "$HOME/0004_retry_one_layer.sh", :to => ["j2", "j3"])
-  job("j2", "$HOME/0004_retry_one_layer.sh", :to => "j4")
-  job("j3", "$HOME/0004_retry_one_layer.sh", :to => "j4")
-  job("j4", "$HOME/0004_retry_one_layer.sh")
+  job("j1", "#{base_dir}/0004_retry_one_layer.sh", :to => ["j2", "j3"])
+  job("j2", "#{base_dir}/0004_retry_one_layer.sh", :to => "j4")
+  job("j3", "#{base_dir}/0004_retry_one_layer.sh", :to => "j4")
+  job("j4", "#{base_dir}/0004_retry_one_layer.sh")
   finally do
-    job("jn0004_f", "$HOME/0004_retry_one_layer.sh")
+    job("jn0004_f", "#{base_dir}/0004_retry_one_layer.sh")
   end
 end

data/examples/0027_parallel_ssh_job

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+pid, name = *ARGV
+
+open("tmp/ssh_job#{pid}", "a") do |f|
+  f.puts("#{Time.now.to_s} #{pid} #{name} start")
+  sleep(ENV['SLEEP'] || 10)
+  f.puts("#{Time.now.to_s} #{pid} #{name} finish")
+end

data/examples/0027_parallel_ssh_jobs.rb

@@ -0,0 +1,14 @@
+require 'tengine_job'
+
+jobnet("rjn0027", job_method: :ssh_job, server_name: "localhost", credential_name: "ssh_pk") do
+  names = (1..20).map{|i| "j%03d" % i }
+
+  boot_jobs(*names)
+
+  filepath = __FILE__.sub(/jobs\.rb/, 'job')
+
+  names.each do |name|
+    job(name, "#{filepath} #{Process.pid} #{name}")
+  end
+
+end

data/lib/tengine/job.rb

@@ -4,56 +4,24 @@ require 'tengine_job'
 require 'active_support/concern'
 
 module Tengine::Job
-  autoload :DslEvaluator        , "tengine/job/dsl_evaluator"
-  autoload :DslLoader           , "tengine/job/dsl_loader"
-  autoload :DslBinder           , "tengine/job/dsl_binder"
-
-  autoload :Signal              , 'tengine/job/signal'
-
-  autoload :Execution           , "tengine/job/execution"
-
-  autoload :Category            , "tengine/job/category"
-  autoload :Edge                , "tengine/job/edge"
-  autoload :Vertex              , "tengine/job/vertex"
-
-  autoload :Start               , "tengine/job/start"
-  autoload :End                 , "tengine/job/end"
-  autoload :Junction            , "tengine/job/junction"
-  autoload :Fork                , "tengine/job/fork"
-  autoload :Join                , "tengine/job/join"
-
-  autoload :Job                 , "tengine/job/job"
-
-  autoload :Jobnet              , "tengine/job/jobnet"
-  autoload :JobnetActual        , "tengine/job/jobnet_actual"
-  autoload :RootJobnetActual    , "tengine/job/root_jobnet_actual"
-  autoload :JobnetTemplate      , "tengine/job/jobnet_template"
-  autoload :RootJobnetTemplate  , "tengine/job/root_jobnet_template"
-
-  autoload :Expansion           , "tengine/job/expansion"
-
-  autoload :Root                , "tengine/job/root"
-  autoload :NamePath            , "tengine/job/name_path"
-  autoload :Executable          , "tengine/job/executable"
-  autoload :Stoppable           , "tengine/job/stoppable"
-  autoload :Killing             , "tengine/job/killing"
-  autoload :Connectable         , "tengine/job/connectable"
-  autoload :ScriptExecutable    , "tengine/job/script_executable"
-
-  autoload :MmCompatibility     , "tengine/job/mm_compatibility"
-  autoload :ElementSelectorNotation, "tengine/job/element_selector_notation"
+  autoload :Structure, "tengine/job/structure"
+  autoload :Template , "tengine/job/template"
+  autoload :Runtime  , "tengine/job/runtime"
+  autoload :Dsl      , "tengine/job/dsl"
 
   class << self
     # tengine_coreからそのプラグインへ通知を受けるための
     def notify(sender, msg)
-Tengine::Core.stdout_logger.info("*" * 100)
-        Dir[File.expand_path("job/drivers/*.rb", File.dirname(__FILE__))].each do |f|
-Tengine::Core.stdout_logger.info("#{self.name}.notify  #{f}")
+      Tengine::Core.stdout_logger.debug("*" * 100)
+      Dir[File.expand_path("job/runtime/drivers/*.rb", File.dirname(__FILE__))].each do |f|
+        Tengine::Core.stdout_logger.debug("#{self.name}.notify  #{f}")
         end
       # if (msg == :before___evaluate__) # だと、最初にtengine/jobがrequireされる前に実行されるのでフックできません
+
+      Tengine::Core::Driveable.module_eval{ include Tengine::Job::Dsl::Binder }
+
       if (msg == :after___evaluate__)
-        Tengine::Core::Driveable.module_eval{ include Tengine::Job::DslBinder }
-        Dir[File.expand_path("job/drivers/*.rb", File.dirname(__FILE__))].each do |f|
+        Dir[File.expand_path("job/runtime/drivers/*.rb", File.dirname(__FILE__))].each do |f|
           # Tengine::Core.stdout_logger.debug("#{self.name} now evaluating #{f}")
           # sender.instance_eval(File.read(f), f)
           load(f)
@@ -61,10 +29,17 @@ Tengine::Core.stdout_logger.info("#{self.name}.notify  #{f}")
       end
       if (msg == :after_load_dsl) && sender.respond_to?(:config)
         # RootJobnetTemplateのdsl_filepathからCategoryを生成します
-        Tengine::Job::Category.update_for(sender.config.dsl_dir_path)
+        Tengine::Job::Structure::Category.update_for(sender.config.dsl_dir_path)
       end
     end
 
+    def dsl_loader
+      ::Tengine::Job::Dsl::Loader
+    end
+
+    def dsl_binder
+      ::Tengine::Job::Dsl::Binder
+    end
 
     # 自動テストで呼び出しをフックするためのメソッド
     def test_harness(idx, msg)
@@ -84,11 +59,6 @@ Tengine::Core.stdout_logger.info("#{self.name}.notify  #{f}")
       Tengine.logger.debug("#{self.name}.test_harness(#{@test_harness_index}, #{msg.inspect})")
       test_harness(@test_harness_index, msg)
     end
-
-  end
-
-  # DSLの記述に問題があることを示す例外
-  class DslError < ::Tengine::DslError
   end
 
 end

data/lib/tengine/job/dsl.rb

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+require 'tengine/job'
+
+module Tengine::Job::Dsl
+  autoload :Binder              , 'tengine/job/dsl/binder'
+  autoload :Evaluator           , 'tengine/job/dsl/evaluator'
+  autoload :Loader              , 'tengine/job/dsl/loader'
+
+  # DSLの記述に問題があることを示す例外
+  class Error < ::Tengine::DslError
+  end
+
+end

data/lib/tengine/job/{dsl_binder.rb → dsl/binder.rb}

@@ -1,12 +1,12 @@
 # -*- coding: utf-8 -*-
-require 'tengine/job'
+require 'tengine/job/dsl'
 
 # ジョブDSLをロードする際に使用される語彙に関するメソッドを定義するモジュール
-module Tengine::Job::DslBinder
-  include Tengine::Job::DslEvaluator
+module Tengine::Job::Dsl::Binder
+  include Tengine::Job::Dsl::Evaluator
 
   def jobnet(name, *args, &block)
-    Tengine::Job::RootJobnetTemplate.by_name(name)
+    Tengine::Job::Template::RootJobnet.by_name(name)
   end
 
 end