patriot-workflow-scheduler 0.6.1

data/lib/patriot/tool/patriot_command.rb

@@ -0,0 +1,48 @@
+require 'rubygems'
+require 'thor'
+
+module Patriot
+  # namespace for command line tools
+  module Tool
+
+    # new command can be added as follows
+    # require 'patriot/tool/patriot_command'
+    # module Hoge
+    #   Patriot::Tool::PatriotCommand.class_eval do
+    #     desc 'fuga', 'fuga description'
+    #     def fuga
+    #       puts 'fuga'
+    #     end
+    #   end
+    # end
+    class PatriotCommand < Thor
+      include Patriot::Util::Config
+      class_option :config,
+          :aliases  => '-c',
+          :type     => :string,
+          :desc     => 'path to configuration file'
+      no_tasks do 
+        def exit_on_failure?
+          return true
+        end
+
+        def symbolize_options(opts = {})
+          symbolized_options = {}
+          opts.each{|k,v| symbolized_options[k.to_sym] = v }
+          return symbolized_options 
+        end
+      end
+    end
+  end
+end
+
+# command implementations
+require 'patriot/tool/patriot_commands/execute'
+require 'patriot/tool/patriot_commands/register'
+require 'patriot/tool/patriot_commands/worker'
+require 'patriot/tool/patriot_commands/worker_admin'
+require 'patriot/tool/patriot_commands/validate'
+require 'patriot/tool/patriot_commands/job'
+require 'patriot/tool/patriot_commands/plugin'
+require 'patriot/tool/patriot_commands/upgrade'
+

data/lib/patriot/tool/patriot_commands/execute.rb

@@ -0,0 +1,92 @@
+module Patriot
+  module Tool
+    # namesapce for patriot comman line tools
+    module PatriotCommands
+      # execute PBC directory
+      module Execute
+
+        Patriot::Tool::PatriotCommand.class_eval do
+          desc 'execute [options] <yyyy-mm-dd[,yyyy-mm-dd]> <files/paths>+',
+              'execute patriot jobs directly'
+          method_option :filter,
+              :aliases  => '-f',
+              :type     => :string,
+              :desc     => 'regular expression for job_id'
+          method_option :debug,
+              :aliases  => '-d',
+              :type     => :boolean,
+              :default  => false,
+              :desc     => 'run in debug mode'
+          method_option :test,
+              :aliases  => '-t',
+              :type     => :boolean,
+              :default  => false,
+              :desc     => 'run in test mode'
+          method_option :strict,
+              :type     => :boolean,
+              :default  => false,
+              :desc     => 'run in strict mode (according to dependency)'
+          def execute(date, *paths)
+            begin
+              # set config/options 
+              opts = symbolize_options(options)
+              conf        = {:type => 'execute'}
+              conf[:path] = opts[:config] if opts.has_key?(:config)
+              config      = load_config(conf)
+              if opts[:debug] && opts[:test]
+                message = "invalid option: both of debug and test are specified"
+                raise ArgumentError, message
+              end
+
+              # parse and process commands
+              parser = Patriot::Tool::BatchParser.new(config)
+              commands = parser.process(date, paths, opts)
+              if opts[:strict]
+                job_store = create_job_store_with_commands(commands, config)
+                until (executables = job_store.get_job_tickets(nil,nil)).empty?
+                  executables.each do |job_ticket|
+                    cmd = job_store.offer_to_execute(job_ticket)
+                    execute_command(cmd[:command], opts)
+                    job_ticket.exit_code = Patriot::Command::ExitCode::SUCCEEDED
+                    job_store.report_completion_status(job_ticket)
+                  end
+                end
+              else
+                commands.each{|cmd| execute_command(cmd, opts)}
+              end
+            rescue => e
+              puts e.message
+              $@.each {|message| puts message}
+              raise e
+            end
+          end
+
+          no_tasks do
+            def create_job_store_with_commands(commands, config)
+              job_store = Patriot::JobStore::InMemoryStore.new(Patriot::JobStore::ROOT_STORE_ID, config)
+              # ignore products not defined here
+              products = commands.map{|cmd| cmd['products']}.flatten.compact
+              jobs = commands.map do |cmd|
+                cmd.instance_variable_set(:@state, Patriot::JobStore::JobState::INIT)
+                cmd['requisites'].delete_if{|ref| !products.include?(ref)}
+                cmd.to_job
+              end
+              job_store.register(Time.now.to_i, jobs)
+              return job_store
+            end
+
+            def execute_command(command, opts = {})
+              puts "executing #{command.job_id}"
+              if opts[:debug]
+                puts command.description
+                return
+              end
+              command.test_mode = true if opts[:test]
+              command.execute
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/patriot/tool/patriot_commands/job.rb

@@ -0,0 +1,62 @@
+module Patriot
+  module Tool
+    module PatriotCommands
+      # handle jobs in JobStore
+      module Job
+
+        Patriot::Tool::PatriotCommand.class_eval do
+          desc 'job [delete|show_dependency] job_id [job_id ..]', 'manage job(s) in job store'
+          def job(subcmd, *job_id)
+            opts = symbolize_options(options)
+            conf        = {:type => 'job'}
+            conf[:path] = opts[:config] if opts.has_key?(:config)
+            config      = load_config(conf)
+            job_store   = Patriot::JobStore::Factory.create_jobstore(Patriot::JobStore::ROOT_STORE_ID, config)
+
+            case subcmd
+            when "delete"
+              job_id.each do |jid|
+                if job_store.delete_job(jid)
+                  puts "#{jid} is deleted" 
+                else
+                  puts "#{jid} does not exist"
+                end
+              end
+            when "show_dependency"
+              job_id.each do |jid|
+                dep = ""
+                dep = [job_id] | build_producer_string_values(job_store, jid, 1)
+                puts dep.join("\n")+"\n"
+              end
+            else
+              puts "unknown sub command #{subcmd}"
+              help('job')
+            end
+          end
+
+          no_tasks do
+            def build_producer_string_values(job_store, job_id, indent)
+              values = []
+              job = job_store.get(job_id)
+              job[Patriot::Command::REQUISITES_ATTR].each do |product|
+                products = job_store.get_producers(product)
+                values << "#{'  '*indent}<= #{product} = WARN: no producer exists" if products.empty?
+                products.each do |jid, attrs|
+                  dep_status = "#{jid}, #{attrs[Patriot::Command::STATE_ATTR]}"
+                  producer_job = job_store.get(jid, :include_dependency => true)
+                  unless producer_job['consumers'].keys.include?(job_id)
+                    dep_status = "WARN: currupted dependency #{dep_status}"
+                  end
+                  values << "#{'  '*indent}<= #{product} = #{dep_status}"
+                  values |= build_producer_string_values(job_store, jid, indent+1)
+                end
+              end
+              return values
+            end
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/patriot/tool/patriot_commands/plugin.rb

@@ -0,0 +1,41 @@
+module Patriot
+  module Tool
+    module PatriotCommands
+      # manage plugins
+      module Plugin
+
+        Patriot::Tool::PatriotCommand.class_eval do
+          desc 'plugin [options] install <path to plugin>',
+              'manage plugins'
+          method_option :force,
+              :aliases  => '-f',
+              :type     => :boolean,
+              :desc     => 'force operation'
+          method_option :unpack,
+              :type     => :boolean,
+              :desc     => 'unpack gem into plugin dir'
+          def plugin(sub_cmd, *plugin)
+            opts        = symbolize_options(options)
+            conf        = {:ignore_plugin => true}
+            conf[:path] = opts[:config] if opts.has_key?(:config)
+            config      = load_config(conf)
+            controller  = Patriot::Controller::PackageController.new(config)
+            plugins = []
+            if plugin.nil? || plugin.empty?
+              plugins = config.get(Patriot::Util::Config::PLUGIN_KEY, plugin)
+            else
+              plugins = plugin
+            end
+            sub_cmd = sub_cmd.to_sym
+            if sub_cmd == :install
+              plugins.each{|name| controller.install_plugin(name, opts) }
+            else
+              help("plugin")
+            end
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/patriot/tool/patriot_commands/register.rb

@@ -0,0 +1,77 @@
+module Patriot
+  module Tool
+    module PatriotCommands
+      # register jobs to JobStore
+      module Register
+
+        Patriot::Tool::PatriotCommand.class_eval do
+          desc 'register [OPTIONS] yyyy-mm-dd[,yyyy-mm-dd] path [path file ...]', 'register jobs'
+          method_option :filter,
+              :aliases  => '-f',
+              :type     => :string,
+              :desc     => 'regular expression for Ruby'
+          method_option :debug,
+              :aliases  => '-d',
+              :type     => :boolean,
+              :default  => false,
+              :desc     => 'debug mode flag'
+          method_option :priority,
+              :aliases  => '-p',
+              :type     => :numeric,
+              :desc     => 'job priority'
+          method_option :state,
+              :aliases  => '-s',
+              :type     => :numeric,
+              :desc     => 'register as specified state'
+          method_option :keep_state,
+              :type     => :boolean,
+              :default  => false,
+              :desc     => "don't change current state of jobs (only change definition)"
+          method_option :retry_dep, 
+              :type     => :boolean, 
+              :desc     =>  'set states of dependent jobs to WAIT'
+          method_option :update_id,
+              :type     => :string,
+              :default  => Time.now.to_i,
+              :desc     => 'default value is current unixtime (default value is Time.now.to_i)'
+          def register(date, *paths)
+            begin
+              # set config/options
+              opts        = symbolize_options(options)
+              conf        = {:type => 'register'}
+              conf[:path] = opts[:config] if opts.has_key?(:config)
+              config      = load_config(conf)
+              opts        = {:update_id      => Time.now.to_i,
+                             :store_id       => Patriot::JobStore::ROOT_STORE_ID,
+                             :retry_interval => 300,
+                             :retry_limite   => 10}.merge(opts)
+              opts[:state] = nil if opts[:keep_state]
+
+              job_store = Patriot::JobStore::Factory.create_jobstore(opts[:store_id], config)
+              parser  = Patriot::Tool::BatchParser.new(config)
+              jobs    = []
+              parser.process(date, paths, opts) do |cmd|
+                job = cmd.to_job
+                job[Patriot::Command::PRIORITY_ATTR] = opts[:priority] if opts.has_key?(:priority)
+                job[Patriot::Command::STATE_ATTR] = opts[:state] if opts.has_key?(:state)
+                jobs << job
+              end
+              return if opts[:debug]
+              Patriot::Util::Retry.execute_with_retry{ job_store.register(opts[:update_id], jobs) }
+              if opts[:retry_dep]
+                job_store.process_subsequent(jobs.map(&:job_id)) do |job_store, jobs|
+                  job_store.set_state(opts[:update_id], jobs.map(&:job_id), Patriot::JobStore::JobState::WAIT)
+                end
+              end
+            rescue => e
+              puts e
+              $@.each{|m| puts m}
+              raise e.message
+            end
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/patriot/tool/patriot_commands/upgrade.rb

@@ -0,0 +1,24 @@
+module Patriot
+  module Tool
+    module PatriotCommands
+      # upgrade tool
+      module Upgrade
+
+        Patriot::Tool::PatriotCommand.class_eval do
+          desc 'upgrade',
+               'upgrade installation'
+          def upgrade(*path_to_gem)
+            opts        = symbolize_options(options)
+            conf        = {:ignore_plugin => true}
+            conf[:path] = opts[:config] if opts.has_key?(:config)
+            config      = load_config(conf)
+            controller  = Patriot::Controller::PackageController.new(config)
+            controller.upgrade(path_to_gem.empty? ? nil : path_to_gem[0])
+          end
+
+        end
+      end
+    end
+  end
+end
+

data/lib/patriot/tool/patriot_commands/validate.rb

@@ -0,0 +1,84 @@
+module Patriot
+  module Tool
+    module PatriotCommands
+      # validate PBC files
+      module Validate
+
+        Patriot::Tool::PatriotCommand.class_eval do
+          VALIDATE_SAMPLE_DATE = '1999-01-31'
+          desc 'validate [OPTIONS] path [path file ...]', 'validate pbc files'
+          method_option :stop_on_detection,
+              :aliases  => '-s',
+              :type     => :boolean,
+              :default  => false,
+              :desc     => 'stop immediately when invalid config detected'
+          method_option :date,
+              :type     => :string,
+              :desc     => 'date passed to parser'
+          def validate(*paths)
+            begin
+              opts        = symbolize_options(options)
+              conf        = {:type => 'validator'}
+              conf[:path] = opts[:config] if opts.has_key?(:config)
+              config      = load_config(conf)
+              # the value of :all is passed to Patriot::Util::Script.get_batch_files(path, opt = {})
+              opts = {:all => true, :date => VALIDATE_SAMPLE_DATE}.merge(opts)
+
+              job_store      = Patriot::JobStore::Factory.create_jobstore(Patriot::JobStore::ROOT_STORE_ID, config)
+              job_ids        = {}
+              invalid_syntax = []
+              valid           = true
+
+              parser = Patriot::Tool::BatchParser.new(config)
+              parser.process(opts[:date], paths, opts) do |cmd, source|
+                unless job_store.acceptable?(cmd.to_job)
+                  invalid_syntax << "#{command.job_id} in ${source[:path]}"
+                  valid = false
+                end
+                get_all_job_ids(cmd).each do |jid|
+                  if job_ids.has_key?(jid)
+                    job_ids[jid] << source[:path]
+                    valid = false
+                  else
+                    job_ids[jid] = [source[:path]]
+                  end
+                end
+                break if opts[:stop_on_detection] && !valid
+              end
+
+              unless valid
+                unless invalid_syntax.empty?
+                  puts "#{invalid_syntax.size} jobs are serialized to invalid syntax:"
+                  invalid_syntax.each{|i| puts "\ti" }
+                end
+
+                # count dupliates
+                dup_cnt = job_ids.values.select{|files| files.size > 1}
+                unless dup_cnt == 0
+                  puts "#{dup_cnt} duplications are detected:"
+                  job_ids.each do |jid, files|
+                    next if files.size == 1
+                    puts "#{files.size} #{jid} in "
+                    files.each{|d| puts "\t#{d} " }
+                  end
+                end
+                raise "invalid batch config is detected"
+              end
+              puts "no invalid config is detected"
+            end
+          end
+
+          no_tasks do
+            def get_all_job_ids(cmd)
+              ids = [cmd.job_id]
+              if cmd.is_a?(Patriot::Command::CompositeCommand)
+                ids |= cmd.instance_variable_get(:@contained_commands).map{|cc| get_all_job_ids(cc)}.flatten
+              end
+              return ids
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/patriot/tool/patriot_commands/worker.rb

@@ -0,0 +1,35 @@
+module Patriot
+  module Tool
+    module PatriotCommands
+      # a tool to start/stop a worker
+      module Worker
+
+        Patriot::Tool::PatriotCommand.class_eval do
+          desc 'worker [options] [start|stop]',
+               'controll worker'
+          method_option :foreground,
+              :type     => :boolean,
+              :default  => false,
+              :desc     => 'run as a foreground job'
+          def worker(sub_cmd)
+            opts = symbolize_options(options)
+            conf        = {:type => 'worker'}
+            conf[:path] = opts[:config] if opts.has_key?(:config)
+            Process.daemon unless opts[:foreground]
+            config      = load_config(conf)
+            worker_cls  = config.get("worker_class", "Patriot::Worker::MultiNodeWorker")
+            worker      = eval(worker_cls).new(config)
+            case sub_cmd
+            when "start"
+              then worker.start_worker
+            when "stop"
+              then worker.request_shutdown
+            else
+              raise "unknown sub command #{sub_cmd}"
+            end
+          end
+        end
+      end
+    end
+  end
+end