patriot-workflow-scheduler 0.7.0 → 0.7.1

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MjBkMGRiMGQ3ZGU1MTk0OThiN2VkNjBhYmU3YjA1YjVjMmQ4OGU1Mg==
+    ZDBiZjc1ZDU4YjBkNzIzZjlhODliMmE2NzVhM2YyZTQxMzY1ZGZhMA==
   data.tar.gz: !binary |-
-    ODMwYTA4NWZhNzQ0MWY1M2ZkZDg0NjI4ZmIwYTA3ZTkzOGRlZTQyZg==
+    YzliMTkzNDg4NDAzNTJjNGRhOGM0ZWZiY2VmNzQ3OTNhNDQyMmEyMQ==
 SHA512:
   metadata.gz: !binary |-
-    YzRkMTVjOThmYTYzYTIxODEwMmNhODNjZGQ1YTBmYzBjYzVjNDhjODg0OTdj
-    NDBkZDZhOTkwNDFhYjM0ZjZlYjlkNzQ4ZWUzMDc1NmU4MWJjMWNkYzc5ZGE1
-    ZWIxNjU1MDJhYTE0MWI1YmQ2YTE3NjE4OTQ0NGFhNzNkZjk4MGU=
+    NjgxNmU0ZTJiZTk0MzFhMmE4NzAxOTliY2I2YmQ2ZjJlMDM5OWUyYjQ1YjJh
+    NWQ3OTVmOWUyZGJiYzY5OTViNThhNTExZTRiMWRhOWQ1ZThjMGVlNmFmOTdk
+    MzZmMmY3ZTlkYTkzYjVlMTU5YTYyNGRlYmQzMTRlYjZjYmY5YmY=
   data.tar.gz: !binary |-
-    ODdkZTM5YjliNWI5MDI3YmNkNzVlM2FkNzNkNzg4MDI5YTM0ZDYzNzJkZTc3
-    ODlkMDljNGE1YzA0MWVhNWM4MzAyOTExNDMzNGY1YWEyOGU5YzdhM2M1YTc1
-    M2UxYzMzMzM0NDM1ZjdjNWZkZGM4MmRkYzAxNTc0NTRkMmJlYTg=
+    NGRmZDU1ZmVmZjJlNGE4ZTg0MmJkNjc2NzI5OTk2MDZkYTNkOWRjZGIzMjdm
+    MjBmZDVlNjU2ZTAzZTQzMjAzYWVhMDhhZTA3NjkwOGJkNzY3OWY1YzYzNDU4
+    ZDdmMmZiNTJmMWU0ODJlZmUwYTM1OGM0OWFjY2I0M2M5NzJiYTc=

data/lib/patriot/command/composite.rb

@@ -9,7 +9,7 @@ module Patriot
 
       # @return [String] the identifier of this composite command
       # @see Patriot::Command::Base#job_id
-      def job_id  
+      def job_id
         return "#{command_name}_#{@name}_#{@name_suffix}"
       end
 

data/lib/patriot/command/post_processor/retrial.rb

@@ -25,6 +25,9 @@ module Patriot
             cmd.start_datetime = Time.now + pp.props[INTERVAL_PROP_KEY]
           end
           job = cmd.to_job
+          current_config = worker.job_store.get_job(job.job_id)
+          job[Patriot::Command::PRODUCTS_ATTR] = current_config[Patriot::Command::PRODUCTS_ATTR]
+          job[Patriot::Command::REQUISITES_ATTR] = current_config[Patriot::Command::REQUISITES_ATTR]
           job[Patriot::Command::STATE_ATTR] = Patriot::JobStore::JobState::WAIT
           worker.job_store.register(Time.now.to_i, [job])
         end

data/lib/patriot/job_store/in_memory_store.rb

@@ -39,8 +39,8 @@ module Patriot
               job[Patriot::Command::STATE_ATTR] ||= Patriot::JobStore::JobState::INIT
             end
             @jobs[job_id]      = job
-            @producers[job_id] = job[Patriot::Command::PRODUCTS_ATTR] unless job[Patriot::Command::PRODUCTS_ATTR].nil?
-            @consumers[job_id] = job[Patriot::Command::REQUISITES_ATTR] unless job[Patriot::Command::REQUISITES_ATTR].nil?
+            @producers[job_id] = job[Patriot::Command::PRODUCTS_ATTR] || []
+            @consumers[job_id] = job[Patriot::Command::REQUISITES_ATTR] || []
             if job[Patriot::Command::STATE_ATTR] == Patriot::JobStore::JobState::INIT
               _set_state(job_id, Patriot::JobStore::JobState::WAIT)
             else

data/lib/patriot/job_store/rdb_job_store.rb

@@ -21,7 +21,7 @@ module Patriot
       HISTORY_TABLE  = 'job_histories'
 
       # date format of execution history
-      HISTORY_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+      DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
 
       # attributes included in job_ticket
       TICKET_COLUMNS = ['job_id', 'update_id', 'node']
@@ -72,18 +72,20 @@ module Patriot
 
       def upsert_job(update_id, job, c)
         new_vals = {:job_id => job.job_id, :update_id => update_id, :priority => DEFAULT_PRIORITY}
+        job_attr = job.attributes.dup
         # extract and remove comman attributes
-        requisites = job.delete(Patriot::Command::REQUISITES_ATTR) || []
-        products   = job.delete(Patriot::Command::PRODUCTS_ATTR)   || []
+        requisites = job_attr.delete(Patriot::Command::REQUISITES_ATTR) || []
+        products   = job_attr.delete(Patriot::Command::PRODUCTS_ATTR)   || []
 
         prev_vals = c.select(JOB_TABLE, {:job_id => job.job_id})
         ATTR_TO_COLUMN.each do |a,c|
-          val = job.delete(a)
+          val = job_attr.delete(a)
           next if val.nil? && c == :state
+          val = val.strftime(DATE_FORMAT) if !val.nil? && a == Patriot::Command::START_DATETIME_ATTR
           new_vals[c] = val
         end
         # serialize remaining attributes
-        new_vals[:content] = JSON.generate(job.attributes)
+        new_vals[:content] = JSON.generate(job_attr)
 
         if prev_vals.empty?
           new_vals[:state] ||= Patriot::JobStore::JobState::INIT # set default state
@@ -184,7 +186,7 @@ END_OB_QUERY
                                   :node       => job_ticket.exec_node,
                                   :host       => job_ticket.exec_host,
                                   :thread     => job_ticket.exec_thread,
-                                  :begin_at   => Time.now.strftime(HISTORY_DATE_FORMAT)})
+                                  :begin_at   => Time.now.strftime(DATE_FORMAT)})
           record = c.select(JOB_TABLE, {:job_id => job_ticket.job_id})
           raise "duplicated entry found for #{job_ticket}" if record.size > 1
           raise "no entry found for #{job_ticket}" if record.empty?
@@ -205,7 +207,7 @@ END_OB_QUERY
         post_state = Patriot::JobStore::EXIT_CODE_TO_STATE[exit_code]
         raise "illegal exit_code #{exit_code}" if post_state.nil?
         connect(@db_config) do |c|
-          if c.update(HISTORY_TABLE, {:end_at => Time.now.strftime(HISTORY_DATE_FORMAT), :exit_code => exit_code, :description => job_ticket.description}, {:id => job_ticket.execution_id}) != 1
+          if c.update(HISTORY_TABLE, {:end_at => Time.now.strftime(DATE_FORMAT), :exit_code => exit_code, :description => job_ticket.description}, {:id => job_ticket.execution_id}) != 1
             @logger.warn "illegal state of history for #{job_ticket.job_id}"
           end
           return _check_and_set_state(job_ticket, Patriot::JobStore::JobState::RUNNING, post_state, c)

data/lib/patriot/tool/patriot_commands/register.rb

@@ -57,7 +57,7 @@ module Patriot
                 jobs << job
               end
               return if opts[:debug]
-              Patriot::Util::Retry.execute_with_retry{ job_store.register(opts[:update_id], jobs) }
+              job_store.register(opts[:update_id], jobs)
               if opts[:retry_dep]
                 job_store.process_subsequent(jobs.map(&:job_id)) do |job_store, jobs|
                   job_store.set_state(opts[:update_id], jobs.map(&:job_id), Patriot::JobStore::JobState::WAIT)

data/lib/patriot/tool/patriot_commands/worker.rb

@@ -15,19 +15,56 @@ module Patriot
             opts = symbolize_options(options)
             conf        = {:type => 'worker'}
             conf[:path] = opts[:config] if opts.has_key?(:config)
+            opts[:forground] = false unless sub_cmd == 'start'
+            return if sub_cmd == 'start' && !bootable?(conf)
             Process.daemon unless opts[:foreground]
-            config      = load_config(conf)
-            worker_cls  = config.get("worker_class", "Patriot::Worker::MultiNodeWorker")
-            worker      = eval(worker_cls).new(config)
-            case sub_cmd
-            when "start"
-              then worker.start_worker
-            when "stop"
-              then worker.request_shutdown
-            else
-              raise "unknown sub command #{sub_cmd}"
+            config = load_config(conf)
+            logger = Patriot::Util::Logger::Factory.create_logger(self.class.to_s, config)
+            begin
+              worker_cls  = config.get("worker_class", "Patriot::Worker::MultiNodeWorker")
+              worker      = eval(worker_cls).new(config)
+              case sub_cmd
+              when "start"
+                worker.start_worker
+              when "stop"
+                worker.request_shutdown
+              else
+                raise "unknown sub command #{sub_cmd}"
+              end
+            rescue Exception => e
+              logger.error(e)
+              raise e
             end
           end
+
+          no_tasks do
+            # check resources and judge whether this worker is bootable
+            # @return true if this worker is bootable,
+            #         false if the worker has been already running,
+            #         otherwize raise error
+            def bootable?(conf)
+              conf = conf.merge(:ignore_plugin => true)
+              config = load_config(conf)
+              pid = Patriot::Worker.get_pid(config)
+              unless pid.nil?
+                puts "worker running as #{pid}, stop it first"
+                return false
+              end
+              logger = Patriot::Util::Logger::Factory.create_logger(self.class.to_s, config)
+              pid_file = Patriot::Worker.get_pid_file(config)
+              # check log dir permission by writing a log message
+              logger.info("checking whether this worker is bootable")
+              raise "#{pid_file} is not writable" unless writable_or_creatable?(pid_file)
+              return true
+            end
+
+            def writable_or_creatable?(file)
+              file = File.dirname(file) unless File.exist?(file)
+              return File.writable?(file)
+            end
+            private :writable_or_creatable?
+
+          end
         end
       end
     end

data/lib/patriot/worker/base.rb

@@ -4,6 +4,27 @@ require 'patriot/command'
 module Patriot
   module Worker
 
+    def get_pid(config)
+      pid_file = get_pid_file(config)
+      return nil unless File.exists?(pid_file)
+      pid = nil
+      File.open(pid_file,'r'){|f| pid = f.read.strip.to_i }
+      begin
+        Process.getpgid(pid)
+      rescue Errno::ESRCH
+        @logger.warn("process #{pid} not exist but pid file remains") if @logger
+        return nil
+      end
+      return pid
+    end
+    module_function :get_pid
+
+    def get_pid_file(config)
+      worker_name = config.get('worker_name', Patriot::Worker::DEFAULT_WORKER_NAME)
+      return File.join($home, 'run', "patriot-worker_#{worker_name}.pid")
+    end
+    module_function :get_pid_file
+
     # @abstract
     # base class for worker implementations
     class Base
@@ -24,7 +45,6 @@ module Patriot
         @cycle       = config.get('fetch_cycle', Patriot::Worker::DEFAULT_FETCH_CYCLE).to_i
         @fetch_limit = config.get('fetch_limit', Patriot::Worker::DEFAULT_FETCH_LIMIT).to_i
         @worker_name = config.get('worker_name', Patriot::Worker::DEFAULT_WORKER_NAME)
-        @pid_file    = File.join($home, 'run', "patriot-worker_#{@worker_name}.pid")
         @info_server = Patriot::Worker::InfoServer.new(self,@config)
       end
 
@@ -79,16 +99,7 @@ module Patriot
 
       # @return [Integer] pid if the worker is running, otherwise nil
       def get_pid
-        return nil unless File.exists?(@pid_file)
-        pid = nil
-        File.open(@pid_file,'r'){|f| pid = f.read.strip.to_i }
-        begin
-          Process.getpgid(pid)
-        rescue Errno::ESRCH
-          @logger.warn("process #{pid} not exist but pid file remains")
-          return nil
-        end
-        return pid
+        return Patriot::Worker.get_pid(@config)
       end
 
       # send a request graceful shutdown to a running worker
@@ -106,9 +117,9 @@ module Patriot
       # main entry point of worker processing
       def start_worker
         return unless get_pid.nil?
-
         @logger.info "starting worker #{@node}@#{@host}"
-        File.open(@pid_file, 'w') {|f| f.write($$)} # save pid for shutdown
+        pid_file = Patriot::Worker.get_pid_file(@config)
+        File.open(pid_file, 'w') {|f| f.write($$)} # save pid for shutdown
         set_traps
         @info_server.start_server
         @logger.info "initiating worker #{@node}@#{@host}"

data/lib/patriot/worker/info_server.rb

@@ -21,19 +21,21 @@ module Patriot
       include Patriot::Util::Config
       include Patriot::Util::Logger
 
+      attr_accessor :port
+
       # @param worker [Patriot::Worker::Base] worker managed through this server
       # @param config [Patriot::Util::Config::Bae]
       def initialize(worker, config)
         @logger = create_logger(config)
         @worker = worker
         @config = config
+        @port = @config.get(Patriot::Worker::InfoServer::PORT_KEY,
+                            Patriot::Worker::InfoServer::DEFAULT_PORT)
       end
 
       # start the server
       def start_server
-        port = @config.get(Patriot::Worker::InfoServer::PORT_KEY,
-                           Patriot::Worker::InfoServer::DEFAULT_PORT)
-        if port.nil?
+        if @port.nil?
           @logger.info("port is not set. starting info server is skipped")
           return
         end
@@ -42,7 +44,7 @@ module Patriot
             @handler = eval(@config.get(RACK_HANDLER_KEY, DEFAULT_RACK_HANDLER))
             app = Rack::URLMap.new(get_url_map)
             app = Rack::CommonLogger.new(app, build_access_logger)
-            @handler.run app, {:Port => port, :Host => '0.0.0.0'}
+            @handler.run app, {:Port => @port, :Host => '0.0.0.0'}
           rescue => e
             @logger.error e
           end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: patriot-workflow-scheduler
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.1
 platform: ruby
 authors:
 - Teruyoshi Zenmyo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-12 00:00:00.000000000 Z
+date: 2015-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport