patriot-workflow-scheduler 0.6.1

data/lib/patriot/worker/info_server.rb

@@ -0,0 +1,90 @@
+module Patriot
+  module Worker
+    # info server (web management console and for monitoring)
+    class InfoServer
+
+      # configuratio key for port used by this server
+      PORT_KEY = 'info_server.port'
+      # default port number
+      DEFAULT_PORT = '36104'
+
+      # urls used by this server
+      URLS_KEY           = 'info_server.urls'
+      # mapping from the url to a servlet for the url
+      URL_MAP_KEY_PREFIX = 'info_server.urlmap'
+
+      # configuration key for rack handler used to start this server
+      RACK_HANDLER_KEY   = 'info_server.rack.handler'
+      # default rack handler
+      DEFAULT_RACK_HANDLER = 'Rack::Handler::WEBrick'
+
+      include Patriot::Util::Config
+      include Patriot::Util::Logger
+
+      # @param worker [Patriot::Worker::Base] worker managed through this server
+      # @param config [Patriot::Util::Config::Bae]
+      def initialize(worker, config)
+        @logger = create_logger(config)
+        @worker = worker
+        @config = config
+      end
+
+      # start the server
+      def start_server
+        port = @config.get(Patriot::Worker::InfoServer::PORT_KEY,
+                           Patriot::Worker::InfoServer::DEFAULT_PORT)
+        if port.nil?
+          @logger.info("port is not set. starting info server is skipped")
+          return
+        end
+        @server_thread = Thread.new do 
+          begin
+            @handler = eval(@config.get(RACK_HANDLER_KEY, DEFAULT_RACK_HANDLER))
+            app = Rack::URLMap.new(get_url_map)
+            app = Rack::CommonLogger.new(app, build_access_logger)
+            @handler.run app, {:Port => port, :Host => '0.0.0.0'}
+          rescue => e
+            @logger.error e
+          end
+        end
+        @logger.info "info server has started"
+        return true
+      end
+
+      # @return [Hash<String, Sinatra::Base>]
+      def get_url_map
+        urls = @config.get(URLS_KEY, nil)
+        if urls.nil?
+          urlmap = {"/jobs"   => Patriot::Worker::Servlet::JobServlet,
+                    "/worker" => Patriot::Worker::Servlet::WorkerStatusServlet}
+        else
+          urlmap = {}
+          urls = [url] unless urls.is_a? Array
+          urls.each do |u|
+            servlet = eval( @config.get("#{URL_MAP_KEY_PREFIX}.#{u}") )
+            u = "/#{u}" unless u.start_with?("/")
+            urlmap[u] = servlet
+          end
+        end
+        urlmap.values.each{|servlet| servlet.configure(@worker, @config)}
+        return urlmap
+      end
+
+      def build_access_logger
+        config = load_config(:path => @config.path, :type => 'web', :ignore_plugin => true)
+        return Patriot::Util::Logger::Factory.create_logger(@hanlder.class.to_s, config)
+      end
+      private :build_access_logger
+
+      # instruct to shutdown server
+      def shutdown_server
+        return false if @server.nil?
+        unless @server_thread.nil?
+          @handler.shutdown
+          @logger.info "info server shutdowned"
+        end
+      end
+
+    end
+  end
+end

data/lib/patriot/worker/job_store_server.rb

@@ -0,0 +1,32 @@
+require 'thread'
+require 'monitor'
+
+module Patriot
+  module Worker
+    # a worker as info server
+    class JobStoreServer < Base
+
+      # @see Patriot::Worker::Base#build_infoserver
+      def build_infoserver
+        return Patriot::Worker::InfoServer.new(self,@config)
+      end
+
+      # @see Patriot::Worker::Base#init_worker
+      def init_worker
+      end
+
+      # @see Patriot::Worker::Base#run_worker
+      def run_worker
+        while(@status != Patriot::Worker::Status::SHUTDOWN)
+          sleep @cycle
+        end
+      end
+
+      # @see Patriot::Worker::Base#stop_worker
+      def stop_worker
+        @logger.info "terminated"
+      end
+
+    end
+  end
+end

data/lib/patriot/worker/multi_node_worker.rb

@@ -0,0 +1,157 @@
+require 'thread'
+require 'monitor'
+
+module Patriot
+  module Worker
+
+    # a worker implementation which can host multiple node on one process
+    class MultiNodeWorker < Base
+
+      include MonitorMixin
+
+      ### type of node
+      # handle any jobs without label or same label
+      ANY = 'any'
+      # handle only jobs with same label
+      OWN = 'own' 
+      # supporte node types
+      SUPPORTED_TYPES = [ANY,OWN]
+
+      ### type of job
+      # executed by any node
+      ANY_EXCLUDE_TYPE_OWN = 0
+      # executed by only same labeled node
+      ONLY_SPECIFIED_NODE  = 1
+      # unknown
+      UNEXPECTED           = 2
+
+      # @see Patriot::Worker::Base#init_worker
+      def init_worker 
+        nodes = @config.get('nodes')
+        raise "nodes are not configured" if nodes.nil?
+        nodes = [nodes] unless nodes.is_a?(Array)
+        @nodes = {}
+        nodes.each do |n|
+          node_config = get_node_config(@config, n)
+          raise "node #{n} is not configured" if node_config.nil?
+          @nodes[n] = {:queue => Queue.new }.merge(node_config)
+        end
+      end
+
+      def get_node_config(config, n)
+        type    = config.get("node.#{n}.type")
+        threads = config.get("node.#{n}.threads")
+        raise "the number of threads for node #{n} is not set" if threads.nil?
+        raise "unsupported node type: #{n} with #{type} " unless SUPPORTED_TYPES.include?(type)
+        return {:type =>  type, :threads => threads.to_i}
+      end
+      private :get_node_config
+
+      # @see Patriot::Worker::Base#run_worker
+      def run_worker
+        @threads = []
+        # create node threads for job execution
+        @nodes.each do |node,conf|
+          1.upto(conf[:threads]) do |i| 
+            @threads << create_thread(node, i, conf[:queue])
+          end
+        end
+        # start main thread for updating queues
+        Thread.current[:name] = 'main'
+        while(alive?)
+          if @status == Patriot::Worker::Status::ACTIVE
+            begin
+              job_tickets = @job_store.get_job_tickets(@host, @nodes.keys, {:fetch_limit => @fetch_limit})
+              @logger.info "get #{job_tickets.size} jobs"
+              update_queue(job_tickets) unless job_tickets.nil?
+            rescue => e
+              @logger.error e
+            end
+          end
+          sleep @cycle
+        end
+      end
+
+      def create_thread(node, thread_number, queue)
+        Thread.start(queue) do |q|
+          Thread.current[:name] = "worker_#{node}_#{thread_number}"
+          Thread.current[:node] = node
+          begin
+            while(@status != Patriot::Worker::Status::SHUTDOWN)
+              job_ticket = q.pop
+              if job_ticket == :TERM
+                @logger.info "terminating"
+              else
+                @logger.debug "fetch job #{job_ticket.job_id}"
+                Thread.current[Patriot::Worker::JOB_ID_IN_EXECUTION] = job_ticket.job_id
+                execute_job(job_ticket)
+                Thread.current[Patriot::Worker::JOB_ID_IN_EXECUTION] = nil
+              end
+            end
+          rescue Exception => e
+            @logger.error e
+            raise e, "exception in worker thread" , $@
+          ensure
+            @logger.info "terminated"
+            interrupted_job = Thread.current[Patriot::Worker::JOB_ID_IN_EXECUTION]
+            @logger.warn "job: #{interrupted_job} could be interrupted." unless interrupted_job.nil?
+          end
+        end
+      end
+      private :create_thread
+
+      # check thread status
+      def alive?
+        @threads.each do |t|
+          if t.status.nil? # thread stopped by some error
+            @logger.error "#{t[:name]} : nil " 
+            @status = Patriot::Worker::Status::SHUTDOWN
+          elsif t.status == false # stopped by signal
+            @logger.debug "#{t[:name]} : false "
+            @status = Patriot::Worker::Status::SHUTDOWN
+          else
+            @logger.debug "#{t[:name]} : #{t.status}"
+          end
+        end
+        return @status != Patriot::Worker::Status::SHUTDOWN
+      end
+      private :alive?
+
+      # update job queues
+      def update_queue(job_tickets)
+        @nodes.each{|node,conf| conf[:queue].clear }
+        job_tickets.each do |job_ticket|
+          case type_of_job(job_ticket)
+          when ANY_EXCLUDE_TYPE_OWN 
+            @nodes.each{|node,conf| conf[:queue].push(job_ticket) unless conf[:type]==OWN }
+          when ONLY_SPECIFIED_NODE
+            @nodes[job_ticket.node][:queue].push(job_ticket)
+          else
+            @logger.warn "receive unexpected job #{job_ticket.to_s}"
+          end
+        end
+      end
+      private :update_queue
+
+      def type_of_job(job_ticket)
+        return ANY_EXCLUDE_TYPE_OWN if job_ticket.node.nil?
+        return ONLY_SPECIFIED_NODE if @nodes.has_key?(job_ticket.node)
+        return UNEXPECTED;
+      end
+      private :type_of_job
+
+      # @see Patriot::Worker::Base#run_worker
+      def stop_worker
+        @status = Patriot::Worker::Status::SHUTDOWN
+        @logger.info "terminating worker"
+        @nodes.each do |node,conf|
+          conf[:queue].clear
+          1.upto(conf[:threads]) {|i| conf[:queue].push(:TERM) }
+        end
+        @threads.each{|t| t.join}
+        @logger.info "terminated"
+      end
+
+    end
+  end
+end

data/lib/patriot/worker/servlet.rb

@@ -0,0 +1,23 @@
+require 'sinatra/base'
+require 'sinatra/contrib'
+
+module Patriot
+  module Worker
+    # namespacce infoserve servlet
+    module Servlet
+      # default limit in searching jobs
+      DEFAULT_JOB_LIMIT = 50
+      # default offset for searching jobs
+      DEFAULT_JOB_OFFSET = 0
+
+      # configuratio key for admin user name
+      USERNAME_KEY = 'info_server.admin.username'
+      # configuratio key for admin password
+      PASSWORD_KEY = 'info_server.admin.password'
+
+      require 'patriot/worker/servlet/job_servlet.rb'
+      require 'patriot/worker/servlet/worker_status_servlet.rb'
+
+    end
+  end
+end

data/lib/patriot/worker/servlet/job_servlet.rb

@@ -0,0 +1,128 @@
+require 'sinatra/base'
+require 'sinatra/contrib'
+
+module Patriot
+  module Worker
+    module Servlet
+      # excepton thrown in case job not found
+      class JobNotFoundException < Exception; end
+      # provide job management functionalities
+      class JobServlet < Sinatra::Base
+        register Sinatra::Contrib
+
+        set :public_folder, File.join($home, "public")
+        set :views, File.join($home, "public", "templates")
+        set :show_exceptions, :after_handler
+
+        # @param worker [Patriot::Wokrer::Base]
+        # @param config [Patriot::Util::Config::Base]
+        def self.configure(worker, config)
+          @@job_store = worker.job_store
+          @@username  = config.get(USERNAME_KEY, "")
+          @@password  = config.get(PASSWORD_KEY, "")
+        end
+
+        ### Helper Methods
+        helpers do
+          # return link to each job information
+          def to_job_link(job_id)
+            return "<a href='/jobs/#{ERB::Util.url_encode(job_id)}'>#{job_id}</a>"
+          end
+          # require authorization for updating
+          def protected!
+            return if authorized?
+            headers['WWW-Authenticate'] = 'Basic Realm="Admin Only"'
+            halt 401, "Not Authorized"
+          end
+          # authorize user (basic authentication)
+          def authorized?
+            @auth ||= Rack::Auth::Basic::Request.new(request.env)
+            return @auth.provided? && @auth.basic? && @auth.credentials && @auth.credentials == [@@username, @@password]
+          end
+        end
+
+        get '/' do
+          state = params['state'] || Patriot::JobStore::JobState::FAILED
+          update_job_size(state)
+          query = {:limit  => params['limit']  || DEFAULT_JOB_LIMIT,
+                   :offset => params['offset'] || DEFAULT_JOB_OFFSET}
+          query[:filter_exp] = params['filter_exp'] if params.has_key?('filter_exp') && params['filter_exp'] != ""
+          jobs = @@job_store.find_jobs_by_state(state.to_i, query)
+          jobs ||= []
+          respond_with :jobs, {:jobs => jobs, :state => state, :filter_exp => query[:filter_exp]}.merge(query)
+        end
+
+        post '/' do
+          protected!
+          job_ids   = params['job_ids']
+          if params['_method'] == 'put'
+            state = params['state']
+            set_state_of_jobs(job_ids, state)
+          elsif params['_method'] == 'delete'
+            delete_jobs(job_ids)
+          else
+            raise NotImplementedError
+          end
+        end
+
+        get '/:job_id' do
+          job_id = params[:job_id]
+          history_size = params['history_size'] || 1
+          job = @@job_store.get(job_id, {:include_dependency => true})
+          raise JobNotFoundException, job_id if job.nil?
+          update_job_size(job[Patriot::Command::STATE_ATTR])
+          histories = @@job_store.get_execution_history(job_id, {:limit => history_size})
+          # respond_with :job, {:job => job, :histories => histories} 
+          respond_to do |f|
+            f.on('text/html'){ erb :job, :locals => {:job => job, :histories => histories} }
+            # for monitoring
+            f.on('*/*'){ JSON.generate(job.attributes.merge({'job_id' => job_id})) }
+          end
+        end
+
+        post '/:job_id' do
+          protected!
+          if params['_method'] == 'put'
+            set_state_of_jobs([params['job_id']], params['state'], {:include_subsequent => params['include_subsequent'] == 'true'})
+            respond_with :state_updated, {:jobs => job_ids, :state => state}
+          else
+            raise NotImplementedError
+          end
+        end
+
+        error JobNotFoundException do
+          "Job #{env['sinatra.error'].message} is not found"
+        end
+
+        # update jobs size
+        def update_job_size(state)
+          @size = @@job_store.get_job_size(:ignore_states => [Patriot::JobStore::JobState::SUCCEEDED])
+        end
+
+        # @private
+        # update state of jobs
+        def set_state_of_jobs(job_ids, state, opts = {})
+          opts = {:include_subsequent => false}.merge(opts)
+          update_id = Time.now.to_i
+          @@job_store.set_state(update_id, job_ids, state)
+          if opts[:include_subsequent]
+            @@job_store.process_subsequent(job_ids) do |job_store, jobs|
+              @@job_store.set_state(update_id, jobs.map(&:job_id), state)
+            end
+          end
+          update_job_size(state)
+          respond_with :state_updated, {:jobs => job_ids, :state => state}
+        end
+        private :set_state_of_jobs
+
+        def delete_jobs(job_ids)
+          job_ids.each{|jid| @@job_store.delete_job(jid) }
+          update_job_size(Patriot::JobStore::JobState::DISCARDED)
+          respond_with :jobs_deleted, {:jobs => job_ids}
+        end
+        private :delete_jobs
+
+      end
+    end
+  end
+end

data/lib/patriot/worker/servlet/worker_status_servlet.rb

@@ -0,0 +1,44 @@
+require 'sinatra/base'
+require 'sinatra/contrib'
+
+module Patriot
+  module Worker
+    module Servlet
+
+      # allow to monitor worker status
+      class WorkerStatusServlet < Sinatra::Base
+        register Sinatra::Contrib
+
+        set :public_folder, File.join($home, "public")
+        set :views, File.join($home, "public", "templates")
+
+        # @param worker [Patriot::Worker::Base]
+        # @param config [Patriot::Util::Config::Base]
+        def self.configure(worker, config)
+          @@worker = worker
+        end
+
+        before do
+          @worker = @@worker
+        end
+
+        get '/' do
+          respond_with :worker, {:worker => @worker} do |f|
+            # for monitoring
+            f.on('*/*') { JSON.generate(@worker.host => @worker.status)}
+          end
+        end
+
+        put '/' do
+          new_status = params['status']
+          if [Patriot::Worker::Status::ACTIVE, Patriot::Worker::Status::SLEEP ]
+            @worker.status = new_status
+          else
+            return 400
+          end
+        end
+
+      end
+    end
+  end
+end