patriot-workflow-scheduler 0.7.2 → 0.8.0

data/lib/patriot/util/config.rb

@@ -14,20 +14,31 @@ module Patriot
       # default plugins directory
       DEFAULT_PLUGIN_DIR = "plugins"
       # plugin directory
-      PLUGIN_LIB_DIR = 'lib' 
+      PLUGIN_LIB_DIR = 'lib'
       # plugin initiation script
       PLUGIN_INIT_SCRIPT = 'init.rb'
       # admin user mail address
       ADMIN_USER_KEY = 'admin_user'
 
+      # a comma separated list of hosts where workers are running
+      WORKER_HOST_KEY = 'worker_hosts'
+      # port number used by info server
+      INFO_SERVER_PORT_KEY = 'info_server_port'
+
+      # configuratio key for admin user name
+      USERNAME_KEY = 'info_server.admin.username'
+      # configuratio key for admin password
+      PASSWORD_KEY = 'info_server.admin.password'
+
+
       # load configuration file
       # @param option [Hash]
       # @option option :path [String] path to configuration file
       # @option option :type [String] load type (differe by tool)
       # @option option :ignore_plugin [Boolean] set true not to load plugins
       def load_config(option = {})
-        option = {:path => DEFAULT_CONFIG, 
-                  :type => nil, 
+        option = {:path => DEFAULT_CONFIG,
+                  :type => nil,
                   :ignore_plugin => false }.merge(option)
         conf = nil
         case File.extname(option[:path])

data/lib/patriot/util/config/inifile_config.rb

@@ -5,7 +5,7 @@ module Patriot
       # a configuration implementation definied by the ini-file format
       class IniFileConfig < Patriot::Util::Config::Base
 
-        # comman section name
+        # common section name
         COMMON_SECTION = 'common'
 
         # @param path [String] path to a configuration file

data/lib/patriot/version.rb

@@ -0,0 +1,3 @@
+module Patriot
+  VERSION = "0.8.0"
+end

data/lib/patriot/worker/base.rb

@@ -28,12 +28,13 @@ module Patriot
     # @abstract
     # base class for worker implementations
     class Base
-      
+
       include Patriot::Util::Logger
       include Patriot::Util::Retry
       include Patriot::JobStore::Factory
-        
+
       attr_accessor :host, :status, :cycle, :job_store, :config
+      attr_reader :started_at
 
       # @param config [Patriot::Util::Config::Base]
       def initialize(config)
@@ -73,6 +74,7 @@ module Patriot
           job_ticket.exit_code  = Patriot::Command::ExitCode::SUCCEEDED
         rescue Exception => e
           @logger.warn " job : #{job_ticket.job_id} failed"
+          @logger.warn e
           job_ticket.description = e.to_s
         else
           job_ticket.description = command.description
@@ -122,6 +124,7 @@ module Patriot
         File.open(pid_file, 'w') {|f| f.write($$)} # save pid for shutdown
         set_traps
         @info_server.start_server
+        @started_at = Time.now
         @logger.info "initiating worker #{@node}@#{@host}"
         init_worker
         @status = Patriot::Worker::Status::ACTIVE
@@ -136,7 +139,7 @@ module Patriot
 
       # should be overrided in sub class
       # This method is for implementation-specific configuration
-      def init_worker 
+      def init_worker
         raise NotImplementedError
       end
 

data/lib/patriot/worker/info_server.rb

@@ -1,3 +1,6 @@
+require 'thin'
+require 'rack/rewrite'
+
 module Patriot
   module Worker
     # info server (web management console and for monitoring)
@@ -8,15 +11,11 @@ module Patriot
       # default port number
       DEFAULT_PORT = '36104'
 
-      # urls used by this server
-      URLS_KEY           = 'info_server.urls'
-      # mapping from the url to a servlet for the url
-      URL_MAP_KEY_PREFIX = 'info_server.urlmap'
-
       # configuration key for rack handler used to start this server
       RACK_HANDLER_KEY   = 'info_server.rack.handler'
       # default rack handler
-      DEFAULT_RACK_HANDLER = 'Rack::Handler::WEBrick'
+      # DEFAULT_RACK_HANDLER = 'Rack::Handler::WEBrick'
+      DEFAULT_RACK_HANDLER = 'Rack::Handler::Thin'
 
       include Patriot::Util::Config
       include Patriot::Util::Logger
@@ -39,35 +38,41 @@ module Patriot
           @logger.info("port is not set. starting info server is skipped")
           return
         end
-        @server_thread = Thread.new do 
+        @server_thread = Thread.new do
           begin
             @handler = eval(@config.get(RACK_HANDLER_KEY, DEFAULT_RACK_HANDLER))
             app = Rack::URLMap.new(get_url_map)
             app = Rack::CommonLogger.new(app, build_access_logger)
-            @handler.run app, {:Port => @port, :Host => '0.0.0.0'}
+            app = Rack::Rewrite.new(app){
+              # below 4 rules are for backward compatibility
+              r301 %r{^/jobs/?$}, "/"
+              rewrite %r{^/jobs/([^/]+)$}, "/api/v1/jobs/$1"
+              rewrite "/worker", "/api/v1/workers/this/state"
+              rewrite "/worker/status", "/api/v1/workers/this/state"
+              # for web console
+              rewrite %r{^(?!/api)}, "/"
+            }
+            app = Rack::Static.new(app, :urls => ["/js", "/css"], :root => File.join($home, "public"))
+            # TODO set options based on Handler type
+            @handler.run(app, {:Port => @port, :Host => '0.0.0.0', :signals => false}) do |server|
+              server.threaded = true
+              server.threadpool_size = 5
+              server.timeout = 60
+            end
           rescue => e
             @logger.error e
+            raise e
           end
         end
-        @logger.info "info server has started"
+        @logger.info "info server has started with #{@handler.class}"
         return true
       end
 
       # @return [Hash<String, Sinatra::Base>]
       def get_url_map
-        urls = @config.get(URLS_KEY, nil)
-        if urls.nil?
-          urlmap = {"/jobs"   => Patriot::Worker::Servlet::JobServlet,
-                    "/worker" => Patriot::Worker::Servlet::WorkerStatusServlet}
-        else
-          urlmap = {}
-          urls = [url] unless urls.is_a? Array
-          urls.each do |u|
-            servlet = eval( @config.get("#{URL_MAP_KEY_PREFIX}.#{u}") )
-            u = "/#{u}" unless u.start_with?("/")
-            urlmap[u] = servlet
-          end
-        end
+        urlmap = {"/" => Patriot::Worker::Servlet::IndexServlet,
+                  "/api/v1/jobs" => Patriot::Worker::Servlet::JobAPIServlet,
+                  "/api/v1/workers" => Patriot::Worker::Servlet::WorkerAPIServlet}
         urlmap.values.each{|servlet| servlet.configure(@worker, @config)}
         return urlmap
       end
@@ -82,8 +87,13 @@ module Patriot
       def shutdown_server
         return false if @server.nil?
         unless @server_thread.nil?
-          @handler.shutdown
-          @logger.info "info server shutdowned"
+          begin
+            @handler.shutdown
+            @logger.info "info server shutdowned"
+          rescue => e
+            @logger.error "failed to shutdown infoserver", e
+            raise e
+          end
         end
       end
 

data/lib/patriot/worker/servlet.rb

@@ -10,14 +10,10 @@ module Patriot
       # default offset for searching jobs
       DEFAULT_JOB_OFFSET = 0
 
-      # configuratio key for admin user name
-      USERNAME_KEY = 'info_server.admin.username'
-      # configuratio key for admin password
-      PASSWORD_KEY = 'info_server.admin.password'
-
-      require 'patriot/worker/servlet/job_servlet.rb'
-      require 'patriot/worker/servlet/worker_status_servlet.rb'
-
+      require 'patriot/worker/servlet/index_servlet.rb'
+      require 'patriot/worker/servlet/api_servlet_base.rb'
+      require 'patriot/worker/servlet/job_api_servlet.rb'
+      require 'patriot/worker/servlet/worker_api_servlet.rb'
     end
   end
 end

data/lib/patriot/worker/servlet/api_servlet_base.rb

@@ -0,0 +1,40 @@
+require 'sinatra/base'
+
+module Patriot
+  module Worker
+    module Servlet
+      class APIServletBase < Sinatra::Base
+        set :show_exceptions, :after_handler
+
+        ### Helper Methods
+        helpers do
+          # require authorization for updating
+          def protected!
+            return if authorized?
+            headers['WWW-Authenticate'] = 'Basic Realm="Admin Only"'
+            halt 401, "Not Authorized"
+          end
+
+          # authorize user (basic authentication)
+          def authorized?
+            @auth ||= Rack::Auth::Basic::Request.new(request.env)
+            return @auth.provided? && @auth.basic? && @auth.credentials && @auth.credentials == [@@username, @@password]
+          end
+        end
+
+        # @param worker [Patriot::Wokrer::Base]
+        # @param config [Patriot::Util::Config::Base]
+        def self.configure(worker, config)
+          @@worker = worker
+          @@config = config
+          @@username  = config.get(Patriot::Util::Config::USERNAME_KEY, "")
+          @@password  = config.get(Patriot::Util::Config::PASSWORD_KEY, "")
+        end
+
+        before do
+          content_type 'application/json;charset=utf8'
+        end
+      end
+    end
+  end
+end

data/lib/patriot/worker/servlet/index_servlet.rb

@@ -0,0 +1,32 @@
+require 'sinatra/base'
+require 'sinatra/contrib'
+
+module Patriot
+  module Worker
+    module Servlet
+
+      # allow to monitor worker status
+      class IndexServlet < Sinatra::Base
+        register Sinatra::Contrib
+
+        set :public_folder, File.join($home, "public")
+        set :views, File.join($home, "public", "views")
+
+        # @param worker [Patriot::Worker::Base]
+        # @param config [Patriot::Util::Config::Base]
+        def self.configure(worker, config)
+          @@worker = worker
+        end
+
+        before do
+          @worker = @@worker
+        end
+
+        get '/' do
+          erb :index
+        end
+
+      end
+    end
+  end
+end

data/lib/patriot/worker/servlet/job_api_servlet.rb

@@ -0,0 +1,156 @@
+require 'sinatra/base'
+require 'sinatra/contrib'
+
+module Patriot
+  module Worker
+    module Servlet
+      # excepton thrown in case job not found
+      class JobNotFoundException < Exception; end
+      # provide job management functionalities
+      class JobAPIServlet < Patriot::Worker::Servlet::APIServletBase
+        register Sinatra::Contrib
+        include Patriot::Util::DateUtil
+        include Patriot::Command::Parser
+
+        set :show_exceptions, :after_handler
+
+        get '/stats' do
+          return JSON.generate(@@worker.job_store.get_job_size(:ignore_states => [Patriot::JobStore::JobState::SUCCEEDED]))
+        end
+
+        get '/' do
+          state  = (params['state']  || Patriot::JobStore::JobState::FAILED).to_i
+          limit  = (params['limit']  || DEFAULT_JOB_LIMIT).to_i
+          offset = (params['offset'] || DEFAULT_JOB_OFFSET).to_i
+
+          query = {:limit  => limit, :offset => offset}
+          query[:filter_exp] = params['filter_exp'] unless params['filter_exp'].blank?
+          job_ids = @@worker.job_store.find_jobs_by_state(state, query) || []
+          return JSON.generate(job_ids.map{|job_id| {:job_id => job_id, :state => state}})
+        end
+
+        get '/:job_id' do
+          job_id = params[:job_id]
+          job = @@worker.job_store.get(job_id, {:include_dependency => true})
+          halt(404, json({ERROR: "Job #{job_id} not found"})) if job.nil?
+          return JSON.generate(job.attributes.merge({'job_id' => job_id, "update_id" => job.update_id}))
+        end
+
+        get '/:job_id/histories' do
+          job_id = params[:job_id]
+          history_size = params['size'] || 3
+          histories = @@worker.job_store.get_execution_history(job_id, {:limit => history_size})
+          return JSON.generate(histories)
+        end
+
+        # get dependency graph of the specified job
+        #
+        # @param [String] job_id
+        # @param [String] p_level a max number of producer dependency level to get
+        # @param [String] c_level a max number of consumer dependency level to get
+        get '/:job_id/graph' do
+          job_id = params[:job_id]
+          producer_depth = (params['p_depth'] || 2).to_i
+          consumer_depth = (params['c_depth'] || 2).to_i
+          graph = @@worker.job_store.get_graph(job_id, {:producer_depth => producer_depth, :consumer_depth => consumer_depth})
+          return JSON.generate(graph)
+        end
+
+        post '/' do
+          protected!
+          body = JSON.parse(request.body.read)
+          halt(400, json({ERROR: "COMMAND_CLASS is not provided"})) if body["COMMAND_CLASS"].blank?
+          halt(400, json({ERROR: "Patriot::Command::CommandGroup is not acceptable"})) if body["COMMAND_CLASS"] == "Patriot::Command::CommandGroup"
+          command_class = body.delete("COMMAND_CLASS").gsub(/\./, '::').constantize
+
+          job = _build_command(command_class, body)[0]
+          job[Patriot::Command::STATE_ATTR] ||= body["state"]
+          @@worker.job_store.register(Time.now.to_i, [job])
+          return JSON.generate({:job_id => job.job_id})
+        end
+
+        put '/' do
+          protected!
+          body = JSON.parse(request.body.read)
+          job_ids = body["job_ids"]
+          state = body['state']
+          _set_state_of_jobs(job_ids, state)
+          return JSON.generate(job_ids.map{|job_id| {"job_id" => job_id, "state" => state} })
+        end
+
+        put '/:job_id' do
+          protected!
+          job_id = params['job_id']
+          job = @@worker.job_store.get(job_id)
+          halt(404, json({ERROR: "Job #{job_id} not found"})) if job.nil?
+
+          body = JSON.parse(request.body.read)
+          state = body['state']
+          options = body['option'] || {}
+          job_ids = _set_state_of_jobs(job_id, state, options)
+          return JSON.generate(job_ids.map{|jid| {"job_id" => jid, "state" => state}})
+        end
+
+        delete '/' do
+          protected!
+          body = request.body.read
+          job_ids = []
+          if body != ''
+            body = JSON.parse(body)
+            job_ids = body["job_ids"]
+          else
+            job_ids = JSON.parse(params["job_ids"])
+          end
+          job_ids.each{ |job_id| @@worker.job_store.delete_job(job_id) }
+          return JSON.generate(job_ids.map{|job_id| {"job_id" => job_id} })
+        end
+
+        delete '/:job_id' do
+          protected!
+          job_id = params['job_id']
+          job = @@worker.job_store.get(job_id)
+          halt(404, json({ERROR: "Job #{job_id} not found"})) if job.nil?
+
+          @@worker.job_store.delete_job(job_id)
+          return JSON.generate({"job_id" => job_id})
+        end
+
+        error JobNotFoundException do
+          "Job #{env['sinatra.error'].message} is not found"
+        end
+
+        # @private
+        def _set_state_of_jobs(job_ids, state, opts = {})
+          job_ids = [job_ids] unless job_ids.is_a? Array
+          opts = {'with_subsequent' => false}.merge(opts)
+          opts = {:include_subsequent => false}.merge(opts)
+          update_id = Time.now.to_i
+          @@worker.job_store.set_state(update_id, job_ids, state)
+          if opts['with_subsequent']
+            @@worker.job_store.process_subsequent(job_ids) do |job_store, jobs|
+              next if jobs.empty?
+              subsequent_ids = jobs.map(&:job_id)
+              @@worker.job_store.set_state(update_id, subsequent_ids, state)
+              job_ids |= subsequent_ids
+            end
+          end
+          return job_ids.uniq
+        end
+        private :_set_state_of_jobs
+
+        # @private
+        def _build_command(clazz, params)
+          _params = params.dup
+          _params["target_datetime"] = Date.today
+          cmd = clazz.new(@@config)
+          cmd.produce(_params.delete("products")) unless _params["products"].nil?
+          cmd.require(_params.delete("requisites")) unless _params["requisites"].nil?
+          cmds = cmd.build(_params)
+          return cmds.map{|c| c.to_job}
+        end
+        private :_build_command
+
+      end
+    end
+  end
+end

data/lib/patriot/worker/servlet/worker_api_servlet.rb

@@ -0,0 +1,67 @@
+require 'sinatra/base'
+require 'sinatra/contrib'
+
+module Patriot
+  module Worker
+    module Servlet
+      # excepton thrown when a woker is not accessible
+      class WorkerInaccessibleException < Exception; end
+      # provide worker management functionalities
+      class WorkerAPIServlet < Patriot::Worker::Servlet::APIServletBase
+        register Sinatra::Contrib
+
+        HOST_KEY = "host"
+        VERSION_KEY = "version"
+        CLASS_KEY = "class"
+        STARTED_AT_KEY = "started_at"
+        CONFIG_KEY = "config"
+
+        STATE_KEY = "state"
+
+        LOCALHOST_EXPS = ["localhost", "127.0.0.1"]
+
+        set :show_exceptions, :after_handler
+
+        get '/' do
+          worker_hosts = @@config.get(Patriot::Util::Config::WORKER_HOST_KEY)
+          worker_hosts = [worker_hosts] unless worker_hosts.is_a?(Array)
+          worker_hosts = @@config.get(Patriot::Util::Config::WORKER_HOST_KEY).map do |h|
+            h = h.split(":")
+            port = h.size == 2 ? h[1] : Patriot::Worker::InfoServer::DEFAULT_PORT
+            {'host' => h[0], 'port' => port}
+          end
+          return JSON.generate(worker_hosts)
+        end
+
+        get '/this' do
+          return JSON.generate(
+            STATE_KEY => @@worker.status,
+            HOST_KEY => @@worker.host,
+            VERSION_KEY => Patriot::VERSION,
+            CLASS_KEY => @@worker.class.to_s,
+            STARTED_AT_KEY => @@worker.started_at,
+            CONFIG_KEY => {} # TODO
+          )
+        end
+
+        get '/this/state' do
+          return JSON.generate(
+            STATE_KEY => @@worker.status
+          )
+        end
+
+        put '/this/state' do
+          protected!
+          new_status = params['status']
+          if [Patriot::Worker::Status::ACTIVE, Patriot::Worker::Status::SLEEP ].include?(@@worker.status)
+            @@worker.status = new_status
+          else
+            # state cannot be changed in shutdown process
+            halt 403
+          end
+        end
+
+      end
+    end
+  end
+end