patriot-workflow-scheduler 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. checksums.yaml +15 -0
  2. data/bin/patriot +8 -0
  3. data/bin/patriot-init +35 -0
  4. data/lib/patriot.rb +11 -0
  5. data/lib/patriot/command.rb +71 -0
  6. data/lib/patriot/command/base.rb +199 -0
  7. data/lib/patriot/command/command_group.rb +43 -0
  8. data/lib/patriot/command/command_macro.rb +141 -0
  9. data/lib/patriot/command/composite.rb +49 -0
  10. data/lib/patriot/command/parser.rb +78 -0
  11. data/lib/patriot/command/sh_command.rb +42 -0
  12. data/lib/patriot/controller.rb +2 -0
  13. data/lib/patriot/controller/package_controller.rb +81 -0
  14. data/lib/patriot/controller/worker_admin_controller.rb +159 -0
  15. data/lib/patriot/job_store.rb +66 -0
  16. data/lib/patriot/job_store/base.rb +159 -0
  17. data/lib/patriot/job_store/factory.rb +19 -0
  18. data/lib/patriot/job_store/in_memory_store.rb +252 -0
  19. data/lib/patriot/job_store/job.rb +118 -0
  20. data/lib/patriot/job_store/job_ticket.rb +30 -0
  21. data/lib/patriot/job_store/rdb_job_store.rb +353 -0
  22. data/lib/patriot/tool.rb +2 -0
  23. data/lib/patriot/tool/batch_parser.rb +102 -0
  24. data/lib/patriot/tool/patriot_command.rb +48 -0
  25. data/lib/patriot/tool/patriot_commands/execute.rb +92 -0
  26. data/lib/patriot/tool/patriot_commands/job.rb +62 -0
  27. data/lib/patriot/tool/patriot_commands/plugin.rb +41 -0
  28. data/lib/patriot/tool/patriot_commands/register.rb +77 -0
  29. data/lib/patriot/tool/patriot_commands/upgrade.rb +24 -0
  30. data/lib/patriot/tool/patriot_commands/validate.rb +84 -0
  31. data/lib/patriot/tool/patriot_commands/worker.rb +35 -0
  32. data/lib/patriot/tool/patriot_commands/worker_admin.rb +60 -0
  33. data/lib/patriot/util.rb +14 -0
  34. data/lib/patriot/util/config.rb +58 -0
  35. data/lib/patriot/util/config/base.rb +22 -0
  36. data/lib/patriot/util/config/inifile_config.rb +63 -0
  37. data/lib/patriot/util/cron_format_parser.rb +104 -0
  38. data/lib/patriot/util/date_util.rb +200 -0
  39. data/lib/patriot/util/db_client.rb +65 -0
  40. data/lib/patriot/util/db_client/base.rb +142 -0
  41. data/lib/patriot/util/db_client/hash_record.rb +53 -0
  42. data/lib/patriot/util/db_client/record.rb +25 -0
  43. data/lib/patriot/util/logger.rb +24 -0
  44. data/lib/patriot/util/logger/facade.rb +33 -0
  45. data/lib/patriot/util/logger/factory.rb +59 -0
  46. data/lib/patriot/util/logger/log4r_factory.rb +111 -0
  47. data/lib/patriot/util/logger/webrick_log_factory.rb +47 -0
  48. data/lib/patriot/util/param.rb +73 -0
  49. data/lib/patriot/util/retry.rb +30 -0
  50. data/lib/patriot/util/script.rb +52 -0
  51. data/lib/patriot/util/system.rb +120 -0
  52. data/lib/patriot/worker.rb +35 -0
  53. data/lib/patriot/worker/base.rb +153 -0
  54. data/lib/patriot/worker/info_server.rb +90 -0
  55. data/lib/patriot/worker/job_store_server.rb +32 -0
  56. data/lib/patriot/worker/multi_node_worker.rb +157 -0
  57. data/lib/patriot/worker/servlet.rb +23 -0
  58. data/lib/patriot/worker/servlet/job_servlet.rb +128 -0
  59. data/lib/patriot/worker/servlet/worker_status_servlet.rb +44 -0
  60. data/skel/batch/sample/daily/test.pbc +4 -0
  61. data/skel/config/patriot.ini +21 -0
  62. data/skel/public/css/bootstrap.css +2495 -0
  63. data/skel/public/css/original.css +54 -0
  64. data/skel/public/js/bootstrap-alerts.js +124 -0
  65. data/skel/public/js/bootstrap-buttons.js +64 -0
  66. data/skel/public/js/bootstrap-dropdown.js +55 -0
  67. data/skel/public/js/bootstrap-modal.js +260 -0
  68. data/skel/public/js/bootstrap-popover.js +90 -0
  69. data/skel/public/js/bootstrap-scrollspy.js +107 -0
  70. data/skel/public/js/bootstrap-tabs.js +80 -0
  71. data/skel/public/js/bootstrap-twipsy.js +321 -0
  72. data/skel/public/js/jquery-1.6.4.min.js +4 -0
  73. data/skel/public/templates/_jobs.erb +97 -0
  74. data/skel/public/templates/job.erb +119 -0
  75. data/skel/public/templates/jobs.erb +21 -0
  76. data/skel/public/templates/jobs_deleted.erb +6 -0
  77. data/skel/public/templates/layout.erb +103 -0
  78. data/skel/public/templates/state_updated.erb +6 -0
  79. metadata +235 -0
@@ -0,0 +1,65 @@
1
+ require 'patriot/util/db_client/base'
2
+ require 'patriot/util/db_client/record'
3
+ require 'patriot/util/db_client/hash_record'
4
+
5
+ module Patriot
6
+ module Util
7
+ # name space for connectors of RDBs
8
+ module DBClient
9
+
10
+ # a set of configuration keys for the connectors
11
+ DB_CONFIG_KEYS = ["adapter",
12
+ "database",
13
+ "host",
14
+ "port",
15
+ "username",
16
+ "password",
17
+ "encoding",
18
+ "pool"]
19
+
20
+ # read DB configuration from a given config
21
+ # @param prefix [String] a prefix for DB configuration key
22
+ # @param config [Patriot::Util::Config::Base] configuration
23
+ def read_dbconfig(prefix, config)
24
+ db_config = {}
25
+ Patriot::Util::DBClient::DB_CONFIG_KEYS.each do |k|
26
+ v = config.get([prefix,k].join("."))
27
+ db_config[k.to_sym] = v
28
+ end
29
+ return db_config
30
+ end
31
+
32
+ # get connection to db.
33
+ # if block is given, call the given block and close the connection
34
+ # @param dbconf [Hash]
35
+ # configuration for database
36
+ # type of database should be set with key :adapter
37
+ # @yield
38
+ # block executed with a connection to the database.
39
+ # the connection is passed to the block as a argument
40
+ # @yieldparam [Patriot::Util::DBClient::Base] a connection to db
41
+ def connect(dbconf, &blk)
42
+ client = get_db_client(dbconf)
43
+ return client unless block_given?
44
+ begin
45
+ return yield client
46
+ rescue Exception => e
47
+ raise e
48
+ ensure
49
+ client.close
50
+ end
51
+ end
52
+
53
+ # @private
54
+ # @return [Patriot::Util::DBClient::Base]
55
+ def get_db_client(dbconf = {})
56
+ raise "no adapter is specified" unless dbconf.has_key?(:adapter)
57
+ adapter = dbconf[:adapter].to_sym
58
+ raise "no builder method is registered for #{adapter}" unless self.respond_to?(adapter)
59
+ return self.send(adapter, dbconf)
60
+ end
61
+ private :get_db_client
62
+
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,142 @@
1
+ module Patriot module Util
2
+ module DBClient
3
+
4
+ # @abstract base class for DB connection
5
+ class Base
6
+
7
+ # close this connection
8
+ def close()
9
+ raise NotImplementedError
10
+ end
11
+
12
+ # execute a given statment with this connection.
13
+ # @param [String] stmt SQL statement to be executed
14
+ # @param [Symbol] type type of execution (return type)
15
+ # @return [Array] a list of records if the type is :select
16
+ # @return [Integer] a id of last inserted record if the type is :insert
17
+ # @return [Integer] the nubmer of affected rows if the type is :update
18
+ def execute_statement(stmt, type = :select)
19
+ method = "do_#{type}".to_sym
20
+ begin
21
+ return self.send(method, stmt)
22
+ rescue => e
23
+ raise "Failed to execute #{stmt} : Caused by (#{e})"
24
+ end
25
+ end
26
+
27
+ # insert value into a given table
28
+ # @param [String] tbl the name of table where a row will be inserted
29
+ # @param [Hash] value a Hash represents the inserted value
30
+ # @param [Hash] opts
31
+ # @option [Boolean] :ignore set true if set conflict strategy IGNORE (default false)
32
+ # @return [Integer] a id of the last inserted record
33
+ # @example the below invocation issues "INSERT INTO table (col) VALUES ('val')"
34
+ # insert('table', {:col => 'val'})
35
+ def insert(tbl, value, opts = {})
36
+ query = build_insert_query(tbl, value, opts)
37
+ return execute_statement(query, :insert)
38
+ end
39
+
40
+ # create a insert statement for given arguments.
41
+ # Subclasses can override this method if necessary
42
+ # @param [String] tbl the name of table where a row will be inserted
43
+ # @param [Hash] value a Hash represents the inserted value
44
+ # @param [Hash] opts
45
+ # @option [Boolean] :ignore set true if set conflict strategy IGNORE (default false)
46
+ # @return [String] insert statement expression
47
+ def build_insert_query(tbl, value, opts = {})
48
+ opts = {:ignore => false}.merge(opts)
49
+ cols, vals = [], []
50
+ value.each do |c,v|
51
+ cols << c
52
+ vals << quote(v)
53
+ end
54
+ return "INSERT INTO #{tbl} (#{cols.join(',')}) VALUES (#{vals.join(',')})"
55
+ end
56
+
57
+ # The method which issues an insert statement
58
+ # @param [String] query an insert statement
59
+ # @return [Integer] a id of last inserted record if the type is :insert
60
+ def do_insert(query)
61
+ raise NotImplementedError
62
+ end
63
+
64
+ # select records from a table
65
+ # @param [String] tbl the name of table where the records are searched
66
+ # @param [Hash] cond a set of conditons for each attributes
67
+ # @param [Hash] opts
68
+ # @option [Integer] :limit LIMIT nubmer
69
+ # @option [Integer] :offset OFFSET nubmer
70
+ # @option [ARRAY] :items a list of output items
71
+ # @return [Array] a list of records
72
+ # @example the below invocation issues "SELECT * FROM table WHERE col = 'val'"
73
+ # select('table', {:col => 'val'})
74
+ def select(tbl, cond = {}, opts = {})
75
+ opts = {:items => ['*']}.merge(opts)
76
+ query = "SELECT #{opts[:items].join(', ')} FROM #{tbl} #{cond_exp(cond)}"
77
+ query = "#{query} LIMIT #{opts[:limit]}" if opts.has_key?(:limit)
78
+ query = "#{query} OFFSET #{opts[:offset]}" if opts.has_key?(:offset)
79
+ return execute_statement(query, :select)
80
+ end
81
+
82
+ # The method which issues a select statement
83
+ # @param [String] query a select statement
84
+ # @return [Array] a list of records
85
+ def do_select(query)
86
+ raise NotImplementedError
87
+ end
88
+
89
+ # delete records from a table
90
+ # @param [String] tbl the name of table from which records are deleted
91
+ # @param [Hash] cond a set of conditions on deleted records
92
+ # @return [Integer] the nubmer of affected rows
93
+ # @example the below invocation issues "DELETE FROM table WHERE col = 'val'"
94
+ # delete('table', {:col => 'val'})
95
+ def delete(tbl, cond)
96
+ query = "DELETE FROM #{tbl} #{cond_exp(cond)}"
97
+ return execute_statement(query, :update)
98
+ end
99
+
100
+ # update records of a table
101
+ # @param [String] tbl the name of table in which records are updated
102
+ # @param [Hash] new_val a set of new values
103
+ # @param [Hash] cond a set of conditions on update records
104
+ # @return [Integer] the nubmer of affected rows
105
+ # @example the below invocation issues "UPDATE table SET col = 'val2' WHERE col = 'val1'"
106
+ # update('table', {:col => 'val2'}, {:col => 'val1'})
107
+ def update(tbl, new_val, cond)
108
+ query = "UPDATE #{tbl} SET #{value_exp(new_val)} #{cond_exp(cond)}"
109
+ return execute_statement(query, :update)
110
+ end
111
+
112
+ # The method which issues a delete/update statement
113
+ # @param [String] query a delete/update statement
114
+ # @return [Integer] the nubmer of affected rows
115
+ def do_update(query)
116
+ raise NotImplementedError
117
+ end
118
+
119
+ # get an expression used as values in insert statements
120
+ def value_exp(val)
121
+ raise "illegal type of value_exp #{val.class}" unless val.is_a?(Hash)
122
+ return val.map{|k,v| v.nil? ? "#{k} = NULL" : "#{k} = #{quote(v)}"}.join(",")
123
+ end
124
+
125
+ # get an expression of where clause
126
+ def cond_exp(cond)
127
+ cond = cond.map{|k,v| v.nil? ? "#{k} IS NULL" : "#{k} = #{quote(v)}"}.join(' AND ') if cond.is_a?(Hash)
128
+ raise "illegal type of cond : #{cond.class}" unless cond.is_a?(String)
129
+ cond = "WHERE #{cond}" unless cond == ""
130
+ return cond
131
+ end
132
+
133
+ # quote a given value to use in SQL
134
+ def quote(v)
135
+ raise NotImplementedError
136
+ end
137
+
138
+ end
139
+
140
+ end
141
+ end
142
+ end
@@ -0,0 +1,53 @@
1
+
2
+ module Patriot
3
+ module Util
4
+ module DBClient
5
+
6
+ # a record implementation based on Hash
7
+ class HashRecord < Patriot::Util::DBClient::Record
8
+
9
+ # @param record [Hash]
10
+ def initialize(record)
11
+ @record = {}
12
+ # ignore fixnum which is not column name
13
+ record.each{|k,v| @record[k.to_sym] = v unless k.is_a?(Fixnum)}
14
+ end
15
+
16
+ # @see Patriot::Util::DBClient::Record#get_id
17
+ def get_id
18
+ return @record[:id]
19
+ end
20
+
21
+ # @see Patriot::Util::DBClient::Record#to_mash
22
+ def to_hash(keys = @record.keys)
23
+ hash = {}
24
+ keys.each{|k| hash[k] = @record[k]}
25
+ return hash
26
+ end
27
+
28
+ # used as accessors
29
+ def method_missing(mth, *args, &blk)
30
+ key = mth
31
+ type = :get
32
+ if key.to_s.end_with?("=")
33
+ key = key.to_s.slice(0..-2).to_sym
34
+ type = :set
35
+ end
36
+
37
+ if type == :get
38
+ if @record.has_key?(mth)
39
+ return @record[mth]
40
+ else
41
+ return nil
42
+ end
43
+ elsif type == :set
44
+ @record[key] = args[0]
45
+ return
46
+ end
47
+ super
48
+ end
49
+
50
+ end
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,25 @@
1
+
2
+ module Patriot
3
+ module Util
4
+ module DBClient
5
+
6
+ # a class for abstracting access to records
7
+ # sub classes of this class should provide accessers for columns or select items
8
+ # (e.g. overwrite method_missing)
9
+ class Record
10
+
11
+ # get serial id of this record
12
+ def get_id
13
+ raise NotImplementedError
14
+ end
15
+
16
+ # convert this record to hash
17
+ # @param keys [Hash] attributes included in the returned hash
18
+ def to_hash(keys)
19
+ raise NotImplementedError
20
+ end
21
+ end
22
+ end
23
+ end
24
+ end
25
+
@@ -0,0 +1,24 @@
1
+ require 'rubygems'
2
+ require 'log4r'
3
+
4
+ require 'patriot/util/logger/facade'
5
+ require 'patriot/util/logger/factory'
6
+ require 'patriot/util/logger/log4r_factory'
7
+ require 'patriot/util/logger/webrick_log_factory'
8
+
9
+ module Patriot
10
+ module Util
11
+ # logger namespace
12
+ module Logger
13
+ include Patriot::Util::Config
14
+
15
+ # create logger based on a given configration
16
+ # @param conf [Patriot::Util::Config::Base]
17
+ def create_logger(conf)
18
+ name = self.class.to_s
19
+ @logger = Patriot::Util::Logger::Factory.create_logger(name, conf)
20
+ end
21
+
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,33 @@
1
+ module Patriot
2
+ module Util
3
+ module Logger
4
+ # facade class for loggers
5
+ class Facade
6
+ # supported log levels
7
+ LEVELS = [:debug, :info, :warn, :error, :fatal]
8
+
9
+ # @param logger [Object] logger hidden in this facade
10
+ def initialize(logger)
11
+ @logger = logger
12
+ end
13
+
14
+ LEVELS.each do |lvl|
15
+ define_method(lvl) do |msg|
16
+ if msg.is_a? Exception
17
+ @logger.send(lvl, msg.message)
18
+ msg.backtrace.each{|bt| @logger.send(lvl, bt)}
19
+ else
20
+ @logger.send(lvl, msg)
21
+ end
22
+ end
23
+ end
24
+
25
+ # for Rack::CommonLogger
26
+ def write(msg)
27
+ @logger.info(msg)
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
33
+
@@ -0,0 +1,59 @@
1
+ require 'thread'
2
+ module Patriot
3
+ module Util
4
+ module Logger
5
+
6
+ # logger factory
7
+ class Factory
8
+ # configuration key for logger factory class
9
+ FACTORY_CLASS_KEY = :log_factory
10
+
11
+ # default logger factory class
12
+ DEFAULT_LOGGER_FACTORY_CLASS = 'Patriot::Util::Logger::Log4rFactory'
13
+
14
+ # @param name [String] logger name
15
+ # @param config [Patriot::Util::Config::Base] configuration
16
+ # @return [Patriot::Util::Logger::Facade]
17
+ def self.create_logger(name, config)
18
+ klass = get_factory_class_name(config)
19
+ # implentations of logger facotory should include Singleton
20
+ logger = eval(klass).instance.get_logger(name, config)
21
+ return logger
22
+ end
23
+
24
+ # @param config [Patriot::Util::Config::Base]
25
+ # @return [String] factory class name
26
+ def self.get_factory_class_name(config)
27
+ return config.get(FACTORY_CLASS_KEY, DEFAULT_LOGGER_FACTORY_CLASS).to_s
28
+ end
29
+
30
+ def initialize
31
+ @mutex = Mutex.new
32
+ @loggers = {}
33
+ end
34
+
35
+ # @param name [String] logger name
36
+ # @param config [Patriot::Util::Config::Base] configuration
37
+ # @return [Patriot::Util::Logger::Facade]
38
+ def get_logger(name, config)
39
+ @mutex.synchronize do
40
+ unless @loggers.has_key?(name)
41
+ logger = build(name,config)
42
+ @loggers[name] = Facade.new(logger)
43
+ end
44
+ return @loggers[name]
45
+ end
46
+ end
47
+
48
+ # build logger
49
+ # should be overridden in sub-classes
50
+ # @param name [String] logger name
51
+ # @param config [Patriot::Util::Config::Base] configuration
52
+ def build(name, config)
53
+ raise NotImplementedError
54
+ end
55
+
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,111 @@
1
+ require 'rubygems'
2
+ require 'log4r'
3
+ require 'log4r/outputter/datefileoutputter'
4
+ require 'patriot/util/logger/factory'
5
+ require 'singleton'
6
+
7
+ module Patriot
8
+ module Util
9
+ module Logger
10
+ # a logger factory implementation for Log4r logger
11
+ class Log4rFactory < Patriot::Util::Logger::Factory
12
+ # configuration key for log level
13
+ LOG_LEVEL_KEY = :log_level
14
+ # default log level
15
+ DEFAULT_LOG_LEVEL = 'DEBUG'
16
+ # configuration key for log format
17
+ FORMAT_KEY = :log_format
18
+ # default log format
19
+ DEFAULT_LOG_FORMAT = '%m'
20
+ # configuration key for log outputers
21
+ OUTPUTTERS_KEY = :log_outputters
22
+ # configuration prefix for log outputer
23
+ OUTPUTTER_KEY_PREFIX = :log_outputter
24
+
25
+ include Singleton
26
+
27
+ # @see Patriot::Util::Logger:Factory
28
+ def build(name, config)
29
+ logger = Log4r::Logger.new(name)
30
+ logger = set_log_level(logger, config)
31
+ logger = set_outputters(logger, config)
32
+ return logger
33
+ end
34
+ private :build
35
+
36
+ # set log level to the logger
37
+ def set_log_level(logger, config)
38
+ log_level = get_log_level(config)
39
+ logger.level = log_level
40
+ return logger
41
+ end
42
+ private :set_log_level
43
+
44
+ # set outputters to the logger
45
+ def set_outputters(logger, config)
46
+ formatter = create_formatter(config)
47
+ logger.outputters = get_outputters(config, formatter)
48
+ return logger
49
+ end
50
+ private :set_outputters
51
+
52
+ # create formatter based on the configuration
53
+ def create_formatter(config)
54
+ log_format = get_format_config(config)
55
+ formatter = Log4r::PatternFormatter.new(:pattern => log_format)
56
+ return formatter
57
+ end
58
+ private :set_outputters
59
+
60
+ # @private
61
+ def get_log_level(_conf)
62
+ log_level = _conf.get(LOG_LEVEL_KEY, DEFAULT_LOG_LEVEL)
63
+ log_level = eval("Log4r::#{log_level}")
64
+ return log_level
65
+ end
66
+ private :get_log_level
67
+
68
+ # @private
69
+ def get_format_config(_conf)
70
+ return _conf.get(FORMAT_KEY, DEFAULT_LOG_FORMAT)
71
+ end
72
+ private :get_format_config
73
+
74
+ # @private
75
+ def get_outputters(_conf, _formatter)
76
+ outputters = _conf.get(OUTPUTTERS_KEY, [])
77
+ outputters = [outputters] unless outputters.is_a?(Array)
78
+ return outputters.map{|o| get_outputter(o, _conf, _formatter)}
79
+ return outputters
80
+ end
81
+ private :get_outputters
82
+
83
+ # @private
84
+ def get_outputter(outputter_id, _conf, _formatter)
85
+ class_key = [OUTPUTTER_KEY_PREFIX, outputter_id, "class"].join(".")
86
+ class_name = _conf.get(class_key)
87
+ raise "logger class: #{class_key} is not set" if class_name.nil?
88
+ outputter = nil
89
+ case class_name
90
+ when "Log4r::StdoutOutputter"
91
+ outputter = Log4r::Outputter.stdout
92
+ when "Log4r::StderrOutputter"
93
+ outputter = Log4r::Outputter.stderr
94
+ when "Log4r::DateFileOutputter"
95
+ dir_key = [OUTPUTTER_KEY_PREFIX, outputter_id, "dir"].join(".")
96
+ file_key = [OUTPUTTER_KEY_PREFIX, outputter_id, "file"].join(".")
97
+ outputter = Log4r::DateFileOutputter.new("file", :filename => _conf.get(file_key), :dirname => _conf.get(dir_key))
98
+ else
99
+ raise "unsupported outputter #{klass}"
100
+ end
101
+ # TODO to set outputter-specific formatter
102
+ outputter.formatter = _formatter
103
+ return outputter
104
+ end
105
+ private :get_outputter
106
+
107
+ end
108
+ end
109
+ end
110
+ end
111
+