patriot-workflow-scheduler 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. checksums.yaml +15 -0
  2. data/bin/patriot +8 -0
  3. data/bin/patriot-init +35 -0
  4. data/lib/patriot.rb +11 -0
  5. data/lib/patriot/command.rb +71 -0
  6. data/lib/patriot/command/base.rb +199 -0
  7. data/lib/patriot/command/command_group.rb +43 -0
  8. data/lib/patriot/command/command_macro.rb +141 -0
  9. data/lib/patriot/command/composite.rb +49 -0
  10. data/lib/patriot/command/parser.rb +78 -0
  11. data/lib/patriot/command/sh_command.rb +42 -0
  12. data/lib/patriot/controller.rb +2 -0
  13. data/lib/patriot/controller/package_controller.rb +81 -0
  14. data/lib/patriot/controller/worker_admin_controller.rb +159 -0
  15. data/lib/patriot/job_store.rb +66 -0
  16. data/lib/patriot/job_store/base.rb +159 -0
  17. data/lib/patriot/job_store/factory.rb +19 -0
  18. data/lib/patriot/job_store/in_memory_store.rb +252 -0
  19. data/lib/patriot/job_store/job.rb +118 -0
  20. data/lib/patriot/job_store/job_ticket.rb +30 -0
  21. data/lib/patriot/job_store/rdb_job_store.rb +353 -0
  22. data/lib/patriot/tool.rb +2 -0
  23. data/lib/patriot/tool/batch_parser.rb +102 -0
  24. data/lib/patriot/tool/patriot_command.rb +48 -0
  25. data/lib/patriot/tool/patriot_commands/execute.rb +92 -0
  26. data/lib/patriot/tool/patriot_commands/job.rb +62 -0
  27. data/lib/patriot/tool/patriot_commands/plugin.rb +41 -0
  28. data/lib/patriot/tool/patriot_commands/register.rb +77 -0
  29. data/lib/patriot/tool/patriot_commands/upgrade.rb +24 -0
  30. data/lib/patriot/tool/patriot_commands/validate.rb +84 -0
  31. data/lib/patriot/tool/patriot_commands/worker.rb +35 -0
  32. data/lib/patriot/tool/patriot_commands/worker_admin.rb +60 -0
  33. data/lib/patriot/util.rb +14 -0
  34. data/lib/patriot/util/config.rb +58 -0
  35. data/lib/patriot/util/config/base.rb +22 -0
  36. data/lib/patriot/util/config/inifile_config.rb +63 -0
  37. data/lib/patriot/util/cron_format_parser.rb +104 -0
  38. data/lib/patriot/util/date_util.rb +200 -0
  39. data/lib/patriot/util/db_client.rb +65 -0
  40. data/lib/patriot/util/db_client/base.rb +142 -0
  41. data/lib/patriot/util/db_client/hash_record.rb +53 -0
  42. data/lib/patriot/util/db_client/record.rb +25 -0
  43. data/lib/patriot/util/logger.rb +24 -0
  44. data/lib/patriot/util/logger/facade.rb +33 -0
  45. data/lib/patriot/util/logger/factory.rb +59 -0
  46. data/lib/patriot/util/logger/log4r_factory.rb +111 -0
  47. data/lib/patriot/util/logger/webrick_log_factory.rb +47 -0
  48. data/lib/patriot/util/param.rb +73 -0
  49. data/lib/patriot/util/retry.rb +30 -0
  50. data/lib/patriot/util/script.rb +52 -0
  51. data/lib/patriot/util/system.rb +120 -0
  52. data/lib/patriot/worker.rb +35 -0
  53. data/lib/patriot/worker/base.rb +153 -0
  54. data/lib/patriot/worker/info_server.rb +90 -0
  55. data/lib/patriot/worker/job_store_server.rb +32 -0
  56. data/lib/patriot/worker/multi_node_worker.rb +157 -0
  57. data/lib/patriot/worker/servlet.rb +23 -0
  58. data/lib/patriot/worker/servlet/job_servlet.rb +128 -0
  59. data/lib/patriot/worker/servlet/worker_status_servlet.rb +44 -0
  60. data/skel/batch/sample/daily/test.pbc +4 -0
  61. data/skel/config/patriot.ini +21 -0
  62. data/skel/public/css/bootstrap.css +2495 -0
  63. data/skel/public/css/original.css +54 -0
  64. data/skel/public/js/bootstrap-alerts.js +124 -0
  65. data/skel/public/js/bootstrap-buttons.js +64 -0
  66. data/skel/public/js/bootstrap-dropdown.js +55 -0
  67. data/skel/public/js/bootstrap-modal.js +260 -0
  68. data/skel/public/js/bootstrap-popover.js +90 -0
  69. data/skel/public/js/bootstrap-scrollspy.js +107 -0
  70. data/skel/public/js/bootstrap-tabs.js +80 -0
  71. data/skel/public/js/bootstrap-twipsy.js +321 -0
  72. data/skel/public/js/jquery-1.6.4.min.js +4 -0
  73. data/skel/public/templates/_jobs.erb +97 -0
  74. data/skel/public/templates/job.erb +119 -0
  75. data/skel/public/templates/jobs.erb +21 -0
  76. data/skel/public/templates/jobs_deleted.erb +6 -0
  77. data/skel/public/templates/layout.erb +103 -0
  78. data/skel/public/templates/state_updated.erb +6 -0
  79. metadata +235 -0
@@ -0,0 +1,48 @@
1
+ require 'rubygems'
2
+ require 'thor'
3
+
4
+ module Patriot
5
+ # namespace for command line tools
6
+ module Tool
7
+
8
+ # new command can be added as follows
9
+ # require 'patriot/tool/patriot_command'
10
+ # module Hoge
11
+ # Patriot::Tool::PatriotCommand.class_eval do
12
+ # desc 'fuga', 'fuga description'
13
+ # def fuga
14
+ # puts 'fuga'
15
+ # end
16
+ # end
17
+ # end
18
+ class PatriotCommand < Thor
19
+ include Patriot::Util::Config
20
+ class_option :config,
21
+ :aliases => '-c',
22
+ :type => :string,
23
+ :desc => 'path to configuration file'
24
+ no_tasks do
25
+ def exit_on_failure?
26
+ return true
27
+ end
28
+
29
+ def symbolize_options(opts = {})
30
+ symbolized_options = {}
31
+ opts.each{|k,v| symbolized_options[k.to_sym] = v }
32
+ return symbolized_options
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
38
+
39
+ # command implementations
40
+ require 'patriot/tool/patriot_commands/execute'
41
+ require 'patriot/tool/patriot_commands/register'
42
+ require 'patriot/tool/patriot_commands/worker'
43
+ require 'patriot/tool/patriot_commands/worker_admin'
44
+ require 'patriot/tool/patriot_commands/validate'
45
+ require 'patriot/tool/patriot_commands/job'
46
+ require 'patriot/tool/patriot_commands/plugin'
47
+ require 'patriot/tool/patriot_commands/upgrade'
48
+
@@ -0,0 +1,92 @@
1
+ module Patriot
2
+ module Tool
3
+ # namesapce for patriot comman line tools
4
+ module PatriotCommands
5
+ # execute PBC directory
6
+ module Execute
7
+
8
+ Patriot::Tool::PatriotCommand.class_eval do
9
+ desc 'execute [options] <yyyy-mm-dd[,yyyy-mm-dd]> <files/paths>+',
10
+ 'execute patriot jobs directly'
11
+ method_option :filter,
12
+ :aliases => '-f',
13
+ :type => :string,
14
+ :desc => 'regular expression for job_id'
15
+ method_option :debug,
16
+ :aliases => '-d',
17
+ :type => :boolean,
18
+ :default => false,
19
+ :desc => 'run in debug mode'
20
+ method_option :test,
21
+ :aliases => '-t',
22
+ :type => :boolean,
23
+ :default => false,
24
+ :desc => 'run in test mode'
25
+ method_option :strict,
26
+ :type => :boolean,
27
+ :default => false,
28
+ :desc => 'run in strict mode (according to dependency)'
29
+ def execute(date, *paths)
30
+ begin
31
+ # set config/options
32
+ opts = symbolize_options(options)
33
+ conf = {:type => 'execute'}
34
+ conf[:path] = opts[:config] if opts.has_key?(:config)
35
+ config = load_config(conf)
36
+ if opts[:debug] && opts[:test]
37
+ message = "invalid option: both of debug and test are specified"
38
+ raise ArgumentError, message
39
+ end
40
+
41
+ # parse and process commands
42
+ parser = Patriot::Tool::BatchParser.new(config)
43
+ commands = parser.process(date, paths, opts)
44
+ if opts[:strict]
45
+ job_store = create_job_store_with_commands(commands, config)
46
+ until (executables = job_store.get_job_tickets(nil,nil)).empty?
47
+ executables.each do |job_ticket|
48
+ cmd = job_store.offer_to_execute(job_ticket)
49
+ execute_command(cmd[:command], opts)
50
+ job_ticket.exit_code = Patriot::Command::ExitCode::SUCCEEDED
51
+ job_store.report_completion_status(job_ticket)
52
+ end
53
+ end
54
+ else
55
+ commands.each{|cmd| execute_command(cmd, opts)}
56
+ end
57
+ rescue => e
58
+ puts e.message
59
+ $@.each {|message| puts message}
60
+ raise e
61
+ end
62
+ end
63
+
64
+ no_tasks do
65
+ def create_job_store_with_commands(commands, config)
66
+ job_store = Patriot::JobStore::InMemoryStore.new(Patriot::JobStore::ROOT_STORE_ID, config)
67
+ # ignore products not defined here
68
+ products = commands.map{|cmd| cmd['products']}.flatten.compact
69
+ jobs = commands.map do |cmd|
70
+ cmd.instance_variable_set(:@state, Patriot::JobStore::JobState::INIT)
71
+ cmd['requisites'].delete_if{|ref| !products.include?(ref)}
72
+ cmd.to_job
73
+ end
74
+ job_store.register(Time.now.to_i, jobs)
75
+ return job_store
76
+ end
77
+
78
+ def execute_command(command, opts = {})
79
+ puts "executing #{command.job_id}"
80
+ if opts[:debug]
81
+ puts command.description
82
+ return
83
+ end
84
+ command.test_mode = true if opts[:test]
85
+ command.execute
86
+ end
87
+ end
88
+ end
89
+ end
90
+ end
91
+ end
92
+ end
@@ -0,0 +1,62 @@
1
+ module Patriot
2
+ module Tool
3
+ module PatriotCommands
4
+ # handle jobs in JobStore
5
+ module Job
6
+
7
+ Patriot::Tool::PatriotCommand.class_eval do
8
+ desc 'job [delete|show_dependency] job_id [job_id ..]', 'manage job(s) in job store'
9
+ def job(subcmd, *job_id)
10
+ opts = symbolize_options(options)
11
+ conf = {:type => 'job'}
12
+ conf[:path] = opts[:config] if opts.has_key?(:config)
13
+ config = load_config(conf)
14
+ job_store = Patriot::JobStore::Factory.create_jobstore(Patriot::JobStore::ROOT_STORE_ID, config)
15
+
16
+ case subcmd
17
+ when "delete"
18
+ job_id.each do |jid|
19
+ if job_store.delete_job(jid)
20
+ puts "#{jid} is deleted"
21
+ else
22
+ puts "#{jid} does not exist"
23
+ end
24
+ end
25
+ when "show_dependency"
26
+ job_id.each do |jid|
27
+ dep = ""
28
+ dep = [job_id] | build_producer_string_values(job_store, jid, 1)
29
+ puts dep.join("\n")+"\n"
30
+ end
31
+ else
32
+ puts "unknown sub command #{subcmd}"
33
+ help('job')
34
+ end
35
+ end
36
+
37
+ no_tasks do
38
+ def build_producer_string_values(job_store, job_id, indent)
39
+ values = []
40
+ job = job_store.get(job_id)
41
+ job[Patriot::Command::REQUISITES_ATTR].each do |product|
42
+ products = job_store.get_producers(product)
43
+ values << "#{' '*indent}<= #{product} = WARN: no producer exists" if products.empty?
44
+ products.each do |jid, attrs|
45
+ dep_status = "#{jid}, #{attrs[Patriot::Command::STATE_ATTR]}"
46
+ producer_job = job_store.get(jid, :include_dependency => true)
47
+ unless producer_job['consumers'].keys.include?(job_id)
48
+ dep_status = "WARN: currupted dependency #{dep_status}"
49
+ end
50
+ values << "#{' '*indent}<= #{product} = #{dep_status}"
51
+ values |= build_producer_string_values(job_store, jid, indent+1)
52
+ end
53
+ end
54
+ return values
55
+ end
56
+ end
57
+
58
+ end
59
+ end
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,41 @@
1
+ module Patriot
2
+ module Tool
3
+ module PatriotCommands
4
+ # manage plugins
5
+ module Plugin
6
+
7
+ Patriot::Tool::PatriotCommand.class_eval do
8
+ desc 'plugin [options] install <path to plugin>',
9
+ 'manage plugins'
10
+ method_option :force,
11
+ :aliases => '-f',
12
+ :type => :boolean,
13
+ :desc => 'force operation'
14
+ method_option :unpack,
15
+ :type => :boolean,
16
+ :desc => 'unpack gem into plugin dir'
17
+ def plugin(sub_cmd, *plugin)
18
+ opts = symbolize_options(options)
19
+ conf = {:ignore_plugin => true}
20
+ conf[:path] = opts[:config] if opts.has_key?(:config)
21
+ config = load_config(conf)
22
+ controller = Patriot::Controller::PackageController.new(config)
23
+ plugins = []
24
+ if plugin.nil? || plugin.empty?
25
+ plugins = config.get(Patriot::Util::Config::PLUGIN_KEY, plugin)
26
+ else
27
+ plugins = plugin
28
+ end
29
+ sub_cmd = sub_cmd.to_sym
30
+ if sub_cmd == :install
31
+ plugins.each{|name| controller.install_plugin(name, opts) }
32
+ else
33
+ help("plugin")
34
+ end
35
+ end
36
+
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,77 @@
1
+ module Patriot
2
+ module Tool
3
+ module PatriotCommands
4
+ # register jobs to JobStore
5
+ module Register
6
+
7
+ Patriot::Tool::PatriotCommand.class_eval do
8
+ desc 'register [OPTIONS] yyyy-mm-dd[,yyyy-mm-dd] path [path file ...]', 'register jobs'
9
+ method_option :filter,
10
+ :aliases => '-f',
11
+ :type => :string,
12
+ :desc => 'regular expression for Ruby'
13
+ method_option :debug,
14
+ :aliases => '-d',
15
+ :type => :boolean,
16
+ :default => false,
17
+ :desc => 'debug mode flag'
18
+ method_option :priority,
19
+ :aliases => '-p',
20
+ :type => :numeric,
21
+ :desc => 'job priority'
22
+ method_option :state,
23
+ :aliases => '-s',
24
+ :type => :numeric,
25
+ :desc => 'register as specified state'
26
+ method_option :keep_state,
27
+ :type => :boolean,
28
+ :default => false,
29
+ :desc => "don't change current state of jobs (only change definition)"
30
+ method_option :retry_dep,
31
+ :type => :boolean,
32
+ :desc => 'set states of dependent jobs to WAIT'
33
+ method_option :update_id,
34
+ :type => :string,
35
+ :default => Time.now.to_i,
36
+ :desc => 'default value is current unixtime (default value is Time.now.to_i)'
37
+ def register(date, *paths)
38
+ begin
39
+ # set config/options
40
+ opts = symbolize_options(options)
41
+ conf = {:type => 'register'}
42
+ conf[:path] = opts[:config] if opts.has_key?(:config)
43
+ config = load_config(conf)
44
+ opts = {:update_id => Time.now.to_i,
45
+ :store_id => Patriot::JobStore::ROOT_STORE_ID,
46
+ :retry_interval => 300,
47
+ :retry_limite => 10}.merge(opts)
48
+ opts[:state] = nil if opts[:keep_state]
49
+
50
+ job_store = Patriot::JobStore::Factory.create_jobstore(opts[:store_id], config)
51
+ parser = Patriot::Tool::BatchParser.new(config)
52
+ jobs = []
53
+ parser.process(date, paths, opts) do |cmd|
54
+ job = cmd.to_job
55
+ job[Patriot::Command::PRIORITY_ATTR] = opts[:priority] if opts.has_key?(:priority)
56
+ job[Patriot::Command::STATE_ATTR] = opts[:state] if opts.has_key?(:state)
57
+ jobs << job
58
+ end
59
+ return if opts[:debug]
60
+ Patriot::Util::Retry.execute_with_retry{ job_store.register(opts[:update_id], jobs) }
61
+ if opts[:retry_dep]
62
+ job_store.process_subsequent(jobs.map(&:job_id)) do |job_store, jobs|
63
+ job_store.set_state(opts[:update_id], jobs.map(&:job_id), Patriot::JobStore::JobState::WAIT)
64
+ end
65
+ end
66
+ rescue => e
67
+ puts e
68
+ $@.each{|m| puts m}
69
+ raise e.message
70
+ end
71
+ end
72
+
73
+ end
74
+ end
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,24 @@
1
+ module Patriot
2
+ module Tool
3
+ module PatriotCommands
4
+ # upgrade tool
5
+ module Upgrade
6
+
7
+ Patriot::Tool::PatriotCommand.class_eval do
8
+ desc 'upgrade',
9
+ 'upgrade installation'
10
+ def upgrade(*path_to_gem)
11
+ opts = symbolize_options(options)
12
+ conf = {:ignore_plugin => true}
13
+ conf[:path] = opts[:config] if opts.has_key?(:config)
14
+ config = load_config(conf)
15
+ controller = Patriot::Controller::PackageController.new(config)
16
+ controller.upgrade(path_to_gem.empty? ? nil : path_to_gem[0])
17
+ end
18
+
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
24
+
@@ -0,0 +1,84 @@
1
+ module Patriot
2
+ module Tool
3
+ module PatriotCommands
4
+ # validate PBC files
5
+ module Validate
6
+
7
+ Patriot::Tool::PatriotCommand.class_eval do
8
+ VALIDATE_SAMPLE_DATE = '1999-01-31'
9
+ desc 'validate [OPTIONS] path [path file ...]', 'validate pbc files'
10
+ method_option :stop_on_detection,
11
+ :aliases => '-s',
12
+ :type => :boolean,
13
+ :default => false,
14
+ :desc => 'stop immediately when invalid config detected'
15
+ method_option :date,
16
+ :type => :string,
17
+ :desc => 'date passed to parser'
18
+ def validate(*paths)
19
+ begin
20
+ opts = symbolize_options(options)
21
+ conf = {:type => 'validator'}
22
+ conf[:path] = opts[:config] if opts.has_key?(:config)
23
+ config = load_config(conf)
24
+ # the value of :all is passed to Patriot::Util::Script.get_batch_files(path, opt = {})
25
+ opts = {:all => true, :date => VALIDATE_SAMPLE_DATE}.merge(opts)
26
+
27
+ job_store = Patriot::JobStore::Factory.create_jobstore(Patriot::JobStore::ROOT_STORE_ID, config)
28
+ job_ids = {}
29
+ invalid_syntax = []
30
+ valid = true
31
+
32
+ parser = Patriot::Tool::BatchParser.new(config)
33
+ parser.process(opts[:date], paths, opts) do |cmd, source|
34
+ unless job_store.acceptable?(cmd.to_job)
35
+ invalid_syntax << "#{command.job_id} in ${source[:path]}"
36
+ valid = false
37
+ end
38
+ get_all_job_ids(cmd).each do |jid|
39
+ if job_ids.has_key?(jid)
40
+ job_ids[jid] << source[:path]
41
+ valid = false
42
+ else
43
+ job_ids[jid] = [source[:path]]
44
+ end
45
+ end
46
+ break if opts[:stop_on_detection] && !valid
47
+ end
48
+
49
+ unless valid
50
+ unless invalid_syntax.empty?
51
+ puts "#{invalid_syntax.size} jobs are serialized to invalid syntax:"
52
+ invalid_syntax.each{|i| puts "\ti" }
53
+ end
54
+
55
+ # count dupliates
56
+ dup_cnt = job_ids.values.select{|files| files.size > 1}
57
+ unless dup_cnt == 0
58
+ puts "#{dup_cnt} duplications are detected:"
59
+ job_ids.each do |jid, files|
60
+ next if files.size == 1
61
+ puts "#{files.size} #{jid} in "
62
+ files.each{|d| puts "\t#{d} " }
63
+ end
64
+ end
65
+ raise "invalid batch config is detected"
66
+ end
67
+ puts "no invalid config is detected"
68
+ end
69
+ end
70
+
71
+ no_tasks do
72
+ def get_all_job_ids(cmd)
73
+ ids = [cmd.job_id]
74
+ if cmd.is_a?(Patriot::Command::CompositeCommand)
75
+ ids |= cmd.instance_variable_get(:@contained_commands).map{|cc| get_all_job_ids(cc)}.flatten
76
+ end
77
+ return ids
78
+ end
79
+ end
80
+ end
81
+ end
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,35 @@
1
+ module Patriot
2
+ module Tool
3
+ module PatriotCommands
4
+ # a tool to start/stop a worker
5
+ module Worker
6
+
7
+ Patriot::Tool::PatriotCommand.class_eval do
8
+ desc 'worker [options] [start|stop]',
9
+ 'controll worker'
10
+ method_option :foreground,
11
+ :type => :boolean,
12
+ :default => false,
13
+ :desc => 'run as a foreground job'
14
+ def worker(sub_cmd)
15
+ opts = symbolize_options(options)
16
+ conf = {:type => 'worker'}
17
+ conf[:path] = opts[:config] if opts.has_key?(:config)
18
+ Process.daemon unless opts[:foreground]
19
+ config = load_config(conf)
20
+ worker_cls = config.get("worker_class", "Patriot::Worker::MultiNodeWorker")
21
+ worker = eval(worker_cls).new(config)
22
+ case sub_cmd
23
+ when "start"
24
+ then worker.start_worker
25
+ when "stop"
26
+ then worker.request_shutdown
27
+ else
28
+ raise "unknown sub command #{sub_cmd}"
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end