patriot-workflow-scheduler 0.7.2 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/lib/patriot.rb +1 -0
- data/lib/patriot/command.rb +11 -11
- data/lib/patriot/command/base.rb +3 -3
- data/lib/patriot/command/composite.rb +20 -6
- data/lib/patriot/command/post_processor.rb +2 -2
- data/lib/patriot/command/post_processor/base.rb +3 -2
- data/lib/patriot/command/post_processor/mail_notification.rb +3 -3
- data/lib/patriot/command/post_processor/retrial.rb +3 -3
- data/lib/patriot/command/sh_command.rb +14 -0
- data/lib/patriot/controller/worker_admin_controller.rb +13 -8
- data/lib/patriot/job_store/base.rb +16 -7
- data/lib/patriot/job_store/in_memory_store.rb +121 -29
- data/lib/patriot/job_store/job.rb +7 -7
- data/lib/patriot/job_store/job_ticket.rb +1 -0
- data/lib/patriot/job_store/rdb_job_store.rb +161 -54
- data/lib/patriot/tool/patriot_commands/execute.rb +1 -1
- data/lib/patriot/tool/patriot_commands/job.rb +6 -4
- data/lib/patriot/tool/patriot_commands/register.rb +7 -5
- data/lib/patriot/util/config.rb +14 -3
- data/lib/patriot/util/config/inifile_config.rb +1 -1
- data/lib/patriot/version.rb +3 -0
- data/lib/patriot/worker/base.rb +6 -3
- data/lib/patriot/worker/info_server.rb +34 -24
- data/lib/patriot/worker/servlet.rb +4 -8
- data/lib/patriot/worker/servlet/api_servlet_base.rb +40 -0
- data/lib/patriot/worker/servlet/index_servlet.rb +32 -0
- data/lib/patriot/worker/servlet/job_api_servlet.rb +156 -0
- data/lib/patriot/worker/servlet/worker_api_servlet.rb +67 -0
- data/skel/batch/sample/daily/test.pbc +1 -1
- data/skel/public/css/bootstrap.min.css +7412 -0
- data/skel/public/css/original.css +179 -54
- data/skel/public/js/patriot-workflow-scheduler-0.8.0.js +82252 -0
- data/skel/public/js/patriot-workflow-scheduler-0.8.0.min.js +26 -0
- data/skel/public/js/patriot-workflow-scheduler-0.8.0.min.js.map +1 -0
- data/skel/public/templates/_jobs.erb +4 -5
- data/skel/public/templates/job.erb +2 -4
- data/skel/public/templates/layout.erb +10 -10
- data/skel/public/views/index.erb +13 -0
- metadata +40 -4
- data/lib/patriot/worker/servlet/job_servlet.rb +0 -128
- data/lib/patriot/worker/servlet/worker_status_servlet.rb +0 -52
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
YjNjNzA3NWI2YWU2ZmQ0OWZhM2E1ZmJjNTVlNTBmN2Q0YzkyYjQ3OQ==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
MDRmNGNkOWE0MDAzYmY2NDA5MGZkYzJhNDc2N2Y0ZWM4ZjEzMGM2OQ==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
ZTY5YzAwNjFmYjNlNjllODliNGM4MzRjMjBhYzgwMDZhZWZlZjcyYWE2MmM4
|
10
|
+
ZGYwNDgyNTljOTg5ZTY0MDBmN2M4ZWYyOWQ4YTE5YzkwM2Y2YmE4NGU3NWI5
|
11
|
+
MDdkNDIwMWFmNWYzZjY3MzIzZmMzNmQ0OWQ2MTc2NWRkNWJmZDM=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
NWE5OGY0NDQ2NGY4NjAwNTJmMjIwNGJiMWU5OGExMjY4ZjkxMTQyOGU2MjYy
|
14
|
+
Mjg5YzgxZDVhOTZlZGQzYTg1YWU3ZDYyY2Y4N2FiYzk4NmY1ODQ0MDdlYmVm
|
15
|
+
MWU0MWE1ZTNmY2UyYjE0NWM5Y2M0MWVhMjRjOGZiMzQ2YTFhOTI=
|
data/lib/patriot.rb
CHANGED
data/lib/patriot/command.rb
CHANGED
@@ -1,28 +1,28 @@
|
|
1
1
|
# the root name space for this scheduler
|
2
2
|
module Patriot
|
3
3
|
# a name space for commands
|
4
|
-
module Command
|
4
|
+
module Command
|
5
5
|
# a parameter key for command class name (used in jobs)
|
6
|
-
COMMAND_CLASS_KEY =
|
6
|
+
COMMAND_CLASS_KEY = :COMMAND_CLASS
|
7
7
|
|
8
8
|
# attribute name for required products
|
9
|
-
REQUISITES_ATTR =
|
9
|
+
REQUISITES_ATTR = :requisites
|
10
10
|
# attribute name for produced products
|
11
|
-
PRODUCTS_ATTR =
|
11
|
+
PRODUCTS_ATTR = :products
|
12
12
|
# attribute name for job state
|
13
|
-
STATE_ATTR =
|
13
|
+
STATE_ATTR = :state
|
14
14
|
# attribute name for job priority
|
15
|
-
PRIORITY_ATTR =
|
15
|
+
PRIORITY_ATTR = :priority
|
16
16
|
# attribute name for job execution node constraint
|
17
|
-
EXEC_NODE_ATTR =
|
17
|
+
EXEC_NODE_ATTR = :exec_node
|
18
18
|
# attribute name for job execution host constraint
|
19
|
-
EXEC_HOST_ATTR =
|
19
|
+
EXEC_HOST_ATTR = :exec_host
|
20
20
|
# attribute name for start time constraint
|
21
|
-
START_DATETIME_ATTR =
|
21
|
+
START_DATETIME_ATTR = :start_datetime
|
22
22
|
# attribute name for retry configration
|
23
|
-
POST_PROCESSORS_ATTR =
|
23
|
+
POST_PROCESSORS_ATTR = :post_processors
|
24
24
|
|
25
|
-
# a list of
|
25
|
+
# a list of command attributes
|
26
26
|
COMMON_ATTRIBUTES = [
|
27
27
|
REQUISITES_ATTR,
|
28
28
|
PRODUCTS_ATTR,
|
data/lib/patriot/command/base.rb
CHANGED
@@ -16,7 +16,7 @@ module Patriot
|
|
16
16
|
attr_accessor :config, :parser, :test_mode, :target_datetime, :post_processors
|
17
17
|
attr_writer :start_datetime
|
18
18
|
|
19
|
-
#
|
19
|
+
# command attributes handled distinctively (only effective in top level commands)
|
20
20
|
volatile_attr :requisites,
|
21
21
|
:products,
|
22
22
|
:priority,
|
@@ -145,7 +145,7 @@ module Patriot
|
|
145
145
|
def init_param
|
146
146
|
# set parameter value to instance variable
|
147
147
|
@param.each do |k,v|
|
148
|
-
raise "a reserved word #{k} is used as parameter name" if Patriot::Command::COMMAND_CLASS_KEY == k
|
148
|
+
raise "a reserved word #{k} is used as parameter name" if Patriot::Command::COMMAND_CLASS_KEY == k.to_sym
|
149
149
|
raise "#{k} is already used in #{self.job_id}" unless instance_variable_get("@#{k}".to_sym).nil?
|
150
150
|
# don't evaluate here since all parameters are not set to instance variables
|
151
151
|
instance_variable_set("@#{k}".to_sym,v)
|
@@ -177,7 +177,7 @@ module Patriot
|
|
177
177
|
val = self.instance_variable_get("@#{attr}".to_sym)
|
178
178
|
logics.each do |l|
|
179
179
|
unless l.call(self, attr, val)
|
180
|
-
raise "validation error : #{attr}=#{val} (#{self.class})"
|
180
|
+
raise "validation error : #{attr}=#{val} (#{self.class})"
|
181
181
|
end
|
182
182
|
end
|
183
183
|
end
|
@@ -1,11 +1,11 @@
|
|
1
1
|
module Patriot
|
2
2
|
module Command
|
3
3
|
# a command which is composed of multiple sub commands
|
4
|
-
class CompositeCommand < Patriot::Command::CommandGroup
|
5
|
-
declare_command_name :composite_command
|
6
|
-
declare_command_name :composite_job
|
4
|
+
class CompositeCommand < Patriot::Command::CommandGroup
|
5
|
+
declare_command_name :composite_command
|
6
|
+
declare_command_name :composite_job
|
7
7
|
private_command_attr :contained_commands => []
|
8
|
-
|
8
|
+
command_attr :name, :name_suffix
|
9
9
|
|
10
10
|
# @return [String] the identifier of this composite command
|
11
11
|
# @see Patriot::Command::Base#job_id
|
@@ -27,7 +27,9 @@ module Patriot
|
|
27
27
|
@name_suffix ||= _date_
|
28
28
|
# don't do flatten to handle nested composite commands
|
29
29
|
@subcommands.map do |cmd|
|
30
|
-
|
30
|
+
cmd = cmd.clone
|
31
|
+
cmd.build(@param).each do |cmd|
|
32
|
+
_validate_command(cmd)
|
31
33
|
require cmd['requisites']
|
32
34
|
produce cmd['products']
|
33
35
|
@contained_commands << cmd
|
@@ -42,7 +44,19 @@ module Patriot
|
|
42
44
|
@contained_commands.each do |c|
|
43
45
|
c.execute
|
44
46
|
end
|
45
|
-
end
|
47
|
+
end
|
48
|
+
|
49
|
+
# @private
|
50
|
+
# validate command
|
51
|
+
# @param [Patriot::Command::Base] cmd
|
52
|
+
def _validate_command(cmd)
|
53
|
+
if !cmd['post_processors'].nil?
|
54
|
+
raise 'you cannot set "post_processor" at subcommand of composite_job\'s ' \
|
55
|
+
+ "\n" + 'name: ' + cmd['name'] \
|
56
|
+
+ "\n" + 'command: ' + cmd['commands'].to_s
|
57
|
+
end
|
58
|
+
end
|
59
|
+
private :_validate_command
|
46
60
|
|
47
61
|
end
|
48
62
|
end
|
@@ -1,9 +1,9 @@
|
|
1
1
|
# the root name space for this scheduler
|
2
2
|
module Patriot
|
3
3
|
# a name space for commands
|
4
|
-
module Command
|
4
|
+
module Command
|
5
5
|
module PostProcessor
|
6
|
-
POST_PROCESSOR_CLASS_KEY =
|
6
|
+
POST_PROCESSOR_CLASS_KEY = :POST_PROCESSOR_CLASS
|
7
7
|
require 'patriot/command/post_processor/base'
|
8
8
|
require 'patriot/command/post_processor/skip_on_fail'
|
9
9
|
require 'patriot/command/post_processor/retrial'
|
@@ -22,8 +22,9 @@ module Patriot
|
|
22
22
|
|
23
23
|
# @param props [Hash] properties of this post processor
|
24
24
|
def initialize(props = {})
|
25
|
-
|
26
|
-
@props =
|
25
|
+
@props = {}
|
26
|
+
props.each{|k,v| @props[k.to_sym] = v}
|
27
|
+
validate_props(@props)
|
27
28
|
end
|
28
29
|
|
29
30
|
def validate_props(props)
|
@@ -4,10 +4,10 @@ module Patriot
|
|
4
4
|
module PostProcessor
|
5
5
|
class MailNotification < Patriot::Command::PostProcessor::Base
|
6
6
|
|
7
|
-
TO_PROP_KEY =
|
8
|
-
ON_PROP_KEY =
|
7
|
+
TO_PROP_KEY = :to
|
8
|
+
ON_PROP_KEY = :on
|
9
9
|
|
10
|
-
declare_post_processor_name :mail_notification
|
10
|
+
declare_post_processor_name :mail_notification
|
11
11
|
|
12
12
|
def validate_props(props)
|
13
13
|
raise "#{TO_PROP_KEY} is not specified" unless props.has_key?(TO_PROP_KEY)
|
@@ -3,8 +3,8 @@ module Patriot
|
|
3
3
|
module PostProcessor
|
4
4
|
class Retrial < Patriot::Command::PostProcessor::Base
|
5
5
|
|
6
|
-
COUNT_PROP_KEY =
|
7
|
-
INTERVAL_PROP_KEY =
|
6
|
+
COUNT_PROP_KEY = :count
|
7
|
+
INTERVAL_PROP_KEY = :interval
|
8
8
|
|
9
9
|
declare_post_processor_name :retrial
|
10
10
|
|
@@ -21,7 +21,7 @@ module Patriot
|
|
21
21
|
found = true
|
22
22
|
# count first attempt in
|
23
23
|
pp.props[COUNT_PROP_KEY] = pp.props[COUNT_PROP_KEY] - 1
|
24
|
-
return if pp.props[COUNT_PROP_KEY] == 0
|
24
|
+
return if pp.props[COUNT_PROP_KEY] == 0
|
25
25
|
cmd.start_datetime = Time.now + pp.props[INTERVAL_PROP_KEY]
|
26
26
|
end
|
27
27
|
job = cmd.to_job
|
@@ -1,15 +1,29 @@
|
|
1
1
|
module Patriot
|
2
2
|
module Command
|
3
3
|
# a command which executes shell scripts
|
4
|
+
#
|
5
|
+
# == Example pbc
|
6
|
+
# sh {
|
7
|
+
# name "test"
|
8
|
+
# commands "echo '#{_date_}' > /tmp/test.out"
|
9
|
+
# }
|
4
10
|
class ShCommand < Patriot::Command::Base
|
5
11
|
include Patriot::Util::System
|
6
12
|
|
7
13
|
declare_command_name :sh
|
8
14
|
|
9
15
|
command_attr :connector => '&&'
|
16
|
+
# @!attribute [w] commands
|
17
|
+
# [String, Array] commands to execute
|
10
18
|
command_attr :commands do |cmd, a, v|
|
11
19
|
cmd.commands = v.is_a?(Array)? v : [v]
|
12
20
|
end
|
21
|
+
# @!attribute [w] name
|
22
|
+
# [String] string to be a part of job id
|
23
|
+
# @see Patriot::Command::ShCommand#job_id
|
24
|
+
# @!attribute [w] name_suffix
|
25
|
+
# [String] suffix string to be a part of job id
|
26
|
+
# @see Patriot::Command::ShCommand#job_id
|
13
27
|
command_attr :name, :name_suffix
|
14
28
|
validate_existence :name
|
15
29
|
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'rest_client'
|
2
|
+
require 'base64'
|
2
3
|
|
3
4
|
module Patriot
|
4
5
|
module Controller
|
@@ -18,21 +19,24 @@ module Patriot
|
|
18
19
|
@config = config
|
19
20
|
@logger = create_logger(config)
|
20
21
|
set_default_values
|
22
|
+
username = config.get(Patriot::Util::Config::USERNAME_KEY, "")
|
23
|
+
password = config.get(Patriot::Util::Config::PASSWORD_KEY, "")
|
24
|
+
@auth = 'Basic ' + Base64.encode64("#{username}:#{password}").chomp
|
21
25
|
end
|
22
26
|
|
23
27
|
# @private
|
24
28
|
def set_default_values
|
25
|
-
@default_hosts = @config.get(
|
26
|
-
@default_port = @config.get(
|
27
|
-
@user = @config.get(
|
29
|
+
@default_hosts = @config.get(WORKER_HOST_KEY) || []
|
30
|
+
@default_port = @config.get(INFO_SERVER_PORT_KEY)
|
31
|
+
@user = @config.get(ADMIN_USER_KEY)
|
28
32
|
end
|
29
33
|
private :set_default_values
|
30
34
|
|
31
35
|
# execute block for each target hosts
|
32
36
|
# @param options [Hash]
|
33
37
|
# @option options :host a target host
|
34
|
-
# @option options :hosts a
|
35
|
-
# @option options :all set true to target all hosts in the configuration
|
38
|
+
# @option options :hosts a comma separated value of target hosts
|
39
|
+
# @option options :all set true to target all hosts in the configuration
|
36
40
|
# @return [Hash] a hash from host name to the result of the block
|
37
41
|
def request_to_target_hosts(options = {}, &blk)
|
38
42
|
hosts = []
|
@@ -90,7 +94,8 @@ module Patriot
|
|
90
94
|
# @param host [String] host name of the target host
|
91
95
|
# @param port [String] port number of the worker process on the target host
|
92
96
|
def put_worker_status(host, port, new_status)
|
93
|
-
|
97
|
+
resource = RestClient::Resource.new("http://#{host}:#{port}/worker")
|
98
|
+
return resource.put({:status => new_status}, :Authorization => @auth )
|
94
99
|
end
|
95
100
|
|
96
101
|
# start target workers
|
@@ -109,13 +114,13 @@ module Patriot
|
|
109
114
|
# @param options @see {#request_to_target_hosts}
|
110
115
|
def restart_worker(options = {})
|
111
116
|
options = {:interval => 60}.merge(options)
|
112
|
-
target_nodes = request_to_target_hosts(options){|h,p| controll_worker_at(h,'stop')}
|
117
|
+
target_nodes = request_to_target_hosts(options){|h,p| controll_worker_at(h,'stop')}
|
113
118
|
target_nodes.keys.each{|host| target_nodes[host] = true}
|
114
119
|
|
115
120
|
port = options.has_key?(:port) ? options[:port] : @default_port
|
116
121
|
while(target_nodes.has_value?(true))
|
117
122
|
target_nodes.keys.each do |host|
|
118
|
-
next unless target_nodes[host] # skip already started
|
123
|
+
next unless target_nodes[host] # skip already started
|
119
124
|
res = get_worker_status(host,port)
|
120
125
|
if res.nil?
|
121
126
|
controll_worker_at(host,'start')
|
@@ -63,17 +63,18 @@ module Patriot
|
|
63
63
|
end
|
64
64
|
|
65
65
|
# get a job
|
66
|
-
# @param [String] job_id
|
67
|
-
# @param [Hash]
|
66
|
+
# @param job_id [String] job_id
|
67
|
+
# @param opts [Hash]
|
68
68
|
# @option opts [String] :include_dependency include jobs with 1-hop dependency
|
69
69
|
# @return [Patrio::JobStore::Job] in case of include_dependency is true,
|
70
70
|
# jobs in dependency set to :consumers/:producers as a hash from job_id to state
|
71
71
|
def get(job_id, opts={})
|
72
|
+
return nil if job_id.nil?
|
72
73
|
job = get_job(job_id)
|
73
74
|
return if job.nil?
|
74
75
|
if opts[:include_dependency] == true
|
75
|
-
job[
|
76
|
-
job[
|
76
|
+
job[:consumers] = get_consumers(job[Patriot::Command::PRODUCTS_ATTR]) || []
|
77
|
+
job[:producers] = get_producers(job[Patriot::Command::REQUISITES_ATTR]) || []
|
77
78
|
end
|
78
79
|
return job
|
79
80
|
end
|
@@ -93,7 +94,7 @@ module Patriot
|
|
93
94
|
def get_producers(products, opts = {:include_attrs => [Patriot::Command::STATE_ATTR]})
|
94
95
|
raise NotImplementedError
|
95
96
|
end
|
96
|
-
|
97
|
+
|
97
98
|
# get consumers of products
|
98
99
|
# @param [Array] products a list of product name
|
99
100
|
# @param [Hash] opts
|
@@ -112,6 +113,14 @@ module Patriot
|
|
112
113
|
raise NotImplementedError
|
113
114
|
end
|
114
115
|
|
116
|
+
# get nodes and edges information to render graph
|
117
|
+
# @param [String] job_id JOB ID
|
118
|
+
# @param [Hash] opts options
|
119
|
+
# @return [Array] [nodes, edges]
|
120
|
+
def get_graph(job_id, opts = {})
|
121
|
+
raise NotImplementedError
|
122
|
+
end
|
123
|
+
|
115
124
|
# @param [Patriot::JobStore::JobState] state
|
116
125
|
# @param [Hash] opts
|
117
126
|
# @option ops [Integer] :limit a max nubmer of jobs fetched at once
|
@@ -124,7 +133,7 @@ module Patriot
|
|
124
133
|
|
125
134
|
# @param [Hash] opts
|
126
135
|
# @option [Array<Patriot::JobStore::JobState>] :ignore_states
|
127
|
-
# @return [Hash<Patriot::JobStore::JobState, Integer>] a hash from job state to the number of jobs in the state
|
136
|
+
# @return [Hash<Patriot::JobStore::JobState, Integer>] a hash from job state to the number of jobs in the state
|
128
137
|
def get_job_size(opts = {})
|
129
138
|
raise NotImplementedError
|
130
139
|
end
|
@@ -147,7 +156,7 @@ module Patriot
|
|
147
156
|
}.compact.flatten
|
148
157
|
consumers = get_consumers(products)
|
149
158
|
while !consumers.empty?
|
150
|
-
jobs = consumers.
|
159
|
+
jobs = consumers.map{|job| get_job(job[:job_id])}.compact
|
151
160
|
yield self, jobs
|
152
161
|
products = jobs.map{|j| j[Patriot::Command::PRODUCTS_ATTR]}.compact.flatten
|
153
162
|
consumers = get_consumers(products)
|
@@ -29,26 +29,31 @@ module Patriot
|
|
29
29
|
def register(update_id, jobs)
|
30
30
|
jobs.each{|job| raise "#{job.job_id} is not acceptable" unless acceptable?(job) }
|
31
31
|
@mutex.synchronize do
|
32
|
-
jobs.each
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
32
|
+
jobs.each {|job| _upsert(update_id, job) }
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def _upsert(update_id, job)
|
37
|
+
job_id = job.job_id.to_sym
|
38
|
+
if @jobs.has_key?(job_id) # update
|
39
|
+
original = @jobs[job_id]
|
40
|
+
job[Patriot::Command::STATE_ATTR] ||= original[Patriot::Command::STATE_ATTR]
|
41
|
+
job.update_id = original.update_id
|
42
|
+
else # insert
|
43
|
+
job[Patriot::Command::STATE_ATTR] ||= Patriot::JobStore::JobState::INIT
|
44
|
+
raise "update_id id should not be nil for new jobs" if update_id.nil?
|
45
|
+
job.update_id = update_id
|
46
|
+
end
|
47
|
+
@jobs[job_id] = job
|
48
|
+
@producers[job_id] = job[Patriot::Command::PRODUCTS_ATTR] || []
|
49
|
+
@consumers[job_id] = job[Patriot::Command::REQUISITES_ATTR] || []
|
50
|
+
if job[Patriot::Command::STATE_ATTR] == Patriot::JobStore::JobState::INIT
|
51
|
+
_set_state(job_id, Patriot::JobStore::JobState::WAIT)
|
52
|
+
else
|
53
|
+
_set_state(job_id, job[Patriot::Command::STATE_ATTR])
|
50
54
|
end
|
51
55
|
end
|
56
|
+
private :_upsert
|
52
57
|
|
53
58
|
# @see Patriot::JobStore::Base#acceptable?
|
54
59
|
def acceptable?(job)
|
@@ -141,6 +146,8 @@ module Patriot
|
|
141
146
|
|
142
147
|
# @see Patriot::JobStore::Base#get_job
|
143
148
|
def get_job(job_id)
|
149
|
+
return nil if job_id.nil?
|
150
|
+
raise "string is expected but job_id is a #{job_id.class}" unless job_id.is_a?(String)
|
144
151
|
return @jobs[job_id.to_sym]
|
145
152
|
end
|
146
153
|
|
@@ -148,26 +155,34 @@ module Patriot
|
|
148
155
|
def get_producers(products, opts = {:include_attrs => [Patriot::Command::STATE_ATTR]})
|
149
156
|
opts = {:include_attrs => []}.merge(opts)
|
150
157
|
products = [products] unless products.is_a?(Array)
|
151
|
-
producers =
|
152
|
-
products.each{|product|
|
153
|
-
@producers.map{|pid, prods|
|
154
|
-
|
158
|
+
producers = []
|
159
|
+
products.each{|product|
|
160
|
+
@producers.map{|pid, prods|
|
161
|
+
if prods.include?(product)
|
162
|
+
job = @jobs[pid].filter_attributes(opts[:include_attrs])
|
163
|
+
job[:job_id] = pid.to_s
|
164
|
+
producers.push(job)
|
165
|
+
end
|
155
166
|
}
|
156
167
|
}
|
157
|
-
return producers
|
168
|
+
return producers.uniq
|
158
169
|
end
|
159
|
-
|
170
|
+
|
160
171
|
# @see Patriot::JobStore::Base#get_consumers
|
161
172
|
def get_consumers(products, opts = {:include_attrs => [Patriot::Command::STATE_ATTR]})
|
162
173
|
opts = {:include_attrs => []}.merge(opts)
|
163
174
|
products = [products] unless products.is_a?(Array)
|
164
|
-
consumers =
|
175
|
+
consumers = []
|
165
176
|
products.each{|product|
|
166
|
-
@consumers.map{|pid, prods|
|
167
|
-
|
177
|
+
@consumers.map{|pid, prods|
|
178
|
+
if prods.include?(product)
|
179
|
+
job = @jobs[pid].filter_attributes(opts[:include_attrs])
|
180
|
+
job[:job_id] = pid.to_s
|
181
|
+
consumers.push(job)
|
182
|
+
end
|
168
183
|
}
|
169
184
|
}
|
170
|
-
return consumers
|
185
|
+
return consumers.uniq
|
171
186
|
end
|
172
187
|
|
173
188
|
# @see Patriot::JobStore::Base#find_jobs_by_state
|
@@ -193,6 +208,83 @@ module Patriot
|
|
193
208
|
return @job_history[job_id.to_sym] || []
|
194
209
|
end
|
195
210
|
|
211
|
+
# get nodes and edges information to render graph
|
212
|
+
# @param [String] job_id JOB ID
|
213
|
+
# @param [Hash] opts options
|
214
|
+
# @return [Array] [nodes, edges]
|
215
|
+
def get_graph(job_id, opts = {})
|
216
|
+
job = get(job_id)
|
217
|
+
history = get_execution_history(job_id, {})[0]
|
218
|
+
|
219
|
+
hashed_job = {
|
220
|
+
:job_id => job.job_id,
|
221
|
+
:history => history,
|
222
|
+
:depth => 0
|
223
|
+
}.merge(job.attributes)
|
224
|
+
|
225
|
+
# set self node
|
226
|
+
nodes = {job_id => hashed_job}
|
227
|
+
edges = []
|
228
|
+
|
229
|
+
_set_dependency(
|
230
|
+
:producers,
|
231
|
+
opts[:producer_depth],
|
232
|
+
nodes,
|
233
|
+
edges,
|
234
|
+
hashed_job
|
235
|
+
)
|
236
|
+
|
237
|
+
_set_dependency(
|
238
|
+
:consumers,
|
239
|
+
opts[:consumer_depth],
|
240
|
+
nodes,
|
241
|
+
edges,
|
242
|
+
hashed_job
|
243
|
+
)
|
244
|
+
|
245
|
+
return {:nodes => nodes, :edges => edges}
|
246
|
+
end
|
247
|
+
|
248
|
+
# get dependency and set nodes and edges
|
249
|
+
#
|
250
|
+
# @private
|
251
|
+
# @param [Symbol] direction :producers or :consumers
|
252
|
+
# @param [Integer] depth dependency depth to get
|
253
|
+
# @param [Hash] nodes nodes to set for dager-d3
|
254
|
+
# @param [Array] edges edges to set for dager-d3
|
255
|
+
# @param [Hash] base_job base job to get dependency
|
256
|
+
def _set_dependency(direction, depth, nodes, edges, base_job)
|
257
|
+
return if nodes[base_job[:job_id]][:depth] == depth
|
258
|
+
|
259
|
+
base_job[direction].map{|depend_job|
|
260
|
+
job = get(depend_job[:job_id])
|
261
|
+
history = get_execution_history(depend_job[:job_id], {})[0]
|
262
|
+
|
263
|
+
hashed_job = {
|
264
|
+
:job_id => job.job_id,
|
265
|
+
:history => history,
|
266
|
+
:depth => base_job[:depth] + 1
|
267
|
+
}.merge(job.attributes)
|
268
|
+
|
269
|
+
nodes[job.job_id] = hashed_job
|
270
|
+
if direction == :producers
|
271
|
+
edges.push([job.job_id, base_job[:job_id]])
|
272
|
+
else
|
273
|
+
edges.push([base_job[:job_id], job.job_id])
|
274
|
+
end
|
275
|
+
|
276
|
+
# call recursively
|
277
|
+
_set_dependency(
|
278
|
+
direction,
|
279
|
+
depth,
|
280
|
+
nodes,
|
281
|
+
edges,
|
282
|
+
hashed_job
|
283
|
+
)
|
284
|
+
}
|
285
|
+
end
|
286
|
+
private :_set_dependency
|
287
|
+
|
196
288
|
# @see Patriot::JobStore::Base#get_job_size
|
197
289
|
def get_job_size(opts = {})
|
198
290
|
opts = {:ignore_states => []}.merge(opts)
|
@@ -208,7 +300,7 @@ module Patriot
|
|
208
300
|
# @see Patriot::JobStore::Base#delete_job
|
209
301
|
def delete_job(job_id)
|
210
302
|
job_id = job_id.to_sym
|
211
|
-
@mutex.synchronize do
|
303
|
+
@mutex.synchronize do
|
212
304
|
@job_states.each{|s,js| js.delete(job_id)}
|
213
305
|
@jobs.delete(job_id)
|
214
306
|
@producers.delete(job_id)
|