hodor 1.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +16 -0
- data/.gitmodules +3 -0
- data/.rspec +2 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/.travis.yml +5 -0
- data/Gemfile +4 -0
- data/Guardfile +11 -0
- data/README.md +105 -0
- data/Rakefile +105 -0
- data/bin/hodor +18 -0
- data/hodor.gemspec +47 -0
- data/lib/config/log4r_config.xml +35 -0
- data/lib/hodor.rb +83 -0
- data/lib/hodor/api/hdfs.rb +222 -0
- data/lib/hodor/api/oozie.rb +215 -0
- data/lib/hodor/api/oozie/action.rb +52 -0
- data/lib/hodor/api/oozie/bundle.rb +27 -0
- data/lib/hodor/api/oozie/coordinator.rb +53 -0
- data/lib/hodor/api/oozie/hadoop_job.rb +29 -0
- data/lib/hodor/api/oozie/job.rb +192 -0
- data/lib/hodor/api/oozie/materialization.rb +56 -0
- data/lib/hodor/api/oozie/query.rb +115 -0
- data/lib/hodor/api/oozie/session.rb +170 -0
- data/lib/hodor/api/oozie/workflow.rb +58 -0
- data/lib/hodor/cli.rb +146 -0
- data/lib/hodor/command.rb +164 -0
- data/lib/hodor/configuration.rb +80 -0
- data/lib/hodor/environment.rb +437 -0
- data/lib/hodor/ui/table.rb +130 -0
- data/lib/hodor/version.rb +3 -0
- data/lib/tasks/hdfs.thor +138 -0
- data/lib/tasks/master.thor +61 -0
- data/lib/tasks/oozie.thor +399 -0
- data/lib/tasks/sandbox.thor +87 -0
- data/spec/integration/api/oozie/action_spec.rb +69 -0
- data/spec/integration/api/oozie/bundle_spec.rb +33 -0
- data/spec/integration/api/oozie/coordinator_spec.rb +66 -0
- data/spec/integration/api/oozie/hadoop_job_spec.rb +29 -0
- data/spec/integration/api/oozie/job_spec.rb +15 -0
- data/spec/integration/api/oozie/materialization_spec.rb +66 -0
- data/spec/integration/api/oozie/query_spec.rb +43 -0
- data/spec/integration/api/oozie/session_spec.rb +18 -0
- data/spec/integration/api/oozie/workflow_spec.rb +65 -0
- data/spec/integration/api/oozie_spec.rb +198 -0
- data/spec/integration/fixtures/api/running_coordinators/req_resp_00.memo +6 -0
- data/spec/integration/fixtures/api/sample_action/req_resp_00.memo +5 -0
- data/spec/integration/fixtures/api/sample_action/req_resp_01.memo +7 -0
- data/spec/integration/fixtures/api/sample_bundle/req_resp_00.memo +6 -0
- data/spec/integration/fixtures/api/sample_coordinator/req_resp_00.memo +5 -0
- data/spec/integration/fixtures/api/sample_materialization/req_resp_00.memo +5 -0
- data/spec/integration/fixtures/api/sample_materialization/req_resp_01.memo +7 -0
- data/spec/integration/fixtures/api/sample_workflow/req_resp_00.memo +5 -0
- data/spec/spec_helper.rb +92 -0
- data/spec/support/d_v_r.rb +125 -0
- data/spec/support/hodor_api.rb +15 -0
- data/spec/unit/hodor/api/hdfs_spec.rb +63 -0
- data/spec/unit/hodor/api/oozie_spec.rb +32 -0
- data/spec/unit/hodor/environment_spec.rb +52 -0
- data/topics/hdfs/corresponding_paths.txt +31 -0
- data/topics/hdfs/overview.txt +10 -0
- data/topics/master/clusters.yml.txt +36 -0
- data/topics/master/overview.txt +17 -0
- data/topics/oozie/blocking_coordinators.txt +46 -0
- data/topics/oozie/composing_job_properties.txt +68 -0
- data/topics/oozie/display_job.txt +52 -0
- data/topics/oozie/driver_scenarios.txt +42 -0
- data/topics/oozie/inspecting_jobs.txt +59 -0
- data/topics/oozie/jobs.yml.txt +185 -0
- data/topics/oozie/overview.txt +43 -0
- data/topics/oozie/workers_and_drivers.txt +40 -0
- metadata +455 -0
@@ -0,0 +1,27 @@
|
|
1
|
+
module Hodor::Oozie
|
2
|
+
|
3
|
+
class Bundle < Job
|
4
|
+
|
5
|
+
attr_reader :status, :json
|
6
|
+
|
7
|
+
class << self
|
8
|
+
def default_columns
|
9
|
+
[:index, :id, :status]
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def initialize(json)
|
14
|
+
super()
|
15
|
+
@json = json
|
16
|
+
@status = json["status"]
|
17
|
+
end
|
18
|
+
|
19
|
+
def expand
|
20
|
+
# Expand immediate children
|
21
|
+
@coordinators = json["coords"].map do |item|
|
22
|
+
Coordinator.new(item)
|
23
|
+
end.compact
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
module Hodor::Oozie
|
2
|
+
|
3
|
+
class Coordinator < Job
|
4
|
+
|
5
|
+
attr_reader :id, :json, :name, :path, :timezone, :frequency, :conf, :end_time, :execution_policy, :start_time, :time_unit,
|
6
|
+
:concurrency, :id, :last_action, :acl, :mat_throttling, :timeout, :next_materialized_time, :parent_id,
|
7
|
+
:external_id, :group, :user, :console_url, :actions, :acl, :status, :materializations
|
8
|
+
|
9
|
+
class << self
|
10
|
+
def default_columns
|
11
|
+
[:index, :id, :name, :status, :start_time, :time_unit, :external_id]
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def initialize(json)
|
16
|
+
super()
|
17
|
+
@json = json
|
18
|
+
@name = json["coordJobName"]
|
19
|
+
@path = json["coordJobPath"]
|
20
|
+
@timezone = json["timeZone"]
|
21
|
+
@frequency = json["frequency"]
|
22
|
+
@conf = json["conf"]
|
23
|
+
@end_time = parse_time(json["endTime"])
|
24
|
+
@execution_policy = json["executionPolicy"]
|
25
|
+
@start_time = parse_time(json["startTime"])
|
26
|
+
@time_unit = json["timeUnit"]
|
27
|
+
@concurrency = json["concurrency"]
|
28
|
+
@id = json["coordJobId"]
|
29
|
+
@last_action = parse_time(json["lastAction"])
|
30
|
+
@acl = json["acl"]
|
31
|
+
@mat_throttling = json["mat_throttling"]
|
32
|
+
@timeout = json["timeOut"]
|
33
|
+
@next_materialized_time = parse_time(json["nextMaterializedTime"])
|
34
|
+
@parent_id = @bundle_id = json["bundleId"]
|
35
|
+
@external_id = json["coordExternalId"]
|
36
|
+
@group = json["group"]
|
37
|
+
@user = json["user"]
|
38
|
+
@console_url = json["consoleUrl"]
|
39
|
+
@actions = json["actions"]
|
40
|
+
@acl = json["acl"]
|
41
|
+
@status = json["status"]
|
42
|
+
end
|
43
|
+
|
44
|
+
def expand
|
45
|
+
# Expand immediate children
|
46
|
+
@materializations = json["actions"].map do |item|
|
47
|
+
Materialization.new(item)
|
48
|
+
end.compact.reverse
|
49
|
+
@materializations
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Hodor::Oozie
|
2
|
+
|
3
|
+
class HadoopJob < Job
|
4
|
+
|
5
|
+
attr_reader :id, :parent_id, :rest_call
|
6
|
+
|
7
|
+
class << self
|
8
|
+
def default_columns
|
9
|
+
[:index, :id]
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def initialize(parent_id, job_id)
|
14
|
+
super()
|
15
|
+
@id = job_id
|
16
|
+
@parent_id = parent_id
|
17
|
+
session.verbose = true # force verbosity for hadoop jobs
|
18
|
+
end
|
19
|
+
|
20
|
+
def log
|
21
|
+
hadoop_id = @id.sub('job_','')
|
22
|
+
trash = hadoop_id.index(/[^0-9_]/)
|
23
|
+
hadoop_id = hadoop_id[0..trash-1] if trash
|
24
|
+
session.env.ssh "mapred job -logs job_#{hadoop_id} attempt_#{hadoop_id}_m_000000_0"
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
@@ -0,0 +1,192 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'awesome_print'
|
3
|
+
require 'stringio'
|
4
|
+
require 'ox'
|
5
|
+
|
6
|
+
require_relative 'session'
|
7
|
+
|
8
|
+
module Hodor::Oozie
|
9
|
+
|
10
|
+
class Job
|
11
|
+
attr_reader :index, :id, :parent_id, :skip_to, :conf, :rest_call
|
12
|
+
attr_accessor :columns
|
13
|
+
|
14
|
+
def session
|
15
|
+
Hodor::Oozie::Session.instance
|
16
|
+
end
|
17
|
+
|
18
|
+
def oozie
|
19
|
+
Hodor::Oozie
|
20
|
+
end
|
21
|
+
|
22
|
+
def initialize
|
23
|
+
@index = -1
|
24
|
+
@rest_call = session.last_query
|
25
|
+
end
|
26
|
+
|
27
|
+
def set_index(i)
|
28
|
+
@index = i
|
29
|
+
end
|
30
|
+
|
31
|
+
def indexed_job_id
|
32
|
+
nil
|
33
|
+
end
|
34
|
+
|
35
|
+
def parse_time timestamp
|
36
|
+
Time.strptime(timestamp, "%a, %d %b %Y %H:%M:%S %Z") if timestamp
|
37
|
+
end
|
38
|
+
|
39
|
+
def expand
|
40
|
+
nil
|
41
|
+
end
|
42
|
+
|
43
|
+
def children
|
44
|
+
if @children.nil?
|
45
|
+
@children = expand
|
46
|
+
end
|
47
|
+
@children
|
48
|
+
end
|
49
|
+
|
50
|
+
def display_as_time(val)
|
51
|
+
display_date = val.strftime("%Y-%m-%d %H:%M")
|
52
|
+
cur_date = Time.now.strftime("%Y-")
|
53
|
+
if display_date[0..4].eql?(cur_date)
|
54
|
+
display_date[5..-1]
|
55
|
+
else
|
56
|
+
display_date
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def sanitize val, max_length = 120
|
61
|
+
sval = val.to_s.gsub(/\s+/, ' ')
|
62
|
+
if sval.length > max_length
|
63
|
+
sval.to_s[0..max_length-3] + '...'
|
64
|
+
else
|
65
|
+
sval
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def title
|
70
|
+
"#{self.class.name.split('::').last} Properties"
|
71
|
+
end
|
72
|
+
|
73
|
+
def display_properties
|
74
|
+
if self.class.respond_to?(:suppress_properties)
|
75
|
+
suppress = self.class.suppress_properties
|
76
|
+
else
|
77
|
+
suppress = false
|
78
|
+
end
|
79
|
+
props = suppress ? nil :
|
80
|
+
instance_variables.map { |var| var[1..-1] }.select { |var| !var.eql?("index") }
|
81
|
+
if props
|
82
|
+
rows = props.inject([]) { |result, prop|
|
83
|
+
display_override = "display_#{prop.to_s}".to_sym
|
84
|
+
if respond_to?(display_override)
|
85
|
+
val = method(display_override.to_s).call
|
86
|
+
else
|
87
|
+
val = instance_variable_get("@#{prop}")
|
88
|
+
if val.is_a?(Time)
|
89
|
+
val = display_as_time(val)
|
90
|
+
else
|
91
|
+
val = val
|
92
|
+
end
|
93
|
+
end
|
94
|
+
result << [prop, sanitize(val)]
|
95
|
+
}
|
96
|
+
{ rows: rows }
|
97
|
+
else
|
98
|
+
nil
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def display_as_array(columns, ellipsis = false)
|
103
|
+
row = columns.inject([]) { |cols, head|
|
104
|
+
if ellipsis
|
105
|
+
val = "..."
|
106
|
+
else
|
107
|
+
display_override = "display_#{head.to_s}".to_sym
|
108
|
+
if respond_to?(display_override)
|
109
|
+
val = method(display_override.to_s).call
|
110
|
+
else
|
111
|
+
val = instance_variable_get("@#{head}")
|
112
|
+
val = display_as_time(val) if val.is_a?(Time)
|
113
|
+
end
|
114
|
+
end
|
115
|
+
cols << sanitize(val, 80)
|
116
|
+
}
|
117
|
+
end
|
118
|
+
|
119
|
+
def child_columns
|
120
|
+
first_child = children.first
|
121
|
+
if first_child
|
122
|
+
first_child.class.default_columns
|
123
|
+
else
|
124
|
+
nil
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
def children_title
|
129
|
+
"#{self.class.name.split('::').last} Children"
|
130
|
+
end
|
131
|
+
|
132
|
+
def display_children
|
133
|
+
if children.nil? || children.length == 0
|
134
|
+
nil
|
135
|
+
else
|
136
|
+
headings = child_columns.map { |head|
|
137
|
+
head.to_s.split('_').map { |w| w.capitalize }.join(' ')
|
138
|
+
}
|
139
|
+
children.each_with_index { |c, i|
|
140
|
+
c.set_index(i)
|
141
|
+
}
|
142
|
+
|
143
|
+
truncated = children.length > session.len
|
144
|
+
childrows = truncated ? children[0..session.len-1] : children
|
145
|
+
|
146
|
+
rows = childrows.inject([]) { |result, v|
|
147
|
+
result << v.display_as_array(child_columns)
|
148
|
+
result
|
149
|
+
}
|
150
|
+
|
151
|
+
rows[rows.length-1][0] = "#{rows[rows.length-1][0]}+" if truncated
|
152
|
+
{ headings: headings, rows: rows }
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
class Configuration < ::Ox::Sax
|
157
|
+
attr_reader :map
|
158
|
+
|
159
|
+
def initialize
|
160
|
+
@map = {}
|
161
|
+
end
|
162
|
+
|
163
|
+
def start_element(key)
|
164
|
+
@incoming = key
|
165
|
+
end
|
166
|
+
|
167
|
+
def text(incoming)
|
168
|
+
case @incoming
|
169
|
+
when :name;
|
170
|
+
@key = incoming
|
171
|
+
when :value;
|
172
|
+
@map[@key] = incoming
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
def conf_map
|
178
|
+
io = StringIO.new(conf || "")
|
179
|
+
handler = Configuration.new()
|
180
|
+
Ox.sax_parse(handler, io)
|
181
|
+
handler.map
|
182
|
+
end
|
183
|
+
|
184
|
+
def log
|
185
|
+
session.get_job_state(id, "show=log")
|
186
|
+
end
|
187
|
+
|
188
|
+
def definition
|
189
|
+
session.get_job_state(id, "show=definition")
|
190
|
+
end
|
191
|
+
end
|
192
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module Hodor::Oozie
|
2
|
+
|
3
|
+
class Materialization < Job
|
4
|
+
|
5
|
+
attr_reader :json, :error_message, :last_modified_time, :created_at, :status, :push_missing_dependencies,
|
6
|
+
:external_status, :type, :nominal_time, :external_id, :created_conf, :missing_dependencies,
|
7
|
+
:run_conf, :action_number, :error_code, :tracker_uri, :to_string, :parent_id, :coord_job_id,
|
8
|
+
:console_url
|
9
|
+
|
10
|
+
class << self
|
11
|
+
def default_properties
|
12
|
+
[ :error_message, :last_modified_time, :created_at, :status, :push_missing_dependencies,
|
13
|
+
:external_status, :type, :nominal_time, :external_id, :created_conf, :missing_dependencies,
|
14
|
+
:run_conf, :action_number, :error_code, :tracker_uri, :to_string, :parent_id, :coord_job_id,
|
15
|
+
:console_url]
|
16
|
+
end
|
17
|
+
|
18
|
+
def default_columns
|
19
|
+
[:index, :id, :status, :external_id, :type, :created_at, :nominal_time, :last_modified]
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def initialize(json)
|
24
|
+
super()
|
25
|
+
@json = json
|
26
|
+
@error_message = json["errorMessage"]
|
27
|
+
@last_modified = @last_modified_time = parse_time(json["lastModifiedTime"])
|
28
|
+
@created_at = parse_time(json["createdTime"])
|
29
|
+
@status = json["status"]
|
30
|
+
@push_missing_dependencies = json["pushMissingDependencies"]
|
31
|
+
@external_status = json["externalStatus"]
|
32
|
+
@type = json["type"]
|
33
|
+
@nominal_time = parse_time(json["nominalTime"])
|
34
|
+
@external_id = json["externalId"]
|
35
|
+
@id = json["id"]
|
36
|
+
@created_conf = json["createdConf"]
|
37
|
+
@missing_dependencies = json["missingDependencies"]
|
38
|
+
@run_conf = json["runConf"]
|
39
|
+
@action_number = json["actionNumber"]
|
40
|
+
@error_code = json["errorCode"]
|
41
|
+
@tracker_uri = json["trackerUri"]
|
42
|
+
@to_string = json["toString"]
|
43
|
+
@parent_id = @coord_job_id = json["coordJobId"]
|
44
|
+
@console_url = json["consoleUrl"]
|
45
|
+
end
|
46
|
+
|
47
|
+
def expand
|
48
|
+
[ oozie.job_by_id(@external_id) ]
|
49
|
+
end
|
50
|
+
|
51
|
+
def display_id
|
52
|
+
@id[@id.rindex('C@')..-1]
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
@@ -0,0 +1,115 @@
|
|
1
|
+
module Hodor::Oozie
|
2
|
+
|
3
|
+
class Query < Job
|
4
|
+
attr_reader :request, :json, :jobs, :filter
|
5
|
+
|
6
|
+
def session
|
7
|
+
Hodor::Oozie::Session.instance
|
8
|
+
end
|
9
|
+
|
10
|
+
class << self
|
11
|
+
def suppress_properties
|
12
|
+
true
|
13
|
+
end
|
14
|
+
def default_columns
|
15
|
+
[:index, :name, :status, :id, :start_time, :time_unit, :external_id]
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def initialize(filter = {} )
|
20
|
+
@filter = filter
|
21
|
+
@jobtype = filter[:jobtype] || "coord"
|
22
|
+
@json = run_query(filter)
|
23
|
+
@request = session.last_query
|
24
|
+
end
|
25
|
+
|
26
|
+
def run_query(filter)
|
27
|
+
params = []
|
28
|
+
if filter.has_key?(:user)
|
29
|
+
params << "user%3D#{filter[:user]}"
|
30
|
+
end
|
31
|
+
|
32
|
+
if filter.has_key?(:status)
|
33
|
+
stats = filter[:status]
|
34
|
+
params += stats.map { |val|
|
35
|
+
case val
|
36
|
+
when :running_first;
|
37
|
+
@running_first = true
|
38
|
+
"status%3DRUNNING"
|
39
|
+
when :running; "status%3DRUNNING"
|
40
|
+
when :killed; "status%3DKILLED"
|
41
|
+
when :suspended; "status%3DSUSPENDED"
|
42
|
+
when :timedout; "status%3DTIMEDOUT"
|
43
|
+
when :failed; "status%3DFAILED"
|
44
|
+
when :succeeded; "status%3DSUCCEEDED"
|
45
|
+
end
|
46
|
+
}
|
47
|
+
end
|
48
|
+
|
49
|
+
if params.size > 0
|
50
|
+
filter_exp = "filter=#{params.join(';')}"
|
51
|
+
else
|
52
|
+
filter_exp = ""
|
53
|
+
end
|
54
|
+
|
55
|
+
pagination = []
|
56
|
+
pagination << "offset=#{session.offset}"
|
57
|
+
pagination << "len=#{session.len.to_i+1}"
|
58
|
+
pagination = pagination.join('&')
|
59
|
+
|
60
|
+
response = session.search_jobs("jobtype=#{@jobtype}", filter_exp, pagination)
|
61
|
+
@json = JSON.parse(response)
|
62
|
+
end
|
63
|
+
|
64
|
+
def title
|
65
|
+
if @jobtype.start_with?('coord')
|
66
|
+
"List of Coordinators"
|
67
|
+
elsif @jobtype.start_with?('w')
|
68
|
+
"List of Workflows"
|
69
|
+
elsif @jobtype.start_with?('b')
|
70
|
+
"List of Bundles"
|
71
|
+
else
|
72
|
+
"List of Matching Jobs"
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def expand
|
77
|
+
# expand immediate children
|
78
|
+
if @json.has_key?("workflows")
|
79
|
+
all_jobs = @json["workflows"].map do |item|
|
80
|
+
Hodor::Oozie::Workflow.new(item)
|
81
|
+
end.compact
|
82
|
+
if @running_first
|
83
|
+
more_json = run_query( { status: [:killed, :succeeded, :failed, :suspended] } )
|
84
|
+
if more_json.has_key?("workflows")
|
85
|
+
all_jobs += more_json["workflows"].map do |item|
|
86
|
+
Hodor::Oozie::Workflow.new(item)
|
87
|
+
end.compact
|
88
|
+
end
|
89
|
+
end
|
90
|
+
elsif @json.has_key?("coordinatorjobs")
|
91
|
+
all_jobs = @json["coordinatorjobs"].map do |item|
|
92
|
+
Hodor::Oozie::Coordinator.new(item)
|
93
|
+
end.compact
|
94
|
+
if @running_first
|
95
|
+
more_json = run_query( { status: [:succeeded, :killed, :failed, :suspended] } )
|
96
|
+
if more_json.has_key?("coordinatorjobs")
|
97
|
+
all_jobs += more_json["coordinatorjobs"].map do |item|
|
98
|
+
Hodor::Oozie::Coordinator.new(item)
|
99
|
+
end.compact
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
if @filter[:match]
|
104
|
+
pattern = @filter[:match]
|
105
|
+
@jobs = all_jobs.select { | job|
|
106
|
+
job.name.include?(pattern)
|
107
|
+
}
|
108
|
+
else
|
109
|
+
@jobs = all_jobs
|
110
|
+
end
|
111
|
+
@jobs
|
112
|
+
end
|
113
|
+
|
114
|
+
end
|
115
|
+
end
|