mortar 0.15.36 → 0.15.37
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/mortar/command/base.rb +2 -2
- data/lib/mortar/command/config.rb +52 -11
- data/lib/mortar/command/jobs.rb +1 -1
- data/lib/mortar/command/local.rb +1 -136
- data/lib/mortar/command/spark.rb +26 -57
- data/lib/mortar/local/controller.rb +2 -31
- data/lib/mortar/local/pig.rb +1 -1
- data/lib/mortar/pigversion.rb +52 -43
- data/lib/mortar/version.rb +1 -1
- data/spec/mortar/command/base_spec.rb +2 -2
- data/spec/mortar/command/config_spec.rb +63 -0
- data/spec/mortar/command/describe_spec.rb +4 -4
- data/spec/mortar/command/illustrate_spec.rb +7 -7
- data/spec/mortar/command/jobs_spec.rb +61 -13
- data/spec/mortar/command/local_spec.rb +9 -234
- data/spec/mortar/command/validate_spec.rb +2 -2
- data/spec/mortar/local/controller_spec.rb +7 -41
- data/spec/mortar/local/pig_spec.rb +29 -30
- metadata +5 -8
- data/lib/mortar/local/sqoop.rb +0 -162
- data/lib/mortar/templates/script/sqoop.sh +0 -40
- data/spec/mortar/local/sqoop_spec.rb +0 -57
data/lib/mortar/local/pig.rb
CHANGED
@@ -323,7 +323,7 @@ class Mortar::Local::Pig
|
|
323
323
|
def template_params_classpath(pig_version=nil)
|
324
324
|
# Need to support old watchtower plugins that don't set pig_version
|
325
325
|
if pig_version.nil?
|
326
|
-
pig_version = Mortar::PigVersion::
|
326
|
+
pig_version = Mortar::PigVersion::Pig012.new
|
327
327
|
end
|
328
328
|
[ "#{pig_directory(pig_version)}/*",
|
329
329
|
"#{pig_directory(pig_version)}/lib-local/*",
|
data/lib/mortar/pigversion.rb
CHANGED
@@ -15,57 +15,66 @@
|
|
15
15
|
#
|
16
16
|
|
17
17
|
module Mortar
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
PIG_0_12_TGZ_NAME = "pig-0.12.tar.gz"
|
22
|
-
PIG_0_12_TGZ_DEFAULT_URL_PATH = "resource/pig_0_12"
|
18
|
+
module PigVersion
|
19
|
+
PIG_0_12_TGZ_NAME = "pig-0.12.tar.gz"
|
20
|
+
PIG_0_12_TGZ_DEFAULT_URL_PATH = "resource/pig_0_12"
|
23
21
|
|
24
|
-
|
25
|
-
|
26
|
-
return Pig09.new
|
27
|
-
elsif pig_version_str == '0.12'
|
28
|
-
return Pig012.new
|
29
|
-
else
|
30
|
-
raise ArgumentError, "Unsupported pig version: #{pig_version_str}"
|
31
|
-
end
|
32
|
-
end
|
22
|
+
PIG_0_12_HADOOP_2_TGZ_NAME = "pig-0.12-hadoop-2.tar.gz"
|
23
|
+
PIG_0_12_HADOOP_2_TGZ_DEFAULT_URL_PATH = "resource/pig_0_12_hadoop_2"
|
33
24
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
25
|
+
def PigVersion.from_string(pig_version_str)
|
26
|
+
if pig_version_str == '0.12'
|
27
|
+
return Pig012.new
|
28
|
+
elsif pig_version_str == '0.12-Hadoop-2'
|
29
|
+
return Pig012Hadoop2.new
|
30
|
+
else
|
31
|
+
raise ArgumentError, "Unsupported pig version: #{pig_version_str}. Options are: ['0.12', '0.12-Hadoop-2]."
|
32
|
+
end
|
33
|
+
end
|
38
34
|
|
39
|
-
|
40
|
-
|
41
|
-
|
35
|
+
class Pig012
|
36
|
+
def tgz_name
|
37
|
+
PIG_0_12_TGZ_NAME
|
38
|
+
end
|
42
39
|
|
43
|
-
|
44
|
-
|
45
|
-
|
40
|
+
def tgz_default_url_path
|
41
|
+
PIG_0_12_TGZ_DEFAULT_URL_PATH
|
42
|
+
end
|
46
43
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
end
|
44
|
+
def name
|
45
|
+
"pig-#{version}"
|
46
|
+
end
|
51
47
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
end
|
48
|
+
def version
|
49
|
+
"0.12"
|
50
|
+
end
|
56
51
|
|
57
|
-
|
58
|
-
|
59
|
-
|
52
|
+
def cluster_backend
|
53
|
+
Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1
|
54
|
+
end
|
55
|
+
end
|
60
56
|
|
61
|
-
|
62
|
-
|
63
|
-
|
57
|
+
class Pig012Hadoop2
|
58
|
+
def tgz_name
|
59
|
+
PIG_0_12_HADOOP_2_TGZ_NAME
|
60
|
+
end
|
64
61
|
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
end
|
62
|
+
def tgz_default_url_path
|
63
|
+
PIG_0_12_HADOOP_2_TGZ_DEFAULT_URL_PATH
|
64
|
+
end
|
69
65
|
|
70
|
-
|
66
|
+
def name
|
67
|
+
"pig-#{version}"
|
68
|
+
end
|
69
|
+
|
70
|
+
def version
|
71
|
+
"0.12-Hadoop-2"
|
72
|
+
end
|
73
|
+
|
74
|
+
def cluster_backend
|
75
|
+
Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_2
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
end
|
71
80
|
end
|
data/lib/mortar/version.rb
CHANGED
@@ -106,7 +106,7 @@ other\tgit@github.com:other.git (push)
|
|
106
106
|
context "method_added" do
|
107
107
|
it "replaces help templates" do
|
108
108
|
lines = Base.replace_templates(["line", "start <PIG_VERSION_OPTIONS>"])
|
109
|
-
lines.join("").should == 'linestart 0.
|
109
|
+
lines.join("").should == 'linestart 0.12 and 0.12-Hadoop-2'
|
110
110
|
end
|
111
111
|
end
|
112
112
|
|
@@ -232,7 +232,7 @@ polling_interval=10
|
|
232
232
|
describe_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
|
233
233
|
describe_url = "https://mdog.datadoghq.com/describe/#{describe_id}"
|
234
234
|
|
235
|
-
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.
|
235
|
+
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters=>[]) {Excon::Response.new(:body => {"describe_id" => describe_id})}
|
236
236
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_SUCCESS, "status_description" => "Success", "web_result_url" => describe_url})).ordered
|
237
237
|
write_file(File.join(p.pigscripts_path, "my_script.pig"))
|
238
238
|
|
@@ -113,6 +113,41 @@ B=two three
|
|
113
113
|
STDOUT
|
114
114
|
end
|
115
115
|
end
|
116
|
+
|
117
|
+
it "shows shadowed config vars" do
|
118
|
+
with_git_initialized_project do |p|
|
119
|
+
# stup api request
|
120
|
+
config = {'A' => 'one', 'B' => 'two'}
|
121
|
+
shadow_config = {'B' => 'three'}
|
122
|
+
mock(Mortar::Auth.api).get_config_vars("myproject").returns(Excon::Response.new(:body => {"config" => config,"shadow_config" => shadow_config}))
|
123
|
+
|
124
|
+
stderr, stdout = execute("config", p, @git)
|
125
|
+
stdout.should == <<-STDOUT
|
126
|
+
=== myproject Config Vars
|
127
|
+
A: one
|
128
|
+
B: two
|
129
|
+
|
130
|
+
=== Project config settings overriden by user config
|
131
|
+
B: three
|
132
|
+
STDOUT
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
it "doesn't show shadowed config vars on shell output" do
|
137
|
+
with_git_initialized_project do |p|
|
138
|
+
# stup api request
|
139
|
+
config = {'A' => 'one', 'B' => 'two'}
|
140
|
+
shadow_config = {'B' => 'three'}
|
141
|
+
mock(Mortar::Auth.api).get_config_vars("myproject").returns(Excon::Response.new(:body => {"config" => config,"shadow_config" => shadow_config}))
|
142
|
+
|
143
|
+
stderr, stdout = execute("config --shell", p, @git)
|
144
|
+
stdout.should == <<-STDOUT
|
145
|
+
A=one
|
146
|
+
B=two
|
147
|
+
STDOUT
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
116
151
|
end
|
117
152
|
|
118
153
|
context("get") do
|
@@ -187,6 +222,21 @@ STDOUT
|
|
187
222
|
stdout.should == <<-STDOUT
|
188
223
|
Setting config vars for project myproject... done
|
189
224
|
a: b
|
225
|
+
STDOUT
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
it "sets config vars for user" do
|
230
|
+
with_git_initialized_project do |p|
|
231
|
+
# stub api request
|
232
|
+
configs = {'a' => 'b'}
|
233
|
+
mock(Mortar::Auth.api).put_user_config_vars(configs).returns(Excon::Response.new(:body => {}))
|
234
|
+
|
235
|
+
stderr, stdout = execute("config:set a=b -u", p, @git)
|
236
|
+
stderr.should == ""
|
237
|
+
stdout.should == <<-STDOUT
|
238
|
+
Setting config vars for your user... done
|
239
|
+
a: b
|
190
240
|
STDOUT
|
191
241
|
end
|
192
242
|
end
|
@@ -214,6 +264,19 @@ STDERR
|
|
214
264
|
stderr.should == ""
|
215
265
|
stdout.should == <<-STDOUT
|
216
266
|
Unsetting A for project myproject... done
|
267
|
+
STDOUT
|
268
|
+
end
|
269
|
+
end
|
270
|
+
|
271
|
+
it "unsets one key for user" do
|
272
|
+
with_git_initialized_project do |p|
|
273
|
+
# stub api requests
|
274
|
+
mock(Mortar::Auth.api).delete_user_config_var("A").returns(Excon::Response.new(:body => {}))
|
275
|
+
|
276
|
+
stderr, stdout = execute("config:unset A --user", p, @git)
|
277
|
+
stderr.should == ""
|
278
|
+
stdout.should == <<-STDOUT
|
279
|
+
Unsetting A for your user... done
|
217
280
|
STDOUT
|
218
281
|
end
|
219
282
|
end
|
@@ -98,7 +98,7 @@ STDERR
|
|
98
98
|
describe_url = "https://mdog.datadoghq.com/describe/#{describe_id}"
|
99
99
|
parameters = ["name"=>"key", "value"=>"value" ]
|
100
100
|
|
101
|
-
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.
|
101
|
+
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"describe_id" => describe_id})}
|
102
102
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
103
103
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_GATEWAY_STARTING, "status_description" => "Gateway starting"})).ordered
|
104
104
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_PROGRESS, "status_description" => "Starting pig"})).ordered
|
@@ -129,7 +129,7 @@ STDOUT
|
|
129
129
|
describe_url = "https://mdog.datadoghq.com/describe/#{describe_id}"
|
130
130
|
parameters = ["name"=>"key", "value"=>"value" ]
|
131
131
|
|
132
|
-
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.
|
132
|
+
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"describe_id" => describe_id})}
|
133
133
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
134
134
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_GATEWAY_STARTING, "status_description" => "Gateway starting"})).ordered
|
135
135
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_PROGRESS, "status_description" => "Starting pig"})).ordered
|
@@ -163,7 +163,7 @@ STDOUT
|
|
163
163
|
column_number = 32
|
164
164
|
error_type = 'PigError'
|
165
165
|
|
166
|
-
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.
|
166
|
+
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => []) {Excon::Response.new(:body => {"describe_id" => describe_id})}
|
167
167
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
168
168
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_FAILURE, "status_description" => "Failed",
|
169
169
|
"error_message" => error_message,
|
@@ -198,7 +198,7 @@ STDERR
|
|
198
198
|
describe_url = "https://mdog.datadoghq.com/describe/#{describe_id}"
|
199
199
|
parameters = ["name"=>"key", "value"=>"value" ]
|
200
200
|
|
201
|
-
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.
|
201
|
+
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"describe_id" => describe_id})}
|
202
202
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
203
203
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_GATEWAY_STARTING, "status_description" => "Gateway starting"})).ordered
|
204
204
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_PROGRESS, "status_description" => "Starting pig"})).ordered
|
@@ -74,7 +74,7 @@ STDERR
|
|
74
74
|
parameters = ["name"=>"key", "value"=>"value" ]
|
75
75
|
|
76
76
|
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
77
|
-
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.
|
77
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
78
78
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
79
79
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_GATEWAY_STARTING, "status_description" => "GATEWAY_STARTING"})).ordered
|
80
80
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_PROGRESS, "status_description" => "In progress"})).ordered
|
@@ -108,7 +108,7 @@ STDOUT
|
|
108
108
|
parameters = ["name"=>"key", "value"=>"value" ]
|
109
109
|
|
110
110
|
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
111
|
-
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.
|
111
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
112
112
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
113
113
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_GATEWAY_STARTING, "status_description" => "GATEWAY_STARTING"})).ordered
|
114
114
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_PROGRESS, "status_description" => "In progress"})).ordered
|
@@ -142,7 +142,7 @@ STDOUT
|
|
142
142
|
parameters = ["name"=>"key", "value"=>"value" ]
|
143
143
|
|
144
144
|
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
145
|
-
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.
|
145
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
146
146
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
147
147
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_SUCCESS, "status_description" => "Succeeded", "web_result_url" => illustrate_url})).ordered
|
148
148
|
|
@@ -169,7 +169,7 @@ STDOUT
|
|
169
169
|
parameters = ["name"=>"key", "value"=>"value" ]
|
170
170
|
|
171
171
|
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
172
|
-
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", true, is_a(String), :pig_version => "0.
|
172
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", true, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
173
173
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
174
174
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_SUCCESS, "status_description" => "Succeeded", "web_result_url" => illustrate_url})).ordered
|
175
175
|
|
@@ -195,7 +195,7 @@ STDOUT
|
|
195
195
|
parameters = ["name"=>"key", "value"=>"value" ]
|
196
196
|
|
197
197
|
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
198
|
-
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", nil, false, is_a(String), :pig_version => "0.
|
198
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", nil, false, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
199
199
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
200
200
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_GATEWAY_STARTING, "status_description" => "GATEWAY_STARTING"})).ordered
|
201
201
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_PROGRESS, "status_description" => "In progress"})).ordered
|
@@ -232,7 +232,7 @@ STDOUT
|
|
232
232
|
error_type = 'PigError'
|
233
233
|
|
234
234
|
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
235
|
-
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.
|
235
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => []) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
236
236
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
237
237
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_FAILURE,
|
238
238
|
"error_message" => error_message,
|
@@ -268,7 +268,7 @@ STDERR
|
|
268
268
|
parameters = ["name"=>"key", "value"=>"value" ]
|
269
269
|
|
270
270
|
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
271
|
-
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", nil, false, is_a(String), :pig_version => "0.
|
271
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", nil, false, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
272
272
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
273
273
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_GATEWAY_STARTING, "status_description" => "GATEWAY_STARTING"})).ordered
|
274
274
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_PROGRESS, "status_description" => "In progress"})).ordered
|
@@ -47,7 +47,7 @@ module Mortar::Command
|
|
47
47
|
cluster_size = 5
|
48
48
|
|
49
49
|
mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
|
50
|
-
:pig_version => "0.
|
50
|
+
:pig_version => "0.12",
|
51
51
|
:project_script_path => be_a_kind_of(String),
|
52
52
|
:parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
|
53
53
|
:cluster_type => Jobs::CLUSTER_TYPE__SINGLE_JOB,
|
@@ -83,7 +83,7 @@ STDOUT
|
|
83
83
|
cluster_size = 5
|
84
84
|
|
85
85
|
mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String),cluster_size,
|
86
|
-
:pig_version => "0.
|
86
|
+
:pig_version => "0.12",
|
87
87
|
:project_script_path => be_a_kind_of(String),
|
88
88
|
:parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
|
89
89
|
:cluster_type => Jobs::CLUSTER_TYPE__PERMANENT,
|
@@ -143,7 +143,7 @@ STDERR
|
|
143
143
|
cluster_size = 5
|
144
144
|
|
145
145
|
mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String),cluster_size,
|
146
|
-
:pig_version => "0.
|
146
|
+
:pig_version => "0.12",
|
147
147
|
:project_script_path => be_a_kind_of(String),
|
148
148
|
:parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
|
149
149
|
:cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
|
@@ -179,7 +179,7 @@ STDOUT
|
|
179
179
|
cluster_size = 5
|
180
180
|
|
181
181
|
mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String),cluster_size,
|
182
|
-
:pig_version => "0.
|
182
|
+
:pig_version => "0.12",
|
183
183
|
:project_script_path => be_a_kind_of(String),
|
184
184
|
:parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
|
185
185
|
:cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
|
@@ -215,7 +215,7 @@ STDOUT
|
|
215
215
|
cluster_id = "e2790e7e8c7d48e39157238d58191346"
|
216
216
|
|
217
217
|
mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id,
|
218
|
-
:pig_version => "0.
|
218
|
+
:pig_version => "0.12",
|
219
219
|
:project_script_path => be_a_kind_of(String),
|
220
220
|
:parameters => [],
|
221
221
|
:notify_on_job_finish => false,
|
@@ -249,7 +249,7 @@ STDOUT
|
|
249
249
|
cluster_id = "e2790e7e8c7d48e39157238d58191346"
|
250
250
|
|
251
251
|
mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id,
|
252
|
-
:pig_version => "0.
|
252
|
+
:pig_version => "0.12",
|
253
253
|
:project_script_path => be_a_kind_of(String),
|
254
254
|
:parameters => [],
|
255
255
|
:notify_on_job_finish => false,
|
@@ -283,7 +283,7 @@ STDOUT
|
|
283
283
|
|
284
284
|
mock(Mortar::Auth.api).get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1) {Excon::Response.new(:body => {'clusters' => []})}
|
285
285
|
mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
|
286
|
-
:pig_version => "0.
|
286
|
+
:pig_version => "0.12",
|
287
287
|
:project_script_path => be_a_kind_of(String),
|
288
288
|
:parameters => [],
|
289
289
|
:cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
|
@@ -321,7 +321,7 @@ STDOUT
|
|
321
321
|
|
322
322
|
mock(Mortar::Auth.api).get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1) {Excon::Response.new(:body => {'clusters' => []})}
|
323
323
|
mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
|
324
|
-
:pig_version => "0.
|
324
|
+
:pig_version => "0.12",
|
325
325
|
:project_script_path => be_a_kind_of(String),
|
326
326
|
:parameters => [],
|
327
327
|
:cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
|
@@ -358,7 +358,7 @@ STDOUT
|
|
358
358
|
job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
|
359
359
|
cluster_id = "e2790e7e8c7d48e39157238d58191346"
|
360
360
|
|
361
|
-
mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id, :pig_version => "0.
|
361
|
+
mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id, :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => [], :notify_on_job_finish => false, :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
|
362
362
|
|
363
363
|
write_file(File.join(p.pigscripts_path, "my_script.pig"))
|
364
364
|
stderr, stdout = execute("jobs:run pigscripts/my_script.pig --clusterid e2790e7e8c7d48e39157238d58191346 -d", p, @git)
|
@@ -411,7 +411,7 @@ STDOUT
|
|
411
411
|
]})
|
412
412
|
}
|
413
413
|
mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), large_cluster_id,
|
414
|
-
:pig_version => "0.
|
414
|
+
:pig_version => "0.12",
|
415
415
|
:project_script_path => be_a_kind_of(String),
|
416
416
|
:parameters => [],
|
417
417
|
:notify_on_job_finish => true,
|
@@ -426,6 +426,54 @@ Sending code snapshot to Mortar... done
|
|
426
426
|
Requesting job execution... done
|
427
427
|
job_id: c571a8c7f76a4fd4a67c103d753e2dd5
|
428
428
|
|
429
|
+
Job status can be viewed on the web at:
|
430
|
+
|
431
|
+
http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
|
432
|
+
|
433
|
+
Or by running:
|
434
|
+
|
435
|
+
mortar jobs:status c571a8c7f76a4fd4a67c103d753e2dd5 --poll
|
436
|
+
|
437
|
+
STDOUT
|
438
|
+
|
439
|
+
end
|
440
|
+
end
|
441
|
+
|
442
|
+
it "runs a job by default on the largest existing running cluster for Hadoop 2" do
|
443
|
+
with_git_initialized_project do |p|
|
444
|
+
job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
|
445
|
+
job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
|
446
|
+
|
447
|
+
small_h2_cluster_id = '510beb6b3004860820ab6538'
|
448
|
+
small_h2_cluster_size = 2
|
449
|
+
small_h2_cluster_status = Mortar::API::Clusters::STATUS_RUNNING
|
450
|
+
large_h2_cluster_id = '510bf0db3004860820ab6590'
|
451
|
+
large_h2_cluster_size = 5
|
452
|
+
large_h2_cluster_status = Mortar::API::Clusters::STATUS_RUNNING
|
453
|
+
|
454
|
+
mock(Mortar::Auth.api).get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_2) {
|
455
|
+
Excon::Response.new(:body => {
|
456
|
+
'clusters' => [
|
457
|
+
{ 'cluster_id' => small_h2_cluster_id, 'size' => small_h2_cluster_size, 'running_jobs' => [], 'status_code' => small_h2_cluster_status },
|
458
|
+
{ 'cluster_id' => large_h2_cluster_id, 'size' => large_h2_cluster_size, 'running_jobs' => [], 'status_code' => large_h2_cluster_status },
|
459
|
+
]})
|
460
|
+
}
|
461
|
+
mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), large_h2_cluster_id,
|
462
|
+
:pig_version => "0.12-Hadoop-2",
|
463
|
+
:project_script_path => be_a_kind_of(String),
|
464
|
+
:parameters => [],
|
465
|
+
:notify_on_job_finish => true,
|
466
|
+
:is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
|
467
|
+
|
468
|
+
write_file(File.join(p.pigscripts_path, "my_script.pig"))
|
469
|
+
stderr, stdout = execute("jobs:run pigscripts/my_script.pig -g 0.12-Hadoop-2", p, @git)
|
470
|
+
stdout.should == <<-STDOUT
|
471
|
+
Defaulting to running job on largest existing free cluster, id = 510bf0db3004860820ab6590, size = 5
|
472
|
+
Taking code snapshot... done
|
473
|
+
Sending code snapshot to Mortar... done
|
474
|
+
Requesting job execution... done
|
475
|
+
job_id: c571a8c7f76a4fd4a67c103d753e2dd5
|
476
|
+
|
429
477
|
Job status can be viewed on the web at:
|
430
478
|
|
431
479
|
http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
|
@@ -445,7 +493,7 @@ STDOUT
|
|
445
493
|
cluster_size = 5
|
446
494
|
|
447
495
|
mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
|
448
|
-
:pig_version => "0.
|
496
|
+
:pig_version => "0.12",
|
449
497
|
:project_script_path => be_a_kind_of(String),
|
450
498
|
:parameters => match_array([{"name" => "FIRST", "value" => "FOO"}, {"name" => "SECOND", "value" => "BAR"}, {"name" => "THIRD", "value" => "BEAR"}]),
|
451
499
|
:cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
|
@@ -473,7 +521,7 @@ PARAMS
|
|
473
521
|
cluster_size = 5
|
474
522
|
|
475
523
|
mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
|
476
|
-
:pig_version => "0.
|
524
|
+
:pig_version => "0.12",
|
477
525
|
:project_script_path => be_a_kind_of(String),
|
478
526
|
:parameters => match_array([{"name" => "FIRST", "value" => "FOO"}, {"name" => "SECOND", "value" => "BAR"}, {"name" => "THIRD", "value" => "BEAR"}]),
|
479
527
|
:cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
|
@@ -529,7 +577,7 @@ STDERR
|
|
529
577
|
mock(@git).sync_embedded_project.with_any_args.times(1) { "somewhere_over_the_rainbow" }
|
530
578
|
|
531
579
|
mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
|
532
|
-
:pig_version => "0.
|
580
|
+
:pig_version => "0.12",
|
533
581
|
:project_script_path => be_a_kind_of(String),
|
534
582
|
:parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
|
535
583
|
:cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
|