mortar 0.15.36 → 0.15.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/mortar/command/base.rb +2 -2
- data/lib/mortar/command/config.rb +52 -11
- data/lib/mortar/command/jobs.rb +1 -1
- data/lib/mortar/command/local.rb +1 -136
- data/lib/mortar/command/spark.rb +26 -57
- data/lib/mortar/local/controller.rb +2 -31
- data/lib/mortar/local/pig.rb +1 -1
- data/lib/mortar/pigversion.rb +52 -43
- data/lib/mortar/version.rb +1 -1
- data/spec/mortar/command/base_spec.rb +2 -2
- data/spec/mortar/command/config_spec.rb +63 -0
- data/spec/mortar/command/describe_spec.rb +4 -4
- data/spec/mortar/command/illustrate_spec.rb +7 -7
- data/spec/mortar/command/jobs_spec.rb +61 -13
- data/spec/mortar/command/local_spec.rb +9 -234
- data/spec/mortar/command/validate_spec.rb +2 -2
- data/spec/mortar/local/controller_spec.rb +7 -41
- data/spec/mortar/local/pig_spec.rb +29 -30
- metadata +5 -8
- data/lib/mortar/local/sqoop.rb +0 -162
- data/lib/mortar/templates/script/sqoop.sh +0 -40
- data/spec/mortar/local/sqoop_spec.rb +0 -57
data/lib/mortar/local/pig.rb
CHANGED
@@ -323,7 +323,7 @@ class Mortar::Local::Pig
|
|
323
323
|
def template_params_classpath(pig_version=nil)
|
324
324
|
# Need to support old watchtower plugins that don't set pig_version
|
325
325
|
if pig_version.nil?
|
326
|
-
pig_version = Mortar::PigVersion::
|
326
|
+
pig_version = Mortar::PigVersion::Pig012.new
|
327
327
|
end
|
328
328
|
[ "#{pig_directory(pig_version)}/*",
|
329
329
|
"#{pig_directory(pig_version)}/lib-local/*",
|
data/lib/mortar/pigversion.rb
CHANGED
@@ -15,57 +15,66 @@
|
|
15
15
|
#
|
16
16
|
|
17
17
|
module Mortar
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
PIG_0_12_TGZ_NAME = "pig-0.12.tar.gz"
|
22
|
-
PIG_0_12_TGZ_DEFAULT_URL_PATH = "resource/pig_0_12"
|
18
|
+
module PigVersion
|
19
|
+
PIG_0_12_TGZ_NAME = "pig-0.12.tar.gz"
|
20
|
+
PIG_0_12_TGZ_DEFAULT_URL_PATH = "resource/pig_0_12"
|
23
21
|
|
24
|
-
|
25
|
-
|
26
|
-
return Pig09.new
|
27
|
-
elsif pig_version_str == '0.12'
|
28
|
-
return Pig012.new
|
29
|
-
else
|
30
|
-
raise ArgumentError, "Unsupported pig version: #{pig_version_str}"
|
31
|
-
end
|
32
|
-
end
|
22
|
+
PIG_0_12_HADOOP_2_TGZ_NAME = "pig-0.12-hadoop-2.tar.gz"
|
23
|
+
PIG_0_12_HADOOP_2_TGZ_DEFAULT_URL_PATH = "resource/pig_0_12_hadoop_2"
|
33
24
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
25
|
+
def PigVersion.from_string(pig_version_str)
|
26
|
+
if pig_version_str == '0.12'
|
27
|
+
return Pig012.new
|
28
|
+
elsif pig_version_str == '0.12-Hadoop-2'
|
29
|
+
return Pig012Hadoop2.new
|
30
|
+
else
|
31
|
+
raise ArgumentError, "Unsupported pig version: #{pig_version_str}. Options are: ['0.12', '0.12-Hadoop-2]."
|
32
|
+
end
|
33
|
+
end
|
38
34
|
|
39
|
-
|
40
|
-
|
41
|
-
|
35
|
+
class Pig012
|
36
|
+
def tgz_name
|
37
|
+
PIG_0_12_TGZ_NAME
|
38
|
+
end
|
42
39
|
|
43
|
-
|
44
|
-
|
45
|
-
|
40
|
+
def tgz_default_url_path
|
41
|
+
PIG_0_12_TGZ_DEFAULT_URL_PATH
|
42
|
+
end
|
46
43
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
end
|
44
|
+
def name
|
45
|
+
"pig-#{version}"
|
46
|
+
end
|
51
47
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
end
|
48
|
+
def version
|
49
|
+
"0.12"
|
50
|
+
end
|
56
51
|
|
57
|
-
|
58
|
-
|
59
|
-
|
52
|
+
def cluster_backend
|
53
|
+
Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1
|
54
|
+
end
|
55
|
+
end
|
60
56
|
|
61
|
-
|
62
|
-
|
63
|
-
|
57
|
+
class Pig012Hadoop2
|
58
|
+
def tgz_name
|
59
|
+
PIG_0_12_HADOOP_2_TGZ_NAME
|
60
|
+
end
|
64
61
|
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
end
|
62
|
+
def tgz_default_url_path
|
63
|
+
PIG_0_12_HADOOP_2_TGZ_DEFAULT_URL_PATH
|
64
|
+
end
|
69
65
|
|
70
|
-
|
66
|
+
def name
|
67
|
+
"pig-#{version}"
|
68
|
+
end
|
69
|
+
|
70
|
+
def version
|
71
|
+
"0.12-Hadoop-2"
|
72
|
+
end
|
73
|
+
|
74
|
+
def cluster_backend
|
75
|
+
Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_2
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
end
|
71
80
|
end
|
data/lib/mortar/version.rb
CHANGED
@@ -106,7 +106,7 @@ other\tgit@github.com:other.git (push)
|
|
106
106
|
context "method_added" do
|
107
107
|
it "replaces help templates" do
|
108
108
|
lines = Base.replace_templates(["line", "start <PIG_VERSION_OPTIONS>"])
|
109
|
-
lines.join("").should == 'linestart 0.
|
109
|
+
lines.join("").should == 'linestart 0.12 and 0.12-Hadoop-2'
|
110
110
|
end
|
111
111
|
end
|
112
112
|
|
@@ -232,7 +232,7 @@ polling_interval=10
|
|
232
232
|
describe_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
|
233
233
|
describe_url = "https://mdog.datadoghq.com/describe/#{describe_id}"
|
234
234
|
|
235
|
-
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.
|
235
|
+
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters=>[]) {Excon::Response.new(:body => {"describe_id" => describe_id})}
|
236
236
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_SUCCESS, "status_description" => "Success", "web_result_url" => describe_url})).ordered
|
237
237
|
write_file(File.join(p.pigscripts_path, "my_script.pig"))
|
238
238
|
|
@@ -113,6 +113,41 @@ B=two three
|
|
113
113
|
STDOUT
|
114
114
|
end
|
115
115
|
end
|
116
|
+
|
117
|
+
it "shows shadowed config vars" do
|
118
|
+
with_git_initialized_project do |p|
|
119
|
+
# stup api request
|
120
|
+
config = {'A' => 'one', 'B' => 'two'}
|
121
|
+
shadow_config = {'B' => 'three'}
|
122
|
+
mock(Mortar::Auth.api).get_config_vars("myproject").returns(Excon::Response.new(:body => {"config" => config,"shadow_config" => shadow_config}))
|
123
|
+
|
124
|
+
stderr, stdout = execute("config", p, @git)
|
125
|
+
stdout.should == <<-STDOUT
|
126
|
+
=== myproject Config Vars
|
127
|
+
A: one
|
128
|
+
B: two
|
129
|
+
|
130
|
+
=== Project config settings overriden by user config
|
131
|
+
B: three
|
132
|
+
STDOUT
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
it "doesn't show shadowed config vars on shell output" do
|
137
|
+
with_git_initialized_project do |p|
|
138
|
+
# stup api request
|
139
|
+
config = {'A' => 'one', 'B' => 'two'}
|
140
|
+
shadow_config = {'B' => 'three'}
|
141
|
+
mock(Mortar::Auth.api).get_config_vars("myproject").returns(Excon::Response.new(:body => {"config" => config,"shadow_config" => shadow_config}))
|
142
|
+
|
143
|
+
stderr, stdout = execute("config --shell", p, @git)
|
144
|
+
stdout.should == <<-STDOUT
|
145
|
+
A=one
|
146
|
+
B=two
|
147
|
+
STDOUT
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
116
151
|
end
|
117
152
|
|
118
153
|
context("get") do
|
@@ -187,6 +222,21 @@ STDOUT
|
|
187
222
|
stdout.should == <<-STDOUT
|
188
223
|
Setting config vars for project myproject... done
|
189
224
|
a: b
|
225
|
+
STDOUT
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
it "sets config vars for user" do
|
230
|
+
with_git_initialized_project do |p|
|
231
|
+
# stub api request
|
232
|
+
configs = {'a' => 'b'}
|
233
|
+
mock(Mortar::Auth.api).put_user_config_vars(configs).returns(Excon::Response.new(:body => {}))
|
234
|
+
|
235
|
+
stderr, stdout = execute("config:set a=b -u", p, @git)
|
236
|
+
stderr.should == ""
|
237
|
+
stdout.should == <<-STDOUT
|
238
|
+
Setting config vars for your user... done
|
239
|
+
a: b
|
190
240
|
STDOUT
|
191
241
|
end
|
192
242
|
end
|
@@ -214,6 +264,19 @@ STDERR
|
|
214
264
|
stderr.should == ""
|
215
265
|
stdout.should == <<-STDOUT
|
216
266
|
Unsetting A for project myproject... done
|
267
|
+
STDOUT
|
268
|
+
end
|
269
|
+
end
|
270
|
+
|
271
|
+
it "unsets one key for user" do
|
272
|
+
with_git_initialized_project do |p|
|
273
|
+
# stub api requests
|
274
|
+
mock(Mortar::Auth.api).delete_user_config_var("A").returns(Excon::Response.new(:body => {}))
|
275
|
+
|
276
|
+
stderr, stdout = execute("config:unset A --user", p, @git)
|
277
|
+
stderr.should == ""
|
278
|
+
stdout.should == <<-STDOUT
|
279
|
+
Unsetting A for your user... done
|
217
280
|
STDOUT
|
218
281
|
end
|
219
282
|
end
|
@@ -98,7 +98,7 @@ STDERR
|
|
98
98
|
describe_url = "https://mdog.datadoghq.com/describe/#{describe_id}"
|
99
99
|
parameters = ["name"=>"key", "value"=>"value" ]
|
100
100
|
|
101
|
-
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.
|
101
|
+
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"describe_id" => describe_id})}
|
102
102
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
103
103
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_GATEWAY_STARTING, "status_description" => "Gateway starting"})).ordered
|
104
104
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_PROGRESS, "status_description" => "Starting pig"})).ordered
|
@@ -129,7 +129,7 @@ STDOUT
|
|
129
129
|
describe_url = "https://mdog.datadoghq.com/describe/#{describe_id}"
|
130
130
|
parameters = ["name"=>"key", "value"=>"value" ]
|
131
131
|
|
132
|
-
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.
|
132
|
+
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"describe_id" => describe_id})}
|
133
133
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
134
134
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_GATEWAY_STARTING, "status_description" => "Gateway starting"})).ordered
|
135
135
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_PROGRESS, "status_description" => "Starting pig"})).ordered
|
@@ -163,7 +163,7 @@ STDOUT
|
|
163
163
|
column_number = 32
|
164
164
|
error_type = 'PigError'
|
165
165
|
|
166
|
-
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.
|
166
|
+
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => []) {Excon::Response.new(:body => {"describe_id" => describe_id})}
|
167
167
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
168
168
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_FAILURE, "status_description" => "Failed",
|
169
169
|
"error_message" => error_message,
|
@@ -198,7 +198,7 @@ STDERR
|
|
198
198
|
describe_url = "https://mdog.datadoghq.com/describe/#{describe_id}"
|
199
199
|
parameters = ["name"=>"key", "value"=>"value" ]
|
200
200
|
|
201
|
-
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.
|
201
|
+
mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"describe_id" => describe_id})}
|
202
202
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
203
203
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_GATEWAY_STARTING, "status_description" => "Gateway starting"})).ordered
|
204
204
|
mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_PROGRESS, "status_description" => "Starting pig"})).ordered
|
@@ -74,7 +74,7 @@ STDERR
|
|
74
74
|
parameters = ["name"=>"key", "value"=>"value" ]
|
75
75
|
|
76
76
|
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
77
|
-
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.
|
77
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
78
78
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
79
79
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_GATEWAY_STARTING, "status_description" => "GATEWAY_STARTING"})).ordered
|
80
80
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_PROGRESS, "status_description" => "In progress"})).ordered
|
@@ -108,7 +108,7 @@ STDOUT
|
|
108
108
|
parameters = ["name"=>"key", "value"=>"value" ]
|
109
109
|
|
110
110
|
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
111
|
-
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.
|
111
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
112
112
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
113
113
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_GATEWAY_STARTING, "status_description" => "GATEWAY_STARTING"})).ordered
|
114
114
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_PROGRESS, "status_description" => "In progress"})).ordered
|
@@ -142,7 +142,7 @@ STDOUT
|
|
142
142
|
parameters = ["name"=>"key", "value"=>"value" ]
|
143
143
|
|
144
144
|
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
145
|
-
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.
|
145
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
146
146
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
147
147
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_SUCCESS, "status_description" => "Succeeded", "web_result_url" => illustrate_url})).ordered
|
148
148
|
|
@@ -169,7 +169,7 @@ STDOUT
|
|
169
169
|
parameters = ["name"=>"key", "value"=>"value" ]
|
170
170
|
|
171
171
|
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
172
|
-
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", true, is_a(String), :pig_version => "0.
|
172
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", true, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
173
173
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
174
174
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_SUCCESS, "status_description" => "Succeeded", "web_result_url" => illustrate_url})).ordered
|
175
175
|
|
@@ -195,7 +195,7 @@ STDOUT
|
|
195
195
|
parameters = ["name"=>"key", "value"=>"value" ]
|
196
196
|
|
197
197
|
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
198
|
-
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", nil, false, is_a(String), :pig_version => "0.
|
198
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", nil, false, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
199
199
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
200
200
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_GATEWAY_STARTING, "status_description" => "GATEWAY_STARTING"})).ordered
|
201
201
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_PROGRESS, "status_description" => "In progress"})).ordered
|
@@ -232,7 +232,7 @@ STDOUT
|
|
232
232
|
error_type = 'PigError'
|
233
233
|
|
234
234
|
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
235
|
-
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.
|
235
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => []) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
236
236
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
237
237
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_FAILURE,
|
238
238
|
"error_message" => error_message,
|
@@ -268,7 +268,7 @@ STDERR
|
|
268
268
|
parameters = ["name"=>"key", "value"=>"value" ]
|
269
269
|
|
270
270
|
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
271
|
-
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", nil, false, is_a(String), :pig_version => "0.
|
271
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", nil, false, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
272
272
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
273
273
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_GATEWAY_STARTING, "status_description" => "GATEWAY_STARTING"})).ordered
|
274
274
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_PROGRESS, "status_description" => "In progress"})).ordered
|
@@ -47,7 +47,7 @@ module Mortar::Command
|
|
47
47
|
cluster_size = 5
|
48
48
|
|
49
49
|
mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
|
50
|
-
:pig_version => "0.
|
50
|
+
:pig_version => "0.12",
|
51
51
|
:project_script_path => be_a_kind_of(String),
|
52
52
|
:parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
|
53
53
|
:cluster_type => Jobs::CLUSTER_TYPE__SINGLE_JOB,
|
@@ -83,7 +83,7 @@ STDOUT
|
|
83
83
|
cluster_size = 5
|
84
84
|
|
85
85
|
mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String),cluster_size,
|
86
|
-
:pig_version => "0.
|
86
|
+
:pig_version => "0.12",
|
87
87
|
:project_script_path => be_a_kind_of(String),
|
88
88
|
:parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
|
89
89
|
:cluster_type => Jobs::CLUSTER_TYPE__PERMANENT,
|
@@ -143,7 +143,7 @@ STDERR
|
|
143
143
|
cluster_size = 5
|
144
144
|
|
145
145
|
mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String),cluster_size,
|
146
|
-
:pig_version => "0.
|
146
|
+
:pig_version => "0.12",
|
147
147
|
:project_script_path => be_a_kind_of(String),
|
148
148
|
:parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
|
149
149
|
:cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
|
@@ -179,7 +179,7 @@ STDOUT
|
|
179
179
|
cluster_size = 5
|
180
180
|
|
181
181
|
mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String),cluster_size,
|
182
|
-
:pig_version => "0.
|
182
|
+
:pig_version => "0.12",
|
183
183
|
:project_script_path => be_a_kind_of(String),
|
184
184
|
:parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
|
185
185
|
:cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
|
@@ -215,7 +215,7 @@ STDOUT
|
|
215
215
|
cluster_id = "e2790e7e8c7d48e39157238d58191346"
|
216
216
|
|
217
217
|
mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id,
|
218
|
-
:pig_version => "0.
|
218
|
+
:pig_version => "0.12",
|
219
219
|
:project_script_path => be_a_kind_of(String),
|
220
220
|
:parameters => [],
|
221
221
|
:notify_on_job_finish => false,
|
@@ -249,7 +249,7 @@ STDOUT
|
|
249
249
|
cluster_id = "e2790e7e8c7d48e39157238d58191346"
|
250
250
|
|
251
251
|
mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id,
|
252
|
-
:pig_version => "0.
|
252
|
+
:pig_version => "0.12",
|
253
253
|
:project_script_path => be_a_kind_of(String),
|
254
254
|
:parameters => [],
|
255
255
|
:notify_on_job_finish => false,
|
@@ -283,7 +283,7 @@ STDOUT
|
|
283
283
|
|
284
284
|
mock(Mortar::Auth.api).get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1) {Excon::Response.new(:body => {'clusters' => []})}
|
285
285
|
mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
|
286
|
-
:pig_version => "0.
|
286
|
+
:pig_version => "0.12",
|
287
287
|
:project_script_path => be_a_kind_of(String),
|
288
288
|
:parameters => [],
|
289
289
|
:cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
|
@@ -321,7 +321,7 @@ STDOUT
|
|
321
321
|
|
322
322
|
mock(Mortar::Auth.api).get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1) {Excon::Response.new(:body => {'clusters' => []})}
|
323
323
|
mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
|
324
|
-
:pig_version => "0.
|
324
|
+
:pig_version => "0.12",
|
325
325
|
:project_script_path => be_a_kind_of(String),
|
326
326
|
:parameters => [],
|
327
327
|
:cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
|
@@ -358,7 +358,7 @@ STDOUT
|
|
358
358
|
job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
|
359
359
|
cluster_id = "e2790e7e8c7d48e39157238d58191346"
|
360
360
|
|
361
|
-
mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id, :pig_version => "0.
|
361
|
+
mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id, :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => [], :notify_on_job_finish => false, :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
|
362
362
|
|
363
363
|
write_file(File.join(p.pigscripts_path, "my_script.pig"))
|
364
364
|
stderr, stdout = execute("jobs:run pigscripts/my_script.pig --clusterid e2790e7e8c7d48e39157238d58191346 -d", p, @git)
|
@@ -411,7 +411,7 @@ STDOUT
|
|
411
411
|
]})
|
412
412
|
}
|
413
413
|
mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), large_cluster_id,
|
414
|
-
:pig_version => "0.
|
414
|
+
:pig_version => "0.12",
|
415
415
|
:project_script_path => be_a_kind_of(String),
|
416
416
|
:parameters => [],
|
417
417
|
:notify_on_job_finish => true,
|
@@ -426,6 +426,54 @@ Sending code snapshot to Mortar... done
|
|
426
426
|
Requesting job execution... done
|
427
427
|
job_id: c571a8c7f76a4fd4a67c103d753e2dd5
|
428
428
|
|
429
|
+
Job status can be viewed on the web at:
|
430
|
+
|
431
|
+
http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
|
432
|
+
|
433
|
+
Or by running:
|
434
|
+
|
435
|
+
mortar jobs:status c571a8c7f76a4fd4a67c103d753e2dd5 --poll
|
436
|
+
|
437
|
+
STDOUT
|
438
|
+
|
439
|
+
end
|
440
|
+
end
|
441
|
+
|
442
|
+
it "runs a job by default on the largest existing running cluster for Hadoop 2" do
|
443
|
+
with_git_initialized_project do |p|
|
444
|
+
job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
|
445
|
+
job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
|
446
|
+
|
447
|
+
small_h2_cluster_id = '510beb6b3004860820ab6538'
|
448
|
+
small_h2_cluster_size = 2
|
449
|
+
small_h2_cluster_status = Mortar::API::Clusters::STATUS_RUNNING
|
450
|
+
large_h2_cluster_id = '510bf0db3004860820ab6590'
|
451
|
+
large_h2_cluster_size = 5
|
452
|
+
large_h2_cluster_status = Mortar::API::Clusters::STATUS_RUNNING
|
453
|
+
|
454
|
+
mock(Mortar::Auth.api).get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_2) {
|
455
|
+
Excon::Response.new(:body => {
|
456
|
+
'clusters' => [
|
457
|
+
{ 'cluster_id' => small_h2_cluster_id, 'size' => small_h2_cluster_size, 'running_jobs' => [], 'status_code' => small_h2_cluster_status },
|
458
|
+
{ 'cluster_id' => large_h2_cluster_id, 'size' => large_h2_cluster_size, 'running_jobs' => [], 'status_code' => large_h2_cluster_status },
|
459
|
+
]})
|
460
|
+
}
|
461
|
+
mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), large_h2_cluster_id,
|
462
|
+
:pig_version => "0.12-Hadoop-2",
|
463
|
+
:project_script_path => be_a_kind_of(String),
|
464
|
+
:parameters => [],
|
465
|
+
:notify_on_job_finish => true,
|
466
|
+
:is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
|
467
|
+
|
468
|
+
write_file(File.join(p.pigscripts_path, "my_script.pig"))
|
469
|
+
stderr, stdout = execute("jobs:run pigscripts/my_script.pig -g 0.12-Hadoop-2", p, @git)
|
470
|
+
stdout.should == <<-STDOUT
|
471
|
+
Defaulting to running job on largest existing free cluster, id = 510bf0db3004860820ab6590, size = 5
|
472
|
+
Taking code snapshot... done
|
473
|
+
Sending code snapshot to Mortar... done
|
474
|
+
Requesting job execution... done
|
475
|
+
job_id: c571a8c7f76a4fd4a67c103d753e2dd5
|
476
|
+
|
429
477
|
Job status can be viewed on the web at:
|
430
478
|
|
431
479
|
http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
|
@@ -445,7 +493,7 @@ STDOUT
|
|
445
493
|
cluster_size = 5
|
446
494
|
|
447
495
|
mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
|
448
|
-
:pig_version => "0.
|
496
|
+
:pig_version => "0.12",
|
449
497
|
:project_script_path => be_a_kind_of(String),
|
450
498
|
:parameters => match_array([{"name" => "FIRST", "value" => "FOO"}, {"name" => "SECOND", "value" => "BAR"}, {"name" => "THIRD", "value" => "BEAR"}]),
|
451
499
|
:cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
|
@@ -473,7 +521,7 @@ PARAMS
|
|
473
521
|
cluster_size = 5
|
474
522
|
|
475
523
|
mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
|
476
|
-
:pig_version => "0.
|
524
|
+
:pig_version => "0.12",
|
477
525
|
:project_script_path => be_a_kind_of(String),
|
478
526
|
:parameters => match_array([{"name" => "FIRST", "value" => "FOO"}, {"name" => "SECOND", "value" => "BAR"}, {"name" => "THIRD", "value" => "BEAR"}]),
|
479
527
|
:cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
|
@@ -529,7 +577,7 @@ STDERR
|
|
529
577
|
mock(@git).sync_embedded_project.with_any_args.times(1) { "somewhere_over_the_rainbow" }
|
530
578
|
|
531
579
|
mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
|
532
|
-
:pig_version => "0.
|
580
|
+
:pig_version => "0.12",
|
533
581
|
:project_script_path => be_a_kind_of(String),
|
534
582
|
:parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
|
535
583
|
:cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
|