mortar 0.15.36 → 0.15.37

Sign up to get free protection for your applications and to get access to all the features.
@@ -323,7 +323,7 @@ class Mortar::Local::Pig
323
323
  def template_params_classpath(pig_version=nil)
324
324
  # Need to support old watchtower plugins that don't set pig_version
325
325
  if pig_version.nil?
326
- pig_version = Mortar::PigVersion::Pig09.new
326
+ pig_version = Mortar::PigVersion::Pig012.new
327
327
  end
328
328
  [ "#{pig_directory(pig_version)}/*",
329
329
  "#{pig_directory(pig_version)}/lib-local/*",
@@ -15,57 +15,66 @@
15
15
  #
16
16
 
17
17
  module Mortar
18
- module PigVersion
19
- PIG_0_9_TGZ_NAME = "pig-0.9.tar.gz"
20
- PIG_0_9_TGZ_DEFAULT_URL_PATH = "resource/pig_0_9"
21
- PIG_0_12_TGZ_NAME = "pig-0.12.tar.gz"
22
- PIG_0_12_TGZ_DEFAULT_URL_PATH = "resource/pig_0_12"
18
+ module PigVersion
19
+ PIG_0_12_TGZ_NAME = "pig-0.12.tar.gz"
20
+ PIG_0_12_TGZ_DEFAULT_URL_PATH = "resource/pig_0_12"
23
21
 
24
- def PigVersion.from_string(pig_version_str)
25
- if pig_version_str == '0.9'
26
- return Pig09.new
27
- elsif pig_version_str == '0.12'
28
- return Pig012.new
29
- else
30
- raise ArgumentError, "Unsupported pig version: #{pig_version_str}"
31
- end
32
- end
22
+ PIG_0_12_HADOOP_2_TGZ_NAME = "pig-0.12-hadoop-2.tar.gz"
23
+ PIG_0_12_HADOOP_2_TGZ_DEFAULT_URL_PATH = "resource/pig_0_12_hadoop_2"
33
24
 
34
- class Pig09
35
- def tgz_name
36
- PIG_0_9_TGZ_NAME
37
- end
25
+ def PigVersion.from_string(pig_version_str)
26
+ if pig_version_str == '0.12'
27
+ return Pig012.new
28
+ elsif pig_version_str == '0.12-Hadoop-2'
29
+ return Pig012Hadoop2.new
30
+ else
31
+ raise ArgumentError, "Unsupported pig version: #{pig_version_str}. Options are: ['0.12', '0.12-Hadoop-2]."
32
+ end
33
+ end
38
34
 
39
- def tgz_default_url_path
40
- PIG_0_9_TGZ_DEFAULT_URL_PATH
41
- end
35
+ class Pig012
36
+ def tgz_name
37
+ PIG_0_12_TGZ_NAME
38
+ end
42
39
 
43
- def name
44
- "pig-#{version}"
45
- end
40
+ def tgz_default_url_path
41
+ PIG_0_12_TGZ_DEFAULT_URL_PATH
42
+ end
46
43
 
47
- def version
48
- "0.9"
49
- end
50
- end
44
+ def name
45
+ "pig-#{version}"
46
+ end
51
47
 
52
- class Pig012
53
- def tgz_name
54
- PIG_0_12_TGZ_NAME
55
- end
48
+ def version
49
+ "0.12"
50
+ end
56
51
 
57
- def tgz_default_url_path
58
- PIG_0_12_TGZ_DEFAULT_URL_PATH
59
- end
52
+ def cluster_backend
53
+ Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1
54
+ end
55
+ end
60
56
 
61
- def name
62
- "pig-#{version}"
63
- end
57
+ class Pig012Hadoop2
58
+ def tgz_name
59
+ PIG_0_12_HADOOP_2_TGZ_NAME
60
+ end
64
61
 
65
- def version
66
- "0.12"
67
- end
68
- end
62
+ def tgz_default_url_path
63
+ PIG_0_12_HADOOP_2_TGZ_DEFAULT_URL_PATH
64
+ end
69
65
 
70
- end
66
+ def name
67
+ "pig-#{version}"
68
+ end
69
+
70
+ def version
71
+ "0.12-Hadoop-2"
72
+ end
73
+
74
+ def cluster_backend
75
+ Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_2
76
+ end
77
+ end
78
+
79
+ end
71
80
  end
@@ -16,5 +16,5 @@
16
16
 
17
17
  module Mortar
18
18
  # see http://semver.org/
19
- VERSION = "0.15.36"
19
+ VERSION = "0.15.37"
20
20
  end
@@ -106,7 +106,7 @@ other\tgit@github.com:other.git (push)
106
106
  context "method_added" do
107
107
  it "replaces help templates" do
108
108
  lines = Base.replace_templates(["line", "start <PIG_VERSION_OPTIONS>"])
109
- lines.join("").should == 'linestart 0.9 (default) and 0.12'
109
+ lines.join("").should == 'linestart 0.12 and 0.12-Hadoop-2'
110
110
  end
111
111
  end
112
112
 
@@ -232,7 +232,7 @@ polling_interval=10
232
232
  describe_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
233
233
  describe_url = "https://mdog.datadoghq.com/describe/#{describe_id}"
234
234
 
235
- mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.9", :project_script_path => be_a_kind_of(String), :parameters=>[]) {Excon::Response.new(:body => {"describe_id" => describe_id})}
235
+ mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters=>[]) {Excon::Response.new(:body => {"describe_id" => describe_id})}
236
236
  mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_SUCCESS, "status_description" => "Success", "web_result_url" => describe_url})).ordered
237
237
  write_file(File.join(p.pigscripts_path, "my_script.pig"))
238
238
 
@@ -113,6 +113,41 @@ B=two three
113
113
  STDOUT
114
114
  end
115
115
  end
116
+
117
+ it "shows shadowed config vars" do
118
+ with_git_initialized_project do |p|
119
+ # stup api request
120
+ config = {'A' => 'one', 'B' => 'two'}
121
+ shadow_config = {'B' => 'three'}
122
+ mock(Mortar::Auth.api).get_config_vars("myproject").returns(Excon::Response.new(:body => {"config" => config,"shadow_config" => shadow_config}))
123
+
124
+ stderr, stdout = execute("config", p, @git)
125
+ stdout.should == <<-STDOUT
126
+ === myproject Config Vars
127
+ A: one
128
+ B: two
129
+
130
+ === Project config settings overriden by user config
131
+ B: three
132
+ STDOUT
133
+ end
134
+ end
135
+
136
+ it "doesn't show shadowed config vars on shell output" do
137
+ with_git_initialized_project do |p|
138
+ # stup api request
139
+ config = {'A' => 'one', 'B' => 'two'}
140
+ shadow_config = {'B' => 'three'}
141
+ mock(Mortar::Auth.api).get_config_vars("myproject").returns(Excon::Response.new(:body => {"config" => config,"shadow_config" => shadow_config}))
142
+
143
+ stderr, stdout = execute("config --shell", p, @git)
144
+ stdout.should == <<-STDOUT
145
+ A=one
146
+ B=two
147
+ STDOUT
148
+ end
149
+ end
150
+
116
151
  end
117
152
 
118
153
  context("get") do
@@ -187,6 +222,21 @@ STDOUT
187
222
  stdout.should == <<-STDOUT
188
223
  Setting config vars for project myproject... done
189
224
  a: b
225
+ STDOUT
226
+ end
227
+ end
228
+
229
+ it "sets config vars for user" do
230
+ with_git_initialized_project do |p|
231
+ # stub api request
232
+ configs = {'a' => 'b'}
233
+ mock(Mortar::Auth.api).put_user_config_vars(configs).returns(Excon::Response.new(:body => {}))
234
+
235
+ stderr, stdout = execute("config:set a=b -u", p, @git)
236
+ stderr.should == ""
237
+ stdout.should == <<-STDOUT
238
+ Setting config vars for your user... done
239
+ a: b
190
240
  STDOUT
191
241
  end
192
242
  end
@@ -214,6 +264,19 @@ STDERR
214
264
  stderr.should == ""
215
265
  stdout.should == <<-STDOUT
216
266
  Unsetting A for project myproject... done
267
+ STDOUT
268
+ end
269
+ end
270
+
271
+ it "unsets one key for user" do
272
+ with_git_initialized_project do |p|
273
+ # stub api requests
274
+ mock(Mortar::Auth.api).delete_user_config_var("A").returns(Excon::Response.new(:body => {}))
275
+
276
+ stderr, stdout = execute("config:unset A --user", p, @git)
277
+ stderr.should == ""
278
+ stdout.should == <<-STDOUT
279
+ Unsetting A for your user... done
217
280
  STDOUT
218
281
  end
219
282
  end
@@ -98,7 +98,7 @@ STDERR
98
98
  describe_url = "https://mdog.datadoghq.com/describe/#{describe_id}"
99
99
  parameters = ["name"=>"key", "value"=>"value" ]
100
100
 
101
- mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.9", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"describe_id" => describe_id})}
101
+ mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"describe_id" => describe_id})}
102
102
  mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_QUEUED, "status_description" => "Pending"})).ordered
103
103
  mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_GATEWAY_STARTING, "status_description" => "Gateway starting"})).ordered
104
104
  mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_PROGRESS, "status_description" => "Starting pig"})).ordered
@@ -129,7 +129,7 @@ STDOUT
129
129
  describe_url = "https://mdog.datadoghq.com/describe/#{describe_id}"
130
130
  parameters = ["name"=>"key", "value"=>"value" ]
131
131
 
132
- mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.9", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"describe_id" => describe_id})}
132
+ mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"describe_id" => describe_id})}
133
133
  mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_QUEUED, "status_description" => "Pending"})).ordered
134
134
  mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_GATEWAY_STARTING, "status_description" => "Gateway starting"})).ordered
135
135
  mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_PROGRESS, "status_description" => "Starting pig"})).ordered
@@ -163,7 +163,7 @@ STDOUT
163
163
  column_number = 32
164
164
  error_type = 'PigError'
165
165
 
166
- mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.9", :project_script_path => be_a_kind_of(String), :parameters => []) {Excon::Response.new(:body => {"describe_id" => describe_id})}
166
+ mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => []) {Excon::Response.new(:body => {"describe_id" => describe_id})}
167
167
  mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_QUEUED, "status_description" => "Pending"})).ordered
168
168
  mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_FAILURE, "status_description" => "Failed",
169
169
  "error_message" => error_message,
@@ -198,7 +198,7 @@ STDERR
198
198
  describe_url = "https://mdog.datadoghq.com/describe/#{describe_id}"
199
199
  parameters = ["name"=>"key", "value"=>"value" ]
200
200
 
201
- mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.9", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"describe_id" => describe_id})}
201
+ mock(Mortar::Auth.api).post_describe("myproject", "my_script", "my_alias", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"describe_id" => describe_id})}
202
202
  mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_QUEUED, "status_description" => "Pending"})).ordered
203
203
  mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_GATEWAY_STARTING, "status_description" => "Gateway starting"})).ordered
204
204
  mock(Mortar::Auth.api).get_describe(describe_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Describe::STATUS_PROGRESS, "status_description" => "Starting pig"})).ordered
@@ -74,7 +74,7 @@ STDERR
74
74
  parameters = ["name"=>"key", "value"=>"value" ]
75
75
 
76
76
  # These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
77
- mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.9", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
77
+ mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
78
78
  mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
79
79
  mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_GATEWAY_STARTING, "status_description" => "GATEWAY_STARTING"})).ordered
80
80
  mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_PROGRESS, "status_description" => "In progress"})).ordered
@@ -108,7 +108,7 @@ STDOUT
108
108
  parameters = ["name"=>"key", "value"=>"value" ]
109
109
 
110
110
  # These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
111
- mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.9", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
111
+ mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
112
112
  mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
113
113
  mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_GATEWAY_STARTING, "status_description" => "GATEWAY_STARTING"})).ordered
114
114
  mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_PROGRESS, "status_description" => "In progress"})).ordered
@@ -142,7 +142,7 @@ STDOUT
142
142
  parameters = ["name"=>"key", "value"=>"value" ]
143
143
 
144
144
  # These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
145
- mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.9", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
145
+ mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
146
146
  mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
147
147
  mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_SUCCESS, "status_description" => "Succeeded", "web_result_url" => illustrate_url})).ordered
148
148
 
@@ -169,7 +169,7 @@ STDOUT
169
169
  parameters = ["name"=>"key", "value"=>"value" ]
170
170
 
171
171
  # These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
172
- mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", true, is_a(String), :pig_version => "0.9", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
172
+ mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", true, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
173
173
  mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
174
174
  mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_SUCCESS, "status_description" => "Succeeded", "web_result_url" => illustrate_url})).ordered
175
175
 
@@ -195,7 +195,7 @@ STDOUT
195
195
  parameters = ["name"=>"key", "value"=>"value" ]
196
196
 
197
197
  # These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
198
- mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", nil, false, is_a(String), :pig_version => "0.9", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
198
+ mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", nil, false, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
199
199
  mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
200
200
  mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_GATEWAY_STARTING, "status_description" => "GATEWAY_STARTING"})).ordered
201
201
  mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_PROGRESS, "status_description" => "In progress"})).ordered
@@ -232,7 +232,7 @@ STDOUT
232
232
  error_type = 'PigError'
233
233
 
234
234
  # These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
235
- mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.9", :project_script_path => be_a_kind_of(String), :parameters => []) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
235
+ mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => []) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
236
236
  mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
237
237
  mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_FAILURE,
238
238
  "error_message" => error_message,
@@ -268,7 +268,7 @@ STDERR
268
268
  parameters = ["name"=>"key", "value"=>"value" ]
269
269
 
270
270
  # These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
271
- mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", nil, false, is_a(String), :pig_version => "0.9", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
271
+ mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", nil, false, is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
272
272
  mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
273
273
  mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_GATEWAY_STARTING, "status_description" => "GATEWAY_STARTING"})).ordered
274
274
  mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_PROGRESS, "status_description" => "In progress"})).ordered
@@ -47,7 +47,7 @@ module Mortar::Command
47
47
  cluster_size = 5
48
48
 
49
49
  mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
50
- :pig_version => "0.9",
50
+ :pig_version => "0.12",
51
51
  :project_script_path => be_a_kind_of(String),
52
52
  :parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
53
53
  :cluster_type => Jobs::CLUSTER_TYPE__SINGLE_JOB,
@@ -83,7 +83,7 @@ STDOUT
83
83
  cluster_size = 5
84
84
 
85
85
  mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String),cluster_size,
86
- :pig_version => "0.9",
86
+ :pig_version => "0.12",
87
87
  :project_script_path => be_a_kind_of(String),
88
88
  :parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
89
89
  :cluster_type => Jobs::CLUSTER_TYPE__PERMANENT,
@@ -143,7 +143,7 @@ STDERR
143
143
  cluster_size = 5
144
144
 
145
145
  mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String),cluster_size,
146
- :pig_version => "0.9",
146
+ :pig_version => "0.12",
147
147
  :project_script_path => be_a_kind_of(String),
148
148
  :parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
149
149
  :cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
@@ -179,7 +179,7 @@ STDOUT
179
179
  cluster_size = 5
180
180
 
181
181
  mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String),cluster_size,
182
- :pig_version => "0.9",
182
+ :pig_version => "0.12",
183
183
  :project_script_path => be_a_kind_of(String),
184
184
  :parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
185
185
  :cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
@@ -215,7 +215,7 @@ STDOUT
215
215
  cluster_id = "e2790e7e8c7d48e39157238d58191346"
216
216
 
217
217
  mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id,
218
- :pig_version => "0.9",
218
+ :pig_version => "0.12",
219
219
  :project_script_path => be_a_kind_of(String),
220
220
  :parameters => [],
221
221
  :notify_on_job_finish => false,
@@ -249,7 +249,7 @@ STDOUT
249
249
  cluster_id = "e2790e7e8c7d48e39157238d58191346"
250
250
 
251
251
  mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id,
252
- :pig_version => "0.9",
252
+ :pig_version => "0.12",
253
253
  :project_script_path => be_a_kind_of(String),
254
254
  :parameters => [],
255
255
  :notify_on_job_finish => false,
@@ -283,7 +283,7 @@ STDOUT
283
283
 
284
284
  mock(Mortar::Auth.api).get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1) {Excon::Response.new(:body => {'clusters' => []})}
285
285
  mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
286
- :pig_version => "0.9",
286
+ :pig_version => "0.12",
287
287
  :project_script_path => be_a_kind_of(String),
288
288
  :parameters => [],
289
289
  :cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
@@ -321,7 +321,7 @@ STDOUT
321
321
 
322
322
  mock(Mortar::Auth.api).get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1) {Excon::Response.new(:body => {'clusters' => []})}
323
323
  mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
324
- :pig_version => "0.9",
324
+ :pig_version => "0.12",
325
325
  :project_script_path => be_a_kind_of(String),
326
326
  :parameters => [],
327
327
  :cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
@@ -358,7 +358,7 @@ STDOUT
358
358
  job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
359
359
  cluster_id = "e2790e7e8c7d48e39157238d58191346"
360
360
 
361
- mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id, :pig_version => "0.9", :project_script_path => be_a_kind_of(String), :parameters => [], :notify_on_job_finish => false, :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
361
+ mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id, :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => [], :notify_on_job_finish => false, :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
362
362
 
363
363
  write_file(File.join(p.pigscripts_path, "my_script.pig"))
364
364
  stderr, stdout = execute("jobs:run pigscripts/my_script.pig --clusterid e2790e7e8c7d48e39157238d58191346 -d", p, @git)
@@ -411,7 +411,7 @@ STDOUT
411
411
  ]})
412
412
  }
413
413
  mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), large_cluster_id,
414
- :pig_version => "0.9",
414
+ :pig_version => "0.12",
415
415
  :project_script_path => be_a_kind_of(String),
416
416
  :parameters => [],
417
417
  :notify_on_job_finish => true,
@@ -426,6 +426,54 @@ Sending code snapshot to Mortar... done
426
426
  Requesting job execution... done
427
427
  job_id: c571a8c7f76a4fd4a67c103d753e2dd5
428
428
 
429
+ Job status can be viewed on the web at:
430
+
431
+ http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
432
+
433
+ Or by running:
434
+
435
+ mortar jobs:status c571a8c7f76a4fd4a67c103d753e2dd5 --poll
436
+
437
+ STDOUT
438
+
439
+ end
440
+ end
441
+
442
+ it "runs a job by default on the largest existing running cluster for Hadoop 2" do
443
+ with_git_initialized_project do |p|
444
+ job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
445
+ job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
446
+
447
+ small_h2_cluster_id = '510beb6b3004860820ab6538'
448
+ small_h2_cluster_size = 2
449
+ small_h2_cluster_status = Mortar::API::Clusters::STATUS_RUNNING
450
+ large_h2_cluster_id = '510bf0db3004860820ab6590'
451
+ large_h2_cluster_size = 5
452
+ large_h2_cluster_status = Mortar::API::Clusters::STATUS_RUNNING
453
+
454
+ mock(Mortar::Auth.api).get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_2) {
455
+ Excon::Response.new(:body => {
456
+ 'clusters' => [
457
+ { 'cluster_id' => small_h2_cluster_id, 'size' => small_h2_cluster_size, 'running_jobs' => [], 'status_code' => small_h2_cluster_status },
458
+ { 'cluster_id' => large_h2_cluster_id, 'size' => large_h2_cluster_size, 'running_jobs' => [], 'status_code' => large_h2_cluster_status },
459
+ ]})
460
+ }
461
+ mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), large_h2_cluster_id,
462
+ :pig_version => "0.12-Hadoop-2",
463
+ :project_script_path => be_a_kind_of(String),
464
+ :parameters => [],
465
+ :notify_on_job_finish => true,
466
+ :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
467
+
468
+ write_file(File.join(p.pigscripts_path, "my_script.pig"))
469
+ stderr, stdout = execute("jobs:run pigscripts/my_script.pig -g 0.12-Hadoop-2", p, @git)
470
+ stdout.should == <<-STDOUT
471
+ Defaulting to running job on largest existing free cluster, id = 510bf0db3004860820ab6590, size = 5
472
+ Taking code snapshot... done
473
+ Sending code snapshot to Mortar... done
474
+ Requesting job execution... done
475
+ job_id: c571a8c7f76a4fd4a67c103d753e2dd5
476
+
429
477
  Job status can be viewed on the web at:
430
478
 
431
479
  http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
@@ -445,7 +493,7 @@ STDOUT
445
493
  cluster_size = 5
446
494
 
447
495
  mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
448
- :pig_version => "0.9",
496
+ :pig_version => "0.12",
449
497
  :project_script_path => be_a_kind_of(String),
450
498
  :parameters => match_array([{"name" => "FIRST", "value" => "FOO"}, {"name" => "SECOND", "value" => "BAR"}, {"name" => "THIRD", "value" => "BEAR"}]),
451
499
  :cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
@@ -473,7 +521,7 @@ PARAMS
473
521
  cluster_size = 5
474
522
 
475
523
  mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
476
- :pig_version => "0.9",
524
+ :pig_version => "0.12",
477
525
  :project_script_path => be_a_kind_of(String),
478
526
  :parameters => match_array([{"name" => "FIRST", "value" => "FOO"}, {"name" => "SECOND", "value" => "BAR"}, {"name" => "THIRD", "value" => "BEAR"}]),
479
527
  :cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,
@@ -529,7 +577,7 @@ STDERR
529
577
  mock(@git).sync_embedded_project.with_any_args.times(1) { "somewhere_over_the_rainbow" }
530
578
 
531
579
  mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
532
- :pig_version => "0.9",
580
+ :pig_version => "0.12",
533
581
  :project_script_path => be_a_kind_of(String),
534
582
  :parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
535
583
  :cluster_type => Jobs::CLUSTER_TYPE__PERSISTENT,