mortar 0.6.2 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -49,7 +49,8 @@ module Mortar::Command
49
49
  mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
50
50
  :parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
51
51
  :keepalive => false,
52
- :notify_on_job_finish => true) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
52
+ :notify_on_job_finish => true,
53
+ :is_control_script=> false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
53
54
 
54
55
  write_file(File.join(p.pigscripts_path, "my_script.pig"))
55
56
  stderr, stdout = execute("jobs:run my_script -1 --clustersize 5 -p FIRST_PARAM=FOO -p SECOND_PARAM=BAR", p, @git)
@@ -81,7 +82,8 @@ STDOUT
81
82
  mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
82
83
  :parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
83
84
  :keepalive => true,
84
- :notify_on_job_finish => true) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
85
+ :notify_on_job_finish => true,
86
+ :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
85
87
 
86
88
  write_file(File.join(p.pigscripts_path, "my_script.pig"))
87
89
  stderr, stdout = execute("jobs:run my_script --clustersize 5 -p FIRST_PARAM=FOO -p SECOND_PARAM=BAR", p, @git)
@@ -91,6 +93,38 @@ Sending code snapshot to Mortar... done
91
93
  Requesting job execution... done
92
94
  job_id: c571a8c7f76a4fd4a67c103d753e2dd5
93
95
 
96
+ Job status can be viewed on the web at:
97
+
98
+ http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
99
+
100
+ Or by running:
101
+
102
+ mortar jobs:status c571a8c7f76a4fd4a67c103d753e2dd5 --poll
103
+
104
+ STDOUT
105
+ end
106
+ end
107
+
108
+ it "runs a control script" do
109
+ with_git_initialized_project do |p|
110
+ # stub api requests
111
+ job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
112
+ job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
113
+ cluster_id = "e2790e7e8c7d48e39157238d58191346"
114
+
115
+ mock(Mortar::Auth.api).post_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id,
116
+ :parameters => [],
117
+ :notify_on_job_finish => false,
118
+ :is_control_script=>true) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
119
+
120
+ write_file(File.join(p.controlscripts_path, "my_script.py"))
121
+ stderr, stdout = execute("jobs:run my_script --clusterid e2790e7e8c7d48e39157238d58191346 -d", p, @git)
122
+ stdout.should == <<-STDOUT
123
+ Taking code snapshot... done
124
+ Sending code snapshot to Mortar... done
125
+ Requesting job execution... done
126
+ job_id: c571a8c7f76a4fd4a67c103d753e2dd5
127
+
94
128
  Job status can be viewed on the web at:
95
129
 
96
130
  http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
@@ -113,7 +147,8 @@ STDOUT
113
147
  mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
114
148
  :parameters => [],
115
149
  :keepalive => true,
116
- :notify_on_job_finish => true) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
150
+ :notify_on_job_finish => true,
151
+ :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
117
152
 
118
153
  write_file(File.join(p.pigscripts_path, "my_script.pig"))
119
154
  stderr, stdout = execute("jobs:run my_script ", p, @git)
@@ -144,7 +179,7 @@ STDOUT
144
179
  job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
145
180
  cluster_id = "e2790e7e8c7d48e39157238d58191346"
146
181
 
147
- mock(Mortar::Auth.api).post_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id, :parameters => [], :notify_on_job_finish => false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
182
+ mock(Mortar::Auth.api).post_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id, :parameters => [], :notify_on_job_finish => false, :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
148
183
 
149
184
  write_file(File.join(p.pigscripts_path, "my_script.pig"))
150
185
  stderr, stdout = execute("jobs:run my_script --clusterid e2790e7e8c7d48e39157238d58191346 -d", p, @git)
@@ -198,7 +233,8 @@ STDOUT
198
233
  }
199
234
  mock(Mortar::Auth.api).post_job_existing_cluster("myproject", "my_script", is_a(String), large_cluster_id,
200
235
  :parameters => [],
201
- :notify_on_job_finish => true) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
236
+ :notify_on_job_finish => true,
237
+ :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
202
238
 
203
239
  write_file(File.join(p.pigscripts_path, "my_script.pig"))
204
240
  stderr, stdout = execute("jobs:run my_script ", p, @git)
@@ -231,7 +267,8 @@ STDOUT
231
267
  mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
232
268
  :parameters => match_array([{"name" => "FIRST", "value" => "FOO"}, {"name" => "SECOND", "value" => "BAR"}, {"name" => "THIRD", "value" => "BEAR\n"}]),
233
269
  :keepalive => true,
234
- :notify_on_job_finish => true) {Excon::Response.new(:body => {"job_id" => job_id})}
270
+ :notify_on_job_finish => true,
271
+ :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id})}
235
272
 
236
273
  write_file(File.join(p.pigscripts_path, "my_script.pig"))
237
274
 
@@ -256,7 +293,8 @@ PARAMS
256
293
  mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
257
294
  :parameters => match_array([{"name" => "FIRST", "value" => "FOO"}, {"name" => "SECOND", "value" => "BAR"}, {"name" => "THIRD", "value" => "BEAR\n"}]),
258
295
  :keepalive => true,
259
- :notify_on_job_finish => true) {Excon::Response.new(:body => {"job_id" => job_id})}
296
+ :notify_on_job_finish => true,
297
+ :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id})}
260
298
 
261
299
  write_file(File.join(p.pigscripts_path, "my_script.pig"))
262
300
 
@@ -458,6 +496,86 @@ status: Execution error
458
496
  STDOUT
459
497
  end
460
498
  end
499
+
500
+ it "gets status for a control job" do
501
+ with_git_initialized_project do |p|
502
+ job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
503
+ controlscript_name = "my_control"
504
+ project_name = "myproject"
505
+ status_code = Mortar::API::Jobs::STATUS_RUNNING
506
+ progress = 0
507
+ cluster_id = "e2790e7e8c7d48e39157238d58191346"
508
+ start_timestamp = "2012-02-28T03:35:42.831000+00:00"
509
+ stop_timestamp = "2012-02-28T03:44:52.613000+00:00"
510
+ running_timestamp = "2012-02-28T03:41:52.613000+00:00"
511
+ parameters = {"my_param_1" => "value1", "MY_PARAM_2" => "3"}
512
+
513
+ mock(Mortar::Auth.api).get_job(job_id).returns(Excon::Response.new(:body => {"job_id" => job_id,
514
+ "controlscript_name" => controlscript_name,
515
+ "project_name" => project_name,
516
+ "status_code" => status_code,
517
+ "status_description" => "Execution error",
518
+ "progress" => progress,
519
+ "cluster_id" => cluster_id,
520
+ "start_timestamp" => start_timestamp,
521
+ "running_timestamp" => running_timestamp,
522
+ "duration" => "6 mins",
523
+ "num_hadoop_jobs_succeeded" => 1.3,
524
+ "parameters" => parameters
525
+ }))
526
+
527
+ status_code = Mortar::API::Jobs::STATUS_SUCCESS
528
+ progress = 100
529
+ outputs = [{'name'=> 'hottest_songs_of_the_decade',
530
+ 'records' => 10,
531
+ 'alias' => 'output_data',
532
+ 'location' => 's3n://my-bucket/my-folder/hottest_songs_of_the_decade/output_data'},
533
+ {'name'=> 'hottest_songs_of_the_decade',
534
+ 'records' => 100,
535
+ 'alias' => 'output_data_2',
536
+ 'location' => 's3n://my-bucket/my-folder/hottest_songs_of_the_decade/output_data_2'}]
537
+
538
+ mock(Mortar::Auth.api).get_job(job_id).returns(Excon::Response.new(:body => {"job_id" => job_id,
539
+ "controlscript_name" => controlscript_name,
540
+ "project_name" => project_name,
541
+ "status_code" => status_code,
542
+ "status_description" => "Success",
543
+ "progress" => progress,
544
+ "cluster_id" => cluster_id,
545
+ "start_timestamp" => start_timestamp,
546
+ "running_timestamp" => running_timestamp,
547
+ "stop_timestamp" => stop_timestamp,
548
+ "duration" => "6 mins",
549
+ "num_hadoop_jobs" => 4,
550
+ "num_hadoop_jobs_succeeded" => 4,
551
+ "parameters" => parameters,
552
+ "outputs" => outputs
553
+ }))
554
+ stderr, stdout = execute("jobs:status c571a8c7f76a4fd4a67c103d753e2dd5 -p --polling_interval 0.05", p, @git)
555
+ stdout.should == <<-STDOUT
556
+ \r[/] 1.30 MapReduce Jobs complete.\r\e[0K=== myproject: my_control (job_id: c571a8c7f76a4fd4a67c103d753e2dd5)
557
+ cluster_id: e2790e7e8c7d48e39157238d58191346
558
+ hadoop jobs complete: 4.00 / 4.00
559
+ job began running at: 2012-02-28T03:41:52.613000+00:00
560
+ job finished at: 2012-02-28T03:44:52.613000+00:00
561
+ job run with parameters:
562
+ MY_PARAM_2: 3
563
+ my_param_1: value1
564
+ job running for: 6 mins
565
+ job submitted at: 2012-02-28T03:35:42.831000+00:00
566
+ outputs:
567
+ output_data:
568
+ location: s3n://my-bucket/my-folder/hottest_songs_of_the_decade/output_data
569
+ records: 10
570
+ output_data_2:
571
+ location: s3n://my-bucket/my-folder/hottest_songs_of_the_decade/output_data_2
572
+ records: 100
573
+ progress: 100%
574
+ status: Success
575
+ STDOUT
576
+ end
577
+ end
578
+
461
579
  it "gets status for a running job using polling" do
462
580
  with_git_initialized_project do |p|
463
581
  job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
@@ -0,0 +1,144 @@
1
+ #
2
+ # Copyright 2012 Mortar Data Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'spec_helper'
18
+ require 'fakefs/spec_helpers'
19
+ require 'mortar/command/local'
20
+ require 'launchy'
21
+
22
+ module Mortar::Command
23
+ describe Local do
24
+
25
+ context("illustrate") do
26
+ it "errors when an alias is not provided" do
27
+ with_git_initialized_project do |p|
28
+ write_file(File.join(p.pigscripts_path, "my_script.pig"))
29
+ stderr, stdout = execute("local:illustrate my_script", p)
30
+ stderr.should == <<-STDERR
31
+ ! Usage: mortar local:illustrate PIGSCRIPT ALIAS
32
+ ! Must specify PIGSCRIPT and ALIAS.
33
+ STDERR
34
+ end
35
+ end
36
+
37
+ it "errors when the script doesn't exist" do
38
+ with_git_initialized_project do |p|
39
+ write_file(File.join(p.pigscripts_path, "my_other_script.pig"))
40
+ stderr, stdout = execute("local:illustrate my_script some_alias", p)
41
+ stderr.should == <<-STDERR
42
+ ! Unable to find pigscript my_script
43
+ ! Available scripts:
44
+ ! my_other_script
45
+ STDERR
46
+ end
47
+ end
48
+
49
+ it "calls the illustrate command when envoked correctly" do
50
+ with_git_initialized_project do |p|
51
+ script_name = "some_script"
52
+ script_path = File.join(p.pigscripts_path, "#{script_name}.pig")
53
+ write_file(script_path)
54
+ pigscript = Mortar::Project::PigScript.new(script_name, script_path)
55
+ mock(Mortar::Project::PigScript).new(script_name, script_path).returns(pigscript)
56
+ any_instance_of(Mortar::Local::Controller) do |u|
57
+ mock(u).illustrate(pigscript, "some_alias", [], false).returns(nil)
58
+ end
59
+ stderr, stdout = execute("local:illustrate #{script_name} some_alias", p)
60
+ stderr.should == ""
61
+ end
62
+ end
63
+
64
+ # illustrate
65
+ end
66
+
67
+ context("run") do
68
+
69
+ it "errors when the script doesn't exist" do
70
+ with_git_initialized_project do |p|
71
+ write_file(File.join(p.pigscripts_path, "my_other_script.pig"))
72
+ stderr, stdout = execute("local:run my_script", p)
73
+ stderr.should == <<-STDERR
74
+ ! Unable to find pigscript my_script
75
+ ! Available scripts:
76
+ ! my_other_script
77
+ STDERR
78
+ end
79
+ end
80
+
81
+ it "calls the run command when envoked correctly" do
82
+ with_git_initialized_project do |p|
83
+ script_name = "some_script"
84
+ script_path = File.join(p.pigscripts_path, "#{script_name}.pig")
85
+ write_file(script_path)
86
+ pigscript = Mortar::Project::PigScript.new(script_name, script_path)
87
+ mock(Mortar::Project::PigScript).new(script_name, script_path).returns(pigscript)
88
+ any_instance_of(Mortar::Local::Controller) do |u|
89
+ mock(u).run(pigscript, []).returns(nil)
90
+ end
91
+ stderr, stdout = execute("local:run #{script_name}", p)
92
+ stderr.should == ""
93
+ end
94
+ end
95
+ # run
96
+ end
97
+
98
+ context("configure") do
99
+
100
+ it "errors if java can't be found" do
101
+ any_instance_of(Mortar::Local::Java) do |j|
102
+ stub(j).check_install.returns(false)
103
+ end
104
+ stderr, stdout = execute("local:configure")
105
+ stderr.should == Mortar::Local::Controller::NO_JAVA_ERROR_MESSAGE.gsub(/^/, " ! ")
106
+ end
107
+
108
+ it "errors if python can't be found" do
109
+ any_instance_of(Mortar::Local::Java) do |j|
110
+ stub(j).check_install.returns(true)
111
+ end
112
+ any_instance_of(Mortar::Local::Pig) do |j|
113
+ stub(j).install.returns(true)
114
+ end
115
+ any_instance_of(Mortar::Local::Python) do |j|
116
+ stub(j).check_or_install.returns(false)
117
+ end
118
+ stderr, stdout = execute("local:configure")
119
+ stderr.should == Mortar::Local::Controller::NO_PYTHON_ERROR_MESSAGE.gsub(/^/, " ! ")
120
+ end
121
+
122
+ it "checks for java, installs pig/python, and configures a virtualenv" do
123
+ any_instance_of(Mortar::Local::Java) do |j|
124
+ mock(j).check_install.returns(true)
125
+ end
126
+ any_instance_of(Mortar::Local::Pig) do |j|
127
+ mock(j).install.returns(true)
128
+ end
129
+ any_instance_of(Mortar::Local::Python) do |j|
130
+ mock(j).check_or_install.returns(true)
131
+ end
132
+ any_instance_of(Mortar::Local::Python) do |j|
133
+ mock(j).setup_project_python_environment.returns(true)
134
+ end
135
+ stderr, stdout = execute("local:configure")
136
+ stderr.should == ""
137
+ end
138
+
139
+ # configure
140
+ end
141
+
142
+ end
143
+ end
144
+
@@ -46,8 +46,21 @@ STDERR
46
46
  with_git_initialized_project do |p|
47
47
  stderr, stdout = execute("validate does_not_exist", p, @git)
48
48
  stderr.should == <<-STDERR
49
- ! Unable to find pigscript does_not_exist
49
+ ! Unable to find a pigscript or controlscript for does_not_exist
50
+ !
50
51
  ! No pigscripts found
52
+ !
53
+ ! No controlscripts found
54
+ STDERR
55
+ end
56
+ end
57
+
58
+ it "errors when requested with controlscript" do
59
+ with_git_initialized_project do |p|
60
+ write_file(File.join(p.controlscripts_path, "my_script.py"))
61
+ stderr, stdout = execute("validate my_script", p, @git)
62
+ stderr.should == <<-STDERR
63
+ ! Currently Mortar does not support validating control scripts
51
64
  STDERR
52
65
  end
53
66
  end
@@ -0,0 +1,102 @@
1
+ #
2
+ # Copyright 2012 Mortar Data Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'spec_helper'
18
+ require 'fakefs/spec_helpers'
19
+ require 'mortar/local/controller'
20
+ require 'launchy'
21
+
22
+ module Mortar::Local
23
+ describe Controller do
24
+
25
+ before do
26
+ ENV['AWS_ACCESS_KEY'] = "foo"
27
+ ENV['AWS_SECRET_KEY'] = "BAR"
28
+ end
29
+
30
+ context("aws keys") do
31
+ it "exits if they are not present" do
32
+ ENV.delete('AWS_ACCESS_KEY')
33
+ ctrl = Mortar::Local::Controller.new
34
+ previous_stderr, $stderr = $stderr, StringIO.new
35
+ begin
36
+ expect { ctrl.require_aws_keys }.to raise_error(SystemExit)
37
+ $stderr.string.should eq(Mortar::Local::Controller::NO_AWS_KEYS_ERROR_MESSAGE.gsub(/^/, " ! "))
38
+ ensure
39
+ $stderr = previous_stderr
40
+ end
41
+ end
42
+
43
+ it "returns if they are present" do
44
+ ctrl = Mortar::Local::Controller.new
45
+ previous_stderr, $stderr = $stderr, StringIO.new
46
+ begin
47
+ ctrl.require_aws_keys()
48
+ $stderr.string.should eq("")
49
+ ensure
50
+ $stderr = previous_stderr
51
+ end
52
+ end
53
+
54
+ it "returns if they are not present but override is in place" do
55
+ ENV.delete('AWS_ACCESS_KEY')
56
+ ENV['MORTAR_IGNORE_AWS_KEYS'] = 'true'
57
+ ctrl = Mortar::Local::Controller.new
58
+ previous_stderr, $stderr = $stderr, StringIO.new
59
+ begin
60
+ ctrl.require_aws_keys()
61
+ $stderr.string.should eq("")
62
+ ensure
63
+ $stderr = previous_stderr
64
+ end
65
+ end
66
+
67
+ end
68
+
69
+ context("run") do
70
+
71
+ it "checks for aws keys, checks depenendency installation, runs script" do
72
+ c = Mortar::Local::Controller.new
73
+ mock(c).require_aws_keys
74
+ mock(c).install_and_configure
75
+ test_script = "foobar-script"
76
+ the_parameters = []
77
+ any_instance_of(Mortar::Local::Pig) do |p|
78
+ mock(p).run_script(test_script, the_parameters)
79
+ end
80
+ c.run(test_script, the_parameters)
81
+ end
82
+
83
+ end
84
+
85
+ context("illustrate") do
86
+ it "checks for aws keys, checks depenendency installation, runs the illustrate process" do
87
+ c = Mortar::Local::Controller.new
88
+ mock(c).require_aws_keys
89
+ mock(c).install_and_configure
90
+ test_script = "foobar-script"
91
+ script_alias = "some_alias"
92
+ prune = false
93
+ the_parameters = []
94
+ any_instance_of(Mortar::Local::Pig) do |p|
95
+ mock(p).illustrate_alias(test_script, script_alias, prune, the_parameters)
96
+ end
97
+ c.illustrate(test_script, script_alias, the_parameters, prune)
98
+ end
99
+ end
100
+
101
+ end
102
+ end