mortar 0.6.2 → 0.7.0

data/spec/mortar/command/jobs_spec.rb

@@ -49,7 +49,8 @@ module Mortar::Command
           mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]), 
             :keepalive => false,
-            :notify_on_job_finish => true) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
+            :notify_on_job_finish => true,
+            :is_control_script=> false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
 
           write_file(File.join(p.pigscripts_path, "my_script.pig"))
           stderr, stdout = execute("jobs:run my_script -1 --clustersize 5 -p FIRST_PARAM=FOO -p SECOND_PARAM=BAR", p, @git)
@@ -81,7 +82,8 @@ STDOUT
           mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]), 
             :keepalive => true,
-            :notify_on_job_finish => true) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
+            :notify_on_job_finish => true,
+            :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
 
           write_file(File.join(p.pigscripts_path, "my_script.pig"))
           stderr, stdout = execute("jobs:run my_script --clustersize 5 -p FIRST_PARAM=FOO -p SECOND_PARAM=BAR", p, @git)
@@ -91,6 +93,38 @@ Sending code snapshot to Mortar... done
 Requesting job execution... done
 job_id: c571a8c7f76a4fd4a67c103d753e2dd5
 
+Job status can be viewed on the web at:
+
+ http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
+
+Or by running:
+
+  mortar jobs:status c571a8c7f76a4fd4a67c103d753e2dd5 --poll
+
+STDOUT
+        end
+      end
+      
+      it "runs a control script" do
+        with_git_initialized_project do |p|
+          # stub api requests
+          job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
+          job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
+          cluster_id = "e2790e7e8c7d48e39157238d58191346"
+
+          mock(Mortar::Auth.api).post_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id,
+              :parameters => [],
+              :notify_on_job_finish => false,
+              :is_control_script=>true) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
+
+          write_file(File.join(p.controlscripts_path, "my_script.py"))
+          stderr, stdout = execute("jobs:run my_script --clusterid e2790e7e8c7d48e39157238d58191346 -d", p, @git)
+          stdout.should == <<-STDOUT
+Taking code snapshot... done
+Sending code snapshot to Mortar... done
+Requesting job execution... done
+job_id: c571a8c7f76a4fd4a67c103d753e2dd5
+
 Job status can be viewed on the web at:
 
  http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
@@ -113,7 +147,8 @@ STDOUT
           mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :parameters => [], 
             :keepalive => true,
-            :notify_on_job_finish => true) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
+            :notify_on_job_finish => true,
+            :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
 
           write_file(File.join(p.pigscripts_path, "my_script.pig"))
           stderr, stdout = execute("jobs:run my_script ", p, @git)
@@ -144,7 +179,7 @@ STDOUT
           job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
           cluster_id = "e2790e7e8c7d48e39157238d58191346"
 
-          mock(Mortar::Auth.api).post_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id, :parameters => [], :notify_on_job_finish => false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
+          mock(Mortar::Auth.api).post_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id, :parameters => [], :notify_on_job_finish => false, :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
 
           write_file(File.join(p.pigscripts_path, "my_script.pig"))
           stderr, stdout = execute("jobs:run my_script --clusterid e2790e7e8c7d48e39157238d58191346 -d", p, @git)
@@ -198,7 +233,8 @@ STDOUT
           }
           mock(Mortar::Auth.api).post_job_existing_cluster("myproject", "my_script", is_a(String), large_cluster_id, 
             :parameters => [], 
-            :notify_on_job_finish => true) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
+            :notify_on_job_finish => true,
+            :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
 
           write_file(File.join(p.pigscripts_path, "my_script.pig"))
           stderr, stdout = execute("jobs:run my_script ", p, @git)
@@ -231,7 +267,8 @@ STDOUT
           mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :parameters => match_array([{"name" => "FIRST", "value" => "FOO"}, {"name" => "SECOND", "value" => "BAR"}, {"name" => "THIRD", "value" => "BEAR\n"}]), 
             :keepalive => true,
-            :notify_on_job_finish => true) {Excon::Response.new(:body => {"job_id" => job_id})}
+            :notify_on_job_finish => true,
+            :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id})}
 
           write_file(File.join(p.pigscripts_path, "my_script.pig"))
 
@@ -256,7 +293,8 @@ PARAMS
           mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :parameters => match_array([{"name" => "FIRST", "value" => "FOO"}, {"name" => "SECOND", "value" => "BAR"}, {"name" => "THIRD", "value" => "BEAR\n"}]), 
             :keepalive => true,
-            :notify_on_job_finish => true) {Excon::Response.new(:body => {"job_id" => job_id})}
+            :notify_on_job_finish => true,
+            :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id})}
 
           write_file(File.join(p.pigscripts_path, "my_script.pig"))
 
@@ -458,6 +496,86 @@ status:                  Execution error
 STDOUT
         end
       end
+
+      it "gets status for a control job" do
+        with_git_initialized_project do |p|
+          job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
+          controlscript_name = "my_control"
+          project_name = "myproject"
+          status_code = Mortar::API::Jobs::STATUS_RUNNING
+          progress = 0
+          cluster_id = "e2790e7e8c7d48e39157238d58191346"
+          start_timestamp = "2012-02-28T03:35:42.831000+00:00"
+          stop_timestamp = "2012-02-28T03:44:52.613000+00:00"
+          running_timestamp = "2012-02-28T03:41:52.613000+00:00"
+          parameters = {"my_param_1" => "value1", "MY_PARAM_2" => "3"}
+
+          mock(Mortar::Auth.api).get_job(job_id).returns(Excon::Response.new(:body => {"job_id" => job_id,
+            "controlscript_name" => controlscript_name,
+            "project_name" => project_name,
+            "status_code" => status_code,
+            "status_description" => "Execution error",
+            "progress" => progress,
+            "cluster_id" => cluster_id,
+            "start_timestamp" => start_timestamp,
+            "running_timestamp" => running_timestamp,
+            "duration" => "6 mins",
+            "num_hadoop_jobs_succeeded" => 1.3,
+            "parameters" => parameters
+            }))
+
+          status_code = Mortar::API::Jobs::STATUS_SUCCESS
+          progress = 100
+          outputs = [{'name'=> 'hottest_songs_of_the_decade', 
+                      'records' => 10, 
+                      'alias' => 'output_data',
+                      'location' => 's3n://my-bucket/my-folder/hottest_songs_of_the_decade/output_data'},
+                     {'name'=> 'hottest_songs_of_the_decade', 
+                      'records' => 100,
+                      'alias' => 'output_data_2',
+                      'location' => 's3n://my-bucket/my-folder/hottest_songs_of_the_decade/output_data_2'}]
+
+          mock(Mortar::Auth.api).get_job(job_id).returns(Excon::Response.new(:body => {"job_id" => job_id,
+            "controlscript_name" => controlscript_name,
+            "project_name" => project_name,
+            "status_code" => status_code,
+            "status_description" => "Success",
+            "progress" => progress,
+            "cluster_id" => cluster_id,
+            "start_timestamp" => start_timestamp,
+            "running_timestamp" => running_timestamp,
+            "stop_timestamp" => stop_timestamp,
+            "duration" => "6 mins",
+            "num_hadoop_jobs" => 4,
+            "num_hadoop_jobs_succeeded" => 4,
+            "parameters" => parameters,
+            "outputs" => outputs
+            }))
+          stderr, stdout = execute("jobs:status c571a8c7f76a4fd4a67c103d753e2dd5 -p --polling_interval 0.05", p, @git)
+          stdout.should == <<-STDOUT
+\r[/] 1.30 MapReduce Jobs complete.\r\e[0K=== myproject: my_control (job_id: c571a8c7f76a4fd4a67c103d753e2dd5)
+cluster_id:              e2790e7e8c7d48e39157238d58191346
+hadoop jobs complete:    4.00 / 4.00
+job began running at:    2012-02-28T03:41:52.613000+00:00
+job finished at:         2012-02-28T03:44:52.613000+00:00
+job run with parameters: 
+  MY_PARAM_2:   3
+  my_param_1:   value1
+job running for:         6 mins
+job submitted at:        2012-02-28T03:35:42.831000+00:00
+outputs:                 
+  output_data:   
+    location:     s3n://my-bucket/my-folder/hottest_songs_of_the_decade/output_data
+    records:      10
+  output_data_2: 
+    location:     s3n://my-bucket/my-folder/hottest_songs_of_the_decade/output_data_2
+    records:      100
+progress:                100%
+status:                  Success
+STDOUT
+        end
+      end
+
       it "gets status for a running job using polling" do
         with_git_initialized_project do |p|
           job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"

data/spec/mortar/command/local_spec.rb

@@ -0,0 +1,144 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'spec_helper'
+require 'fakefs/spec_helpers'
+require 'mortar/command/local'
+require 'launchy'
+
+module Mortar::Command
+  describe Local do
+
+    context("illustrate") do
+      it "errors when an alias is not provided" do
+        with_git_initialized_project do |p|
+          write_file(File.join(p.pigscripts_path, "my_script.pig"))
+          stderr, stdout = execute("local:illustrate my_script", p)
+          stderr.should == <<-STDERR
+ !    Usage: mortar local:illustrate PIGSCRIPT ALIAS
+ !    Must specify PIGSCRIPT and ALIAS.
+STDERR
+        end
+      end
+
+      it "errors when the script doesn't exist" do
+        with_git_initialized_project do |p|
+          write_file(File.join(p.pigscripts_path, "my_other_script.pig"))
+          stderr, stdout = execute("local:illustrate my_script some_alias", p)
+          stderr.should == <<-STDERR
+ !    Unable to find pigscript my_script
+ !    Available scripts:
+ !    my_other_script
+STDERR
+        end
+      end
+
+      it "calls the illustrate command when envoked correctly" do
+        with_git_initialized_project do |p|
+          script_name = "some_script"
+          script_path = File.join(p.pigscripts_path, "#{script_name}.pig")
+          write_file(script_path)
+          pigscript = Mortar::Project::PigScript.new(script_name, script_path)
+          mock(Mortar::Project::PigScript).new(script_name, script_path).returns(pigscript)
+          any_instance_of(Mortar::Local::Controller) do |u|
+            mock(u).illustrate(pigscript, "some_alias", [], false).returns(nil)
+          end
+          stderr, stdout = execute("local:illustrate #{script_name} some_alias", p)
+          stderr.should == ""
+        end
+      end
+
+    # illustrate
+    end
+
+    context("run") do
+
+      it "errors when the script doesn't exist" do
+        with_git_initialized_project do |p|
+          write_file(File.join(p.pigscripts_path, "my_other_script.pig"))
+          stderr, stdout = execute("local:run my_script", p)
+          stderr.should == <<-STDERR
+ !    Unable to find pigscript my_script
+ !    Available scripts:
+ !    my_other_script
+STDERR
+        end
+      end
+
+      it "calls the run command when envoked correctly" do
+        with_git_initialized_project do |p|
+          script_name = "some_script"
+          script_path = File.join(p.pigscripts_path, "#{script_name}.pig")
+          write_file(script_path)
+          pigscript = Mortar::Project::PigScript.new(script_name, script_path)
+          mock(Mortar::Project::PigScript).new(script_name, script_path).returns(pigscript)
+          any_instance_of(Mortar::Local::Controller) do |u|
+            mock(u).run(pigscript, []).returns(nil)
+          end
+          stderr, stdout = execute("local:run #{script_name}", p)
+          stderr.should == ""
+        end
+      end
+    # run
+    end
+
+    context("configure") do
+
+      it "errors if java can't be found" do
+        any_instance_of(Mortar::Local::Java) do |j|
+          stub(j).check_install.returns(false)
+        end
+        stderr, stdout = execute("local:configure")
+        stderr.should == Mortar::Local::Controller::NO_JAVA_ERROR_MESSAGE.gsub(/^/, " !    ")
+      end
+
+      it "errors if python can't be found" do
+        any_instance_of(Mortar::Local::Java) do |j|
+          stub(j).check_install.returns(true)
+        end
+        any_instance_of(Mortar::Local::Pig) do |j|
+          stub(j).install.returns(true)
+        end
+        any_instance_of(Mortar::Local::Python) do |j|
+          stub(j).check_or_install.returns(false)
+        end
+        stderr, stdout = execute("local:configure")
+        stderr.should == Mortar::Local::Controller::NO_PYTHON_ERROR_MESSAGE.gsub(/^/, " !    ")
+      end
+
+      it "checks for java, installs pig/python, and configures a virtualenv" do
+        any_instance_of(Mortar::Local::Java) do |j|
+          mock(j).check_install.returns(true)
+        end
+        any_instance_of(Mortar::Local::Pig) do |j|
+          mock(j).install.returns(true)
+        end
+        any_instance_of(Mortar::Local::Python) do |j|
+          mock(j).check_or_install.returns(true)
+        end
+        any_instance_of(Mortar::Local::Python) do |j|
+          mock(j).setup_project_python_environment.returns(true)
+        end
+        stderr, stdout = execute("local:configure")
+        stderr.should == ""
+      end
+
+    # configure
+    end
+
+  end
+end
+

data/spec/mortar/command/validate_spec.rb

@@ -46,8 +46,21 @@ STDERR
         with_git_initialized_project do |p|
           stderr, stdout = execute("validate does_not_exist", p, @git)
           stderr.should == <<-STDERR
- !    Unable to find pigscript does_not_exist
+ !    Unable to find a pigscript or controlscript for does_not_exist
+ !    
  !    No pigscripts found
+ !    
+ !    No controlscripts found
+ STDERR
+        end
+      end
+      
+      it "errors when requested with controlscript" do
+        with_git_initialized_project do |p|
+          write_file(File.join(p.controlscripts_path, "my_script.py"))
+          stderr, stdout = execute("validate my_script", p, @git)
+          stderr.should == <<-STDERR
+ !    Currently Mortar does not support validating control scripts
  STDERR
         end
       end

data/spec/mortar/local/controller_spec.rb

@@ -0,0 +1,102 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'spec_helper'
+require 'fakefs/spec_helpers'
+require 'mortar/local/controller'
+require 'launchy'
+
+module Mortar::Local
+  describe Controller do
+
+    before do
+      ENV['AWS_ACCESS_KEY'] = "foo"
+      ENV['AWS_SECRET_KEY'] = "BAR"
+    end
+
+    context("aws keys") do
+      it "exits if they are not present" do
+        ENV.delete('AWS_ACCESS_KEY')
+        ctrl = Mortar::Local::Controller.new
+        previous_stderr, $stderr = $stderr, StringIO.new
+        begin
+          expect { ctrl.require_aws_keys }.to raise_error(SystemExit)
+          $stderr.string.should eq(Mortar::Local::Controller::NO_AWS_KEYS_ERROR_MESSAGE.gsub(/^/, " !    "))
+        ensure
+          $stderr = previous_stderr
+        end
+      end
+
+      it "returns if they are present" do
+        ctrl = Mortar::Local::Controller.new
+        previous_stderr, $stderr = $stderr, StringIO.new
+        begin
+          ctrl.require_aws_keys()
+          $stderr.string.should eq("")
+        ensure
+          $stderr = previous_stderr
+        end
+      end
+
+      it "returns if they are not present but override is in place" do
+        ENV.delete('AWS_ACCESS_KEY')
+        ENV['MORTAR_IGNORE_AWS_KEYS'] = 'true'
+        ctrl = Mortar::Local::Controller.new
+        previous_stderr, $stderr = $stderr, StringIO.new
+        begin
+          ctrl.require_aws_keys()
+          $stderr.string.should eq("")
+        ensure
+          $stderr = previous_stderr
+        end
+      end
+
+    end
+
+    context("run") do
+
+      it "checks for aws keys, checks depenendency installation, runs script" do
+        c = Mortar::Local::Controller.new
+        mock(c).require_aws_keys
+        mock(c).install_and_configure
+        test_script = "foobar-script"
+        the_parameters = []
+        any_instance_of(Mortar::Local::Pig) do |p|
+          mock(p).run_script(test_script, the_parameters)
+        end
+        c.run(test_script, the_parameters)
+      end
+
+    end
+
+    context("illustrate") do
+      it "checks for aws keys, checks depenendency installation, runs the illustrate process" do
+        c = Mortar::Local::Controller.new
+        mock(c).require_aws_keys
+        mock(c).install_and_configure
+        test_script = "foobar-script"
+        script_alias = "some_alias"
+        prune = false
+        the_parameters = []
+        any_instance_of(Mortar::Local::Pig) do |p|
+          mock(p).illustrate_alias(test_script, script_alias, prune, the_parameters)
+        end
+        c.illustrate(test_script, script_alias, the_parameters, prune)
+      end
+    end
+
+  end
+end