mortar 0.6.2 → 0.7.0

data/lib/mortar/auth.rb

@@ -88,6 +88,10 @@ class Mortar::Auth
       get_credentials[1]
     end
 
+    def user_s3_safe
+      return user.gsub(/[^0-9a-zA-Z]/i, '-')
+    end
+
     def api_key(user = get_credentials[0], password = get_credentials[1])
       require("mortar-api-ruby")
       api = Mortar::API.new(default_params)

data/lib/mortar/command/base.rb

@@ -243,6 +243,22 @@ protected
       error("Unable to find git remote for project #{project.name}")
     end
   end
+
+  def validate_script!(script_name)
+    pigscript = project.pigscripts[script_name]
+    controlscript = project.controlscripts[script_name]
+    unless pigscript || controlscript
+      available_pigscripts = project.pigscripts.none? ? "No pigscripts found" : "Available pigscripts:\n#{project.pigscripts.keys.sort.join("\n")}"
+      available_controlscripts = project.controlscripts.none? ? "No controlscripts found" : "Available controlscripts:\n#{project.controlscripts.keys.sort.join("\n")}"
+      error("Unable to find a pigscript or controlscript for #{script_name}\n\n#{available_pigscripts}\n\n#{available_controlscripts}")
+    end
+
+    if pigscript && controlscript
+      error("Naming conflict.  #{script_name} refers to both a pigscript and a controlscript.  Please rename scripts to avoid conflicts.")
+    end
+
+    pigscript or controlscript
+  end
   
   def validate_pigscript!(pigscript_name)
     unless pigscript = project.pigscripts[pigscript_name]

data/lib/mortar/command/describe.rb

@@ -39,9 +39,15 @@ class Mortar::Command::Describe < Mortar::Command::Base
     unless pigscript_name && alias_name
       error("Usage: mortar describe PIGSCRIPT ALIAS\nMust specify PIGSCRIPT and ALIAS.")
     end
+    
     validate_arguments!
+    pigscript = validate_script!(pigscript_name)
+    
+    if pigscript.is_a? Mortar::Project::ControlScript
+      error "Currently Mortar does not support describing control scripts"
+    end
+    
     validate_git_based_project!
-    pigscript = validate_pigscript!(pigscript_name)
     git_ref = git.create_and_push_snapshot_branch(project)
     
     describe_id = nil

data/lib/mortar/command/help.rb

@@ -67,24 +67,6 @@ private
     Mortar::Command.commands
   end
 
-  def legacy_help_for_namespace(namespace)
-    instance = Mortar::Command::Help.groups.map do |group|
-      [ group.title, group.select { |c| c.first =~ /^#{namespace}/ }.length ]
-    end.sort_by { |l| l.last }.last
-    return nil unless instance
-    return nil if instance.last.zero?
-    instance.first
-  end
-
-  def legacy_help_for_command(command)
-    Mortar::Command::Help.groups.each do |group|
-      group.each do |cmd, description|
-        return description if cmd.split(" ").first == command
-      end
-    end
-    nil
-  end
-
   def primary_namespaces
     PRIMARY_NAMESPACES.map { |name| namespaces[name] }.compact
   end
@@ -97,7 +79,6 @@ private
     size = longest(namespaces.map { |n| n[:name] })
     namespaces.sort_by {|namespace| namespace[:name]}.each do |namespace|
       name = namespace[:name]
-      namespace[:description] ||= legacy_help_for_namespace(name)
       puts "  %-#{size}s  # %s" % [ name, namespace[:description] ]
     end
   end
@@ -121,8 +102,6 @@ private
     unless namespace_commands.empty?
       size = longest(namespace_commands.map { |c| c[:banner] })
       namespace_commands.sort_by { |c| c[:banner].to_s }.each do |command|
-        next if command[:help] =~ /DEPRECATED/
-        command[:summary] ||= legacy_help_for_command(command[:command])
         puts "  %-#{size}s  # %s" % [ command[:banner], command[:summary] ]
       end
     end
@@ -140,7 +119,7 @@ private
         puts command[:help].split("\n")[1..-1].join("\n")
       else
         puts
-        puts " " + legacy_help_for_command(name).to_s
+        error "No help available for #{command[:command]}. Please contact us at support@mortardata.com for assistance."
       end
       puts
     end

data/lib/mortar/command/illustrate.rb

@@ -40,16 +40,21 @@ class Mortar::Command::Illustrate < Mortar::Command::Base
     alias_name = shift_argument
     skip_pruning = options[:skippruning] ||= false
     
+    validate_arguments!
+    pigscript = validate_script!(pigscript_name)
+          
     # TODO: When illustrating without alias works, remove the `&& alias_name` to re-enable the feature on CLI
     unless pigscript_name && alias_name
       error("Usage: mortar illustrate PIGSCRIPT ALIAS\nMust specify PIGSCRIPT and ALIAS.")
     end
     
-    validate_arguments!
+    if pigscript.is_a? Mortar::Project::ControlScript
+      error "Currently Mortar does not support illustrating control scripts"
+    end
+    
     validate_git_based_project!
-    pigscript = validate_pigscript!(pigscript_name)
     git_ref = git.create_and_push_snapshot_branch(project)
-    
+
     illustrate_id = nil
     action("Starting illustrate") do
       illustrate_id = api.post_illustrate(project.name, pigscript.name, alias_name, skip_pruning, git_ref, :parameters => pig_parameters).body["illustrate_id"]

data/lib/mortar/command/jobs.rb

@@ -50,7 +50,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
     display_table(jobs, columns, headers)
   end
     
-  # jobs:run PIGSCRIPT
+  # jobs:run SCRIPT
   #
   # Run a job on a Mortar Hadoop cluster.
   #
@@ -66,11 +66,23 @@ class Mortar::Command::Jobs < Mortar::Command::Base
   #    Run the generate_regression_model_coefficients script on a 3 node cluster.
   #        $ mortar jobs:run generate_regression_model_coefficients --clustersize 3
   def run
-    pigscript_name = shift_argument
-    unless pigscript_name
-      error("Usage: mortar jobs:run PIGSCRIPT\nMust specify PIGSCRIPT.")
+    script_name = shift_argument
+    unless script_name
+      error("Usage: mortar jobs:run SCRIPT\nMust specify SCRIPT.")
     end
+    
     validate_arguments!
+    script = validate_script!(script_name)
+    
+    case script
+    when Mortar::Project::PigScript
+      is_control_script = false
+    when Mortar::Project::ControlScript
+      is_control_script = true
+    else
+      error "Unknown Script Type"
+    end
+    
 
     unless options[:clusterid] || options[:clustersize]
       clusters = api.get_clusters().body['clusters']
@@ -98,27 +110,28 @@ class Mortar::Command::Jobs < Mortar::Command::Base
     end
  
     validate_git_based_project!
-    pigscript = validate_pigscript!(pigscript_name)
     git_ref = git.create_and_push_snapshot_branch(project)
     notify_on_job_finish = ! options[:donotnotify]
     
-    # post job to API
+    # post job to API    
     response = action("Requesting job execution") do
       if options[:clustersize]
         cluster_size = options[:clustersize].to_i
         keepalive = ! options[:singlejobcluster]
-        api.post_job_new_cluster(project.name, pigscript.name, git_ref, cluster_size, 
+        api.post_job_new_cluster(project.name, script.name, git_ref, cluster_size, 
           :parameters => pig_parameters,
           :keepalive => keepalive,
-          :notify_on_job_finish => notify_on_job_finish).body
+          :notify_on_job_finish => notify_on_job_finish,
+          :is_control_script => is_control_script).body
       else
         cluster_id = options[:clusterid]
-        api.post_job_existing_cluster(project.name, pigscript.name, git_ref, cluster_id,
+        api.post_job_existing_cluster(project.name, script.name, git_ref, cluster_id,
           :parameters => pig_parameters,
-          :notify_on_job_finish => notify_on_job_finish).body
+          :notify_on_job_finish => notify_on_job_finish,
+          :is_control_script => is_control_script).body
       end
     end
-
+    
     display("job_id: #{response['job_id']}")
     display
     display("Job status can be viewed on the web at:\n\n #{response['web_job_url']}")
@@ -146,7 +159,6 @@ class Mortar::Command::Jobs < Mortar::Command::Base
     def display_job_status(job_status)
       job_display_entries = {
         "status" => job_status["status_description"],
-        "progress" => "#{job_status["progress"]}%",
         "cluster_id" => job_status["cluster_id"],
         "job submitted at" => job_status["start_timestamp"],
         "job began running at" => job_status["running_timestamp"],
@@ -167,8 +179,13 @@ class Mortar::Command::Jobs < Mortar::Command::Base
       end
       
       if job_status["num_hadoop_jobs"] && job_status["num_hadoop_jobs_succeeded"]
+        job_display_entries["progress"] = "#{job_status["progress"]}%"
         job_display_entries["hadoop jobs complete"] = 
           '%0.2f / %0.2f' % [job_status["num_hadoop_jobs_succeeded"], job_status["num_hadoop_jobs"]]
+      elsif job_status["num_hadoop_jobs_succeeded"]
+        job_display_entries["progress"] = '%0.2f MapReduce Jobs complete.' % job_status["num_hadoop_jobs_succeeded"]
+      else
+        job_display_entries["progress"] = "#{job_status["progress"]}%"
       end
       
       if job_status["outputs"] && job_status["outputs"].length > 0
@@ -180,7 +197,13 @@ class Mortar::Command::Jobs < Mortar::Command::Base
         end]
       end
       
-      styled_header("#{job_status["project_name"]}: #{job_status["pigscript_name"]} (job_id: #{job_status["job_id"]})")
+      if job_status["controlscript_name"]
+        script_name = job_status["controlscript_name"]
+      else 
+        script_name = job_status["pigscript_name"]
+      end
+
+      styled_header("#{job_status["project_name"]}: #{script_name} (job_id: #{job_status["job_id"]})")
       styled_hash(job_display_entries)
     end
     
@@ -196,7 +219,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
         end
 
         # If the job is running show the progress bar
-        if job_status["status_code"] == Mortar::API::Jobs::STATUS_RUNNING
+        if job_status["status_code"] == Mortar::API::Jobs::STATUS_RUNNING && job_status["num_hadoop_jobs"]
           progressbar = "=" + ("=" * (job_status["progress"].to_i / 5)) + ">"
 
           if job_status["num_hadoop_jobs"] && job_status["num_hadoop_jobs_succeeded"]
@@ -206,6 +229,10 @@ class Mortar::Command::Jobs < Mortar::Command::Base
 
           printf("\r[#{spinner(ticks)}] Status: [%-22s] %s%% Complete (%s MapReduce jobs finished)", progressbar, job_status["progress"], hadoop_jobs_ratio_complete)
 
+        elsif job_status["status_code"] == Mortar::API::Jobs::STATUS_RUNNING
+          jobs_complete = '%0.2f' % job_status["num_hadoop_jobs_succeeded"]
+          printf("\r[#{spinner(ticks)}] #{jobs_complete} MapReduce Jobs complete.")
+
         # If the job is not complete, but not in the running state, just display its status
         else
           redisplay("[#{spinner(ticks)}] Status: #{job_status['status_description']}")

data/lib/mortar/command/local.rb

@@ -0,0 +1,87 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "mortar/local/controller"
+require "mortar/command/base"
+
+# run select pig commands on your local machine
+#
+class Mortar::Command::Local < Mortar::Command::Base
+
+
+  # configure
+  #
+  # Install dependencies for running this pig project locally, other
+  # commands will also perform this step automatically.
+  #
+  def configure
+    ctrl = Mortar::Local::Controller.new
+    ctrl.install_and_configure
+  end
+
+  # local:run PIGSCRIPT
+  #
+  # Run a job on your local machine
+  #
+  # -p, --parameter NAME=VALUE  # Set a pig parameter value in your script.
+  # -f, --param-file PARAMFILE  # Load pig parameter values from a file.
+  #
+  #Examples:
+  #
+  #    Run the generate_regression_model_coefficients script locally.
+  #        $ mortar local:run generate_regression_model_coefficients
+  def run
+    pigscript_name = shift_argument
+    unless pigscript_name
+      error("Usage: mortar local:run PIGSCRIPT\nMust specify PIGSCRIPT.")
+    end
+    validate_arguments!
+    pigscript = validate_pigscript!(pigscript_name)
+    ctrl = Mortar::Local::Controller.new
+    ctrl.run(pigscript, pig_parameters)
+  end
+
+  # illustrate [PIGSCRIPT] [ALIAS]
+  #
+  # Locallay illustrate the effects and output of a pigscript.
+  #
+  # -s, --skippruning           # Don't try to reduce the illustrate results to the smallest size possible.
+  # -p, --parameter NAME=VALUE  # Set a pig parameter value in your script.
+  # -f, --param-file PARAMFILE  # Load pig parameter values from a file.
+  # --no_browser                # Don't open the illustrate results automatically in the browser.
+  #
+  # Examples:
+  #
+  #     Illustrate the songs_sample relation in the generate_regression_model_coefficients script.
+  #         $ mortar illustrate generate_regression_model_coefficients songs_sample
+  def illustrate
+    pigscript_name = shift_argument
+    alias_name = shift_argument
+    skip_pruning = options[:skippruning] ||= false
+
+    unless pigscript_name && alias_name
+      error("Usage: mortar local:illustrate PIGSCRIPT ALIAS\nMust specify PIGSCRIPT and ALIAS.")
+    end
+
+    validate_arguments!
+    pigscript = validate_pigscript!(pigscript_name)
+
+    ctrl = Mortar::Local::Controller.new
+    ctrl.illustrate(pigscript, alias_name, pig_parameters, skip_pruning)
+  end
+
+
+end

data/lib/mortar/command/projects.rb

@@ -26,7 +26,7 @@ class Mortar::Command::Projects < Mortar::Command::Base
   
   # projects
   #
-  # Display the available set of projects
+  # Display the available set of Mortar projects.
   def index
     validate_arguments!
     projects = api.get_projects().body["projects"]
@@ -40,7 +40,7 @@ class Mortar::Command::Projects < Mortar::Command::Base
   
   # projects:delete PROJECTNAME
   #
-  # Delete a mortar project
+  # Delete the Mortar project PROJECTNAME.
   def delete
     name = shift_argument
     unless name
@@ -58,7 +58,7 @@ class Mortar::Command::Projects < Mortar::Command::Base
   
   # projects:create PROJECTNAME
   #
-  # Generate and register a new Mortar project for code in the current directory, with the name PROJECTNAME.
+  # Used when you want to start a new Mortar project using Mortar generated code.
   def create
     name = shift_argument
     unless name
@@ -75,9 +75,9 @@ class Mortar::Command::Projects < Mortar::Command::Base
   end
   alias_command "new", "projects:create"
   
-  # projects:register PROJECT
+  # projects:register PROJECTNAME
   #
-  # register a mortar project for the current directory with the name PROJECT
+  # Used when you want to start a new Mortar project using your existing code in the current directory.
   def register
     name = shift_argument
     unless name
@@ -145,10 +145,12 @@ class Mortar::Command::Projects < Mortar::Command::Base
   end
   alias_command "register", "projects:register"
 
-  # projects:set_remote PROJECT
-  #
-  # Adds the Mortar remote to the local git project. This is necessary for successfully executing many of the Mortar commands.
+  # projects:set_remote PROJECTNAME
   #
+  # Used after you checkout code for an existing Mortar project from a non-Mortar git repository.  
+  # Adds a remote to your local git repository to the Mortar git repository.  For example if a 
+  # co-worker creates a Mortar project from an internal repository you would clone the internal
+  # repository and then after cloning call mortar projects:set_remote.
   def set_remote
     project_name = shift_argument
 
@@ -176,10 +178,9 @@ class Mortar::Command::Projects < Mortar::Command::Base
 
   end
   
-  # projects:clone PROJECT
-  #
-  # clone the mortar project PROJECT into the current directory.
+  # projects:clone PROJECTNAME
   #
+  # Used when you want to clone an existing Mortar project into the current directory.
   def clone
     name = shift_argument
     unless name

data/lib/mortar/command/validate.rb

@@ -36,8 +36,13 @@ class Mortar::Command::Validate < Mortar::Command::Base
       error("Usage: mortar validate PIGSCRIPT\nMust specify PIGSCRIPT.")
     end
     validate_arguments!
+    pigscript = validate_script!(pigscript_name)
+    
+    if pigscript.is_a? Mortar::Project::ControlScript
+      error "Currently Mortar does not support validating control scripts"
+    end
+    
     validate_git_based_project!
-    pigscript = validate_pigscript!(pigscript_name)
     git_ref = git.create_and_push_snapshot_branch(project)
     
     validate_id = nil

data/lib/mortar/helpers.rb

@@ -494,6 +494,18 @@ module Mortar
       end
     end
 
+    def ensure_dir_exists(dir)
+      unless Dir.exists? dir
+        Dir.mkdir(dir)
+      end
+    end
+
+    def copy_if_not_present_at_dest(res_src, res_dest)
+      unless File.exists?(res_dest)
+        FileUtils.cp(res_src, res_dest)
+      end
+    end
+
     private
 
       def create_display_method(name, colour_code, new_line=true)

data/lib/mortar/local/controller.rb

@@ -0,0 +1,108 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "mortar/helpers"
+require "mortar/local/pig"
+require "mortar/local/java"
+require "mortar/local/python"
+
+
+class Mortar::Local::Controller
+  include Mortar::Helpers
+
+  NO_JAVA_ERROR_MESSAGE = <<EOF
+A suitable java installation could not be found.  If you already have java installed
+please set your JAVA_HOME environment variable before continuing.  Otherwise, a
+suitable java installation will need to be added to your local system.
+
+Installing Java
+On OSX run `javac` from the command line.  This will intiate the installation.  For
+Linux systems please consult the documentation on your relevant package manager.
+EOF
+
+  NO_PYTHON_ERROR_MESSAGE = <<EOF
+pA suitable python installation with virtualenv could not be located.  Please ensure
+you have python 2.6+ installed on your local system.  If you need to obtain a copy
+of virtualenv it can be located here:
+https://pypi.python.org/pypi/virtualenv
+EOF
+
+  NO_AWS_KEYS_ERROR_MESSAGE = <<EOF
+Please specify your aws access key via enviroment variable AWS_ACCESS_KEY
+and your aws secret key via enviroment variable AWS_SECRET_KEY"
+EOF
+
+
+  # Checks if the user has properly specified their AWS keys
+  def verify_aws_keys()
+    if (not (ENV['AWS_ACCESS_KEY'] and ENV['AWS_SECRET_KEY'])) then
+      if not ENV['MORTAR_IGNORE_AWS_KEYS']
+        return false
+      else
+        return true
+      end
+    else
+      return true
+    end
+  end
+
+  # Exits with a helpful message if the user has not setup their aws keys
+  def require_aws_keys()
+    unless verify_aws_keys()
+      error(NO_AWS_KEYS_ERROR_MESSAGE)
+    end
+  end
+
+  # Main entry point to perform installation and configuration necessary
+  # to run pig on the users local machine
+  def install_and_configure
+    java = Mortar::Local::Java.new()
+    unless java.check_install
+      error(NO_JAVA_ERROR_MESSAGE)
+    end
+
+    pig = Mortar::Local::Pig.new()
+    pig.install()
+
+    py = Mortar::Local::Python.new()
+    unless py.check_or_install
+      error(NO_PYTHON_ERROR_MESSAGE)
+    end
+
+    unless py.setup_project_python_environment
+      msg = "\nUnable to setup a python environment with your dependencies, "
+      msg += "see #{py.pip_error_log_path} for more details"
+      error(msg)
+    end
+  end
+
+  # Main entry point for user running a pig script
+  def run(pig_script, pig_parameters)
+    require_aws_keys
+    install_and_configure
+    pig = Mortar::Local::Pig.new()
+    pig.run_script(pig_script, pig_parameters)
+  end
+
+  # Main entry point for illustrating a pig alias
+  def illustrate(pig_script, pig_alias, pig_parameters, skip_pruning)
+    require_aws_keys
+    install_and_configure
+    pig = Mortar::Local::Pig.new()
+    pig.illustrate_alias(pig_script, pig_alias, skip_pruning, pig_parameters)
+  end
+
+end