mortar 0.7.1 → 0.7.2

data/lib/mortar/command/generate.rb

@@ -18,13 +18,14 @@ require "mortar/generators/project_generator"
 require "mortar/generators/udf_generator"
 require "mortar/generators/pigscript_generator"
 require "mortar/generators/macro_generator"
+require "mortar/generators/controlscript_generator"
 require "mortar/command/base"
 
 # generate mortar code (project, pigscript, python_udf, macro)
 #
 class Mortar::Command::Generate < Mortar::Command::Base
 
-  # generate:project [PROJECTNAME]
+  # generate:project PROJECTNAME
   #
   # Generate the files and directory structure necessary for a Mortar project.
   # 
@@ -39,7 +40,7 @@ class Mortar::Command::Generate < Mortar::Command::Base
   end
   alias_command "generate:project", "generate:_project"
   
-  # generate:python_udf [UDFNAME]
+  # generate:python_udf UDFNAME
   #
   # Generate a new python user defined function
   # 
@@ -52,7 +53,24 @@ class Mortar::Command::Generate < Mortar::Command::Base
     udf_generator.generate_python_udf(udf_name, project, options)
   end
 
-  # generate:pigscript [SCRIPTNAME]
+  # generate:controlscript SCRIPTNAME
+  #
+  # Generate new control script.
+  #
+  def controlscript
+    script_name = shift_argument
+    unless script_name
+      error("Usage: mortar generate:controlscript SCRIPTNAME\nMust specify SCRIPTNAME.")
+    end
+
+    # Validates the we're in a mortar project
+    project
+
+    script_generator = Mortar::Generators::ControlscriptGenerator.new
+    script_generator.generate_controlscript(script_name, options)
+  end
+
+  # generate:pigscript SCRIPTNAME
   #
   # Generate new pig script.
   #
@@ -69,7 +87,7 @@ class Mortar::Command::Generate < Mortar::Command::Base
     script_generator.generate_pigscript(script_name, project, options)
   end
 
-  # generate:macro [MACRONAME]
+  # generate:macro MACRONAME
   #
   # Generate a new pig macro.
   #

data/lib/mortar/command/jobs.rb

@@ -189,12 +189,13 @@ class Mortar::Command::Jobs < Mortar::Command::Base
       end
       
       if job_status["outputs"] && job_status["outputs"].length > 0
-        job_display_entries["outputs"] = Hash[job_status["outputs"].select{|o| o["alias"]}.collect do |output|
+        job_display_entries["outputs"] = Hash.new { |h,k| h[k] = [] }
+        job_status["outputs"].select{|o| o["alias"]}.collect{ |output|
           output_hash = {}
           output_hash["location"] = output["location"] if output["location"]
           output_hash["records"] = output["records"] if output["records"]
           [output['alias'], output_hash]
-        end]
+        }.each{ |k,v| job_display_entries["outputs"][k] << v }
       end
       
       if job_status["controlscript_name"]

data/lib/mortar/command/local.rb

@@ -32,7 +32,7 @@ class Mortar::Command::Local < Mortar::Command::Base
     ctrl.install_and_configure
   end
 
-  # local:run PIGSCRIPT
+  # local:run SCRIPT
   #
   # Run a job on your local machine
   #
@@ -44,14 +44,14 @@ class Mortar::Command::Local < Mortar::Command::Base
   #    Run the generate_regression_model_coefficients script locally.
   #        $ mortar local:run generate_regression_model_coefficients
   def run
-    pigscript_name = shift_argument
-    unless pigscript_name
-      error("Usage: mortar local:run PIGSCRIPT\nMust specify PIGSCRIPT.")
+    script_name = shift_argument
+    unless script_name
+      error("Usage: mortar local:run SCRIPT\nMust specify SCRIPT.")
     end
     validate_arguments!
-    pigscript = validate_pigscript!(pigscript_name)
+    script = validate_script!(script_name)
     ctrl = Mortar::Local::Controller.new
-    ctrl.run(pigscript, pig_parameters)
+    ctrl.run(script, pig_parameters)
   end
 
   # illustrate [PIGSCRIPT] [ALIAS]

data/lib/mortar/command/projects.rb

@@ -71,7 +71,6 @@ class Mortar::Command::Projects < Mortar::Command::Base
     git.git("add .")
     git.git("commit -m \"Mortar project scaffolding\"")
     Mortar::Command::run("projects:register", [name])
-    git.git("push mortar master")
   end
   alias_command "new", "projects:create"
   
@@ -137,6 +136,7 @@ class Mortar::Command::Projects < Mortar::Command::Base
       error("Project registration failed.\nError message: #{project_result['error_message']}")
     when Mortar::API::Projects::STATUS_ACTIVE
       git.remote_add("mortar", project_result['git_url'])
+      git.push_master
       display "Your project is ready for use.  Type 'mortar help' to see the commands you can perform on the project.\n\n"
     else
       raise RuntimeError, "Unknown project status: #{project_status} for project_id: #{project_id}"

data/lib/mortar/generators/controlscript_generator.rb

@@ -0,0 +1,37 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "mortar/generators/generator_base"
+module Mortar
+  module Generators
+    class ControlscriptGenerator < Base
+
+      def generate_controlscript(script_name, options)
+        set_script_binding(script_name, options)
+
+        generate_file "controlscript.py", "controlscripts/#{script_name}.py", :recursive => true
+      end
+
+      protected
+
+        def set_script_binding(script_name, options)
+          options = options
+          script_name = script_name
+          @script_binding = binding
+        end
+    end
+  end
+end

data/lib/mortar/generators/project_generator.rb

@@ -38,6 +38,12 @@ module Mortar
           inside "pigscripts" do
             generate_file "pigscript.pig", "#{project_name}.pig"
           end
+
+          mkdir "controlscripts" 
+
+          inside "controlscripts" do
+            copy_file "gitkeep", ".gitkeep"
+          end
           
           mkdir "macros"
           

data/lib/mortar/git.rb

@@ -81,12 +81,45 @@ module Mortar
         end
         output
       end
+
+      def push_master
+        unless has_commits?
+          raise GitError, "No commits found in repository.  You must do an initial commit to initialize the repository."
+        end
+
+        safe_copy do
+          did_stash_changes = stash_working_dir("Stash for push to master")
+          git('push mortar master')
+        end
+
+      end
+
+      #
+      # Create a safe copy of the git directory
+      #
+
+      def safe_copy(&block)
+        # Copy code into a temp directory so we don't confuse editors while snapshotting
+        curdir = Dir.pwd
+        tmpdir = Dir.mktmpdir
+        FileUtils.cp_r(Dir.glob('*', File::FNM_DOTMATCH) - ['.', '..'], tmpdir)
+        Dir.chdir(tmpdir)
+
+        if block
+          yield
+          FileUtils.remove_entry_secure(tmpdir)
+          Dir.chdir(curdir)
+        else
+          return tmpdir
+        end
+      end
     
       #    
       # snapshot
       #
 
       def create_snapshot_branch
+
         # TODO: handle Ctrl-C in the middle
         # TODO: can we do the equivalent of stash without changing the working directory
         unless has_commits?
@@ -95,9 +128,7 @@ module Mortar
 
         # Copy code into a temp directory so we don't confuse editors while snapshotting
         curdir = Dir.pwd
-        tmpdir = Dir.mktmpdir
-        FileUtils.cp_r(Dir.glob('*', File::FNM_DOTMATCH) - ['.', '..'], tmpdir)
-        Dir.chdir(tmpdir)
+        tmpdir = safe_copy
       
         starting_branch = current_branch
         snapshot_branch = "mortar-snapshot-#{Mortar::UUID.create_random.to_s}"

data/lib/mortar/helpers.rb

@@ -422,10 +422,14 @@ module Mortar
             next
           else
             elements = value.sort {|x,y| x.to_s <=> y.to_s}
-            display_with_indent(indent, "#{key}: ".ljust(max_key_length), false)
-            display_with_indent(indent, elements[0])
-            elements[1..-1].each do |element|
-              display_with_indent(indent, "#{' ' * max_key_length}#{element}")
+            display_with_indent(indent, "#{key}: ".ljust(max_key_length))
+            elements[0..-1].each do |element|
+              case element
+              when Hash
+                styled_hash(element, nil, indent + 2)
+              else
+                display_with_indent(indent, "#{' ' * max_key_length}#{element}")
+              end
             end
             if elements.length > 1
               display
@@ -495,7 +499,7 @@ module Mortar
     end
 
     def ensure_dir_exists(dir)
-      unless Dir.exists? dir
+      unless File.directory? dir
         Dir.mkdir(dir)
       end
     end

data/lib/mortar/local/controller.rb

@@ -18,6 +18,7 @@ require "mortar/helpers"
 require "mortar/local/pig"
 require "mortar/local/java"
 require "mortar/local/python"
+require "mortar/local/jython"
 
 
 class Mortar::Local::Controller
@@ -34,7 +35,7 @@ Linux systems please consult the documentation on your relevant package manager.
 EOF
 
   NO_PYTHON_ERROR_MESSAGE = <<EOF
-pA suitable python installation with virtualenv could not be located.  Please ensure
+A suitable python installation with virtualenv could not be located.  Please ensure
 you have python 2.6+ installed on your local system.  If you need to obtain a copy
 of virtualenv it can be located here:
 https://pypi.python.org/pypi/virtualenv
@@ -75,7 +76,7 @@ EOF
     end
 
     pig = Mortar::Local::Pig.new()
-    pig.install()
+    pig.install_or_update()
 
     py = Mortar::Local::Python.new()
     unless py.check_or_install
@@ -87,6 +88,9 @@ EOF
       msg += "see #{py.pip_error_log_path} for more details"
       error(msg)
     end
+
+    jy = Mortar::Local::Jython.new()
+    jy.install_or_update()
   end
 
   # Main entry point for user running a pig script

data/lib/mortar/local/installutil.rb

@@ -16,6 +16,7 @@
 
 require 'zlib'
 require 'excon'
+require 'time'
 require 'rbconfig'
 require 'rubygems/package'
 
@@ -33,6 +34,13 @@ module Mortar
         File.join(Dir.getwd, ".mortar-local")
       end
 
+      def jython_directory
+        local_install_directory + "/jython"
+      end
+
+      def jython_cache_directory
+        jython_directory + "/cachedir"
+      end
 
       # Drops a marker file for an installed package, used
       # to help determine if updates should be performed
@@ -89,6 +97,28 @@ module Mortar
         return os_platform_name.start_with?('darwin')
       end
 
+      def http_date_to_epoch(date_str)
+        return Time.httpdate(date_str).to_i
+      end
+
+      def url_date(url)
+        result = Excon.head(url)
+        http_date_to_epoch(result.get_header('Last-Modified'))
+      end
+
+      # Given a subdirectory where we have installed some software
+      # and a url to the tgz file it's sourced from, check if the
+      # remote version is newer than the installed version
+      def is_newer_version(subdir, url)
+        existing_install_date = install_date(subdir)
+        if not existing_install_date then
+          # There is no existing install
+          return true
+        end
+        remote_archive_date = url_date(url)
+        return existing_install_date < remote_archive_date
+      end
+
     end
   end
 end

data/lib/mortar/local/jython.rb

@@ -0,0 +1,63 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "mortar/local/installutil"
+
+class Mortar::Local::Jython
+  include Mortar::Local::InstallUtil
+
+  JYTHON_VERSION = '2.5.2'
+  JYTHON_JAR_NAME = 'jython_installer-' + JYTHON_VERSION + '.jar'
+  JYTHON_JAR_DIR = "http://s3.amazonaws.com/hawk-dev-software-mirror/jython/jython-2.5.2/"
+
+  def install_or_update
+    if should_install
+      action("Installing jython") do
+        install
+      end
+    elsif should_update
+      action("Updating jython") do
+        update
+      end
+    end
+  end
+
+  def should_install
+    not File.exists?(jython_directory)
+  end
+
+  def install
+    unless File.exists?(local_install_directory + '/' + JYTHON_JAR_NAME)
+        download_file(JYTHON_JAR_DIR + JYTHON_JAR_NAME, local_install_directory)
+    end
+
+    `$JAVA_HOME/bin/java -jar #{local_install_directory + '/' + JYTHON_JAR_NAME} -s -d #{jython_directory}`
+    FileUtils.mkdir_p jython_cache_directory
+    FileUtils.chmod_R 0777, jython_cache_directory
+
+    FileUtils.rm(local_install_directory + '/' + JYTHON_JAR_NAME)
+    note_install('jython')
+  end
+
+  def should_update
+    return is_newer_version('jython', JYTHON_JAR_DIR + JYTHON_JAR_NAME)
+  end
+
+  def update
+    FileUtils.rm_r(jython_directory)
+    install
+  end
+end

data/lib/mortar/local/pig.rb

@@ -23,7 +23,7 @@ class Mortar::Local::Pig
   include Mortar::Local::InstallUtil
 
   PIG_LOG_FORMAT = "humanreadable"
-  PIG_TAR_DEFAULT_URL = "https://s3.amazonaws.com/mortar-public-artifacts/pig.tgz"
+  PIG_TAR_DEFAULT_URL = "https://s3.amazonaws.com/mhc-software-mirror/cli/pig.tgz"
 
   # Tempfile objects have a hook to delete the file when the object is
   # destroyed by the garbage collector.  In practice this means that a
@@ -84,34 +84,48 @@ class Mortar::Local::Pig
     File.basename(pig_archive_url)
   end
 
-  # Determines if a pig install needs to occur, true if no
-  # pig install present or a newer version is available
+  # Determines if a pig install needs to occur, true if no pig install present
   def should_do_pig_install?
     not (File.exists?(pig_directory))
   end
 
-  # Installs pig for this project if it is not already present
-  def install
+  # Determines if a pig install needs to occur, true if server side
+  # pig tgz is newer than date of the existing install
+  def should_do_pig_update?
+    return is_newer_version('pig', pig_archive_url)
+  end
+
+  def install_or_update()
+    call_install = false
     if should_do_pig_install?
-      FileUtils.mkdir_p(local_install_directory)
       action "Installing pig" do
-        download_file(pig_archive_url, local_install_directory)
-        local_tgz = File.join(local_install_directory, pig_archive_file)
-        extract_tgz(local_tgz, local_install_directory)
-
-        # This has been seening coming out of the tgz w/o +x so we do
-        # here to be sure it has the necessary permissions
-        FileUtils.chmod(0755, command)
-
-        File.delete(local_tgz)
-        note_install("pig")
+        install()
+      end
+    elsif should_do_pig_update?
+      action "Updating to latest pig" do
+        install()
       end
     end
   end
 
+  # Installs pig for this project if it is not already present
+  def install
+    FileUtils.mkdir_p(local_install_directory)
+    download_file(pig_archive_url, local_install_directory)
+    local_tgz = File.join(local_install_directory, pig_archive_file)
+    extract_tgz(local_tgz, local_install_directory)
+
+    # This has been seening coming out of the tgz w/o +x so we do
+    # here to be sure it has the necessary permissions
+    FileUtils.chmod(0755, command)
+
+    File.delete(local_tgz)
+    note_install("pig")
+  end
+
   # run the pig script with user supplied pig parameters
   def run_script(pig_script, pig_parameters)
-    run_pig_command(" -f #{pig_script.path}", pig_parameters)
+    run_pig_command(" -f #{pig_script.path}", pig_parameters, true)
   end
 
   # Create a temp file to be used for writing the illustrate
@@ -146,8 +160,8 @@ class Mortar::Local::Pig
     ensure_dir_exists("illustrate-output/resources/js")
     ensure_dir_exists("illustrate-output/resources/flash")
 
-    ["illustrate_css", 
-     "jquery", "jquery_transit", "jquery_stylestack", 
+    ["illustrate_css",
+     "jquery", "jquery_transit", "jquery_stylestack",
      "mortar_table", "zeroclipboard", "zeroclipboard_swf"].each { |resource|
       copy_if_not_present_at_dest(@resource_locations[resource], @resource_destinations[resource])
     }
@@ -199,14 +213,17 @@ class Mortar::Local::Pig
     end
     cmd += " #{pig_alias} '"
 
-    run_pig_command(cmd, [])
-    show_illustrate_output(illustrate_outpath)
+    result = run_pig_command(cmd, [], false)
+    if result
+      show_illustrate_output(illustrate_outpath)
+    end
   end
 
   # Run pig with the specified command ('command' is anything that
   # can be appended to the command line invocation of Pig that will
   # get it to do something interesting, such as '-f some-file.pig'
-  def run_pig_command(cmd, parameters = nil)
+  def run_pig_command(cmd, parameters = nil, jython_output = true)
+    unset_hadoop_env_vars
     # Generate the script for running the command, then
     # write it to a temp script which will be exectued
     script_text = script_for_command(cmd, parameters)
@@ -216,12 +233,21 @@ class Mortar::Local::Pig
     FileUtils.chmod(0755, script.path)
     system(script.path)
     script.unlink
+    return (0 == $?.to_i)
+  end
+
+  # so Pig doesn't try to load the wrong hadoop jar/configuration
+  # this doesn't mess up the env vars in the terminal, just this process (ruby)
+  def unset_hadoop_env_vars
+    ENV['HADOOP_HOME'] = ''
+    ENV['HADOOP_CONF_DIR'] = ''
   end
 
   # Generates a bash script which sets up the necessary environment and
   # then runs the pig command
-  def script_for_command(cmd, parameters)
+  def script_for_command(cmd, parameters, jython_output = true)
     template_params = pig_command_script_template_parameters(cmd, parameters)
+    template_params['pig_opts']['jython.output'] = jython_output
     erb = ERB.new(File.read(pig_command_script_template_path), 0, "%<>")
     erb.result(BindingClazz.new(template_params).get_binding)
   end
@@ -236,8 +262,8 @@ class Mortar::Local::Pig
     template_params = {}
     template_params['pig_params_file'] = make_pig_param_file(pig_parameters)
     template_params['pig_home'] = pig_directory
-    template_params['pig_classpath'] = "#{pig_directory}/lib-pig/*"
-    template_params['classpath'] = "#{pig_directory}/lib/*:#{pig_directory}/conf/jets3t.properties"
+    template_params['pig_classpath'] = "#{pig_directory}/lib-pig/*:#{jython_directory}/jython.jar"
+    template_params['classpath'] = "#{pig_directory}/lib/*:#{jython_directory}/jython.jar:#{pig_directory}/conf/jets3t.properties"
     template_params['project_home'] = File.expand_path("..", local_install_directory)
     template_params['local_install_dir'] = local_install_directory
     template_params['pig_sub_command'] = cmd
@@ -252,6 +278,10 @@ class Mortar::Local::Pig
     opts['fs.s3n.awsAccessKeyId'] = ENV['AWS_ACCESS_KEY']
     opts['fs.s3n.awsSecretAccessKey'] = ENV['AWS_SECRET_KEY']
     opts['pig.events.logformat'] = PIG_LOG_FORMAT
+    opts['python.verbose'] = 'error'
+    opts['jython.output'] = true
+    opts['python.home'] = jython_directory
+    opts['python.cachedir'] = jython_cache_directory
     return opts
   end
 

data/lib/mortar/local/python.rb

@@ -19,7 +19,7 @@ require "mortar/local/installutil"
 class Mortar::Local::Python
   include Mortar::Local::InstallUtil
 
-  PYTHON_DEFAULT_TGZ_URL = "https://s3.amazonaws.com/mortar-public-artifacts/mortar-python-osx.tgz"
+  PYTHON_DEFAULT_TGZ_URL = "https://s3.amazonaws.com/mhc-software-mirror/cli/mortar-python-osx.tgz"
 
   # Path to the python binary that should be used
   # for running UDFs
@@ -31,33 +31,45 @@ class Mortar::Local::Python
   def check_or_install
     if osx?
       # We currently only install python for osx
-      install_python_osx
+      install_or_update_osx
     else
       # Otherwise we check that the system supplied python will be sufficient
       check_system_python
     end
   end
 
+  def should_do_update?
+    return is_newer_version('python', python_archive_url)
+  end
+
   # Performs an installation of python specific to this project, this
   # install includes pip and virtualenv
-  def install_python_osx
+  def install_or_update_osx
     @command = "#{local_install_directory}/python/bin/python"
     if should_do_python_install?
-      FileUtils.mkdir_p(local_install_directory)
       action "Installing python" do
-        download_file(python_archive_url, local_install_directory)
-        extract_tgz(local_install_directory + "/" + python_archive_file, local_install_directory)
-
-        # This has been seening coming out of the tgz w/o +x so we do
-        # here to be sure it has the necessary permissions
-        FileUtils.chmod(0755, @command)
-        File.delete(local_install_directory + "/" + python_archive_file)
-        note_install("python")
+        install_osx
+      end
+    elsif should_do_update?
+      action "Updating to latest python" do
+        install_osx
       end
     end
     true
   end
 
+  def install_osx
+    FileUtils.mkdir_p(local_install_directory)
+    download_file(python_archive_url, local_install_directory)
+    extract_tgz(local_install_directory + "/" + python_archive_file, local_install_directory)
+
+    # This has been seening coming out of the tgz w/o +x so we do
+    # here to be sure it has the necessary permissions
+    FileUtils.chmod(0755, @command)
+    File.delete(local_install_directory + "/" + python_archive_file)
+    note_install("python")
+  end
+
   # Determines if a python install needs to occur, true if no
   # python install present or a newer version is available
   def should_do_python_install?

data/lib/mortar/templates/controlscript/controlscript.py

@@ -0,0 +1,11 @@
+def run_script():
+    import os
+    from org.apache.pig.scripting import Pig
+
+    # compile the pig code
+    P = Pig.compileFromFile("../pigscripts/#{script_name}.pig")
+    bound = P.bind()
+    bound.runSingle()
+
+if __name__ == '__main__':
+    run_script()

data/lib/mortar/templates/pigscript/pigscript.pig

@@ -6,7 +6,7 @@
  * Parameters - set default values here; you can override with -p on the command-line.
  */
  
-<%= '%default'%> INPUT_PATH 's3n://hawk-example-data/tutorial/excite.log.bz2'
+<%= '%default'%> INPUT_PATH 's3n://mortar-example-data/tutorial/excite.log.bz2'
 <%= '%default'%> OUTPUT_PATH 's3n://my-output-bucket/$MORTAR_EMAIL_S3_ESCAPED/<%= script_name %>'
 
 <% if not options[:skip_udf] %>

data/lib/mortar/templates/project/pigscripts/pigscript.pig

@@ -6,7 +6,7 @@
  * Parameters - set default values here; you can override with -p on the command-line.
  */
  
-<%= '%default'%> INPUT_PATH 's3n://hawk-example-data/tutorial/excite.log.bz2'
+<%= '%default'%> INPUT_PATH 's3n://mortar-example-data/tutorial/excite.log.bz2'
 <%= '%default'%> OUTPUT_PATH 's3n://my-output-bucket/$MORTAR_EMAIL_S3_ESCAPED/<%= project_name %>'
 
 /**

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.7.1"
+  VERSION = "0.7.2"
 end

data/spec/mortar/command/generate_spec.rb

@@ -34,6 +34,7 @@ describe Mortar::Command::Generate do
       File.exists?("Test/macros").should be_true
       File.exists?("Test/fixtures").should be_true
       File.exists?("Test/pigscripts").should be_true
+      File.exists?("Test/controlscripts").should be_true
       File.exists?("Test/udfs").should be_true
       File.exists?("Test/README.md").should be_true
       File.exists?("Test/Gemfile").should be_false
@@ -99,6 +100,26 @@ STDERR
     end
   end
 
+  describe "generate:controlscript" do
+    it "Generate a new controlscript in a project" do
+      with_blank_project do |p| 
+        stderr, stdout = execute("generate:controlscript Oink", p)
+        File.exists?(File.join(p.root_path, "controlscripts/Oink.py")).should be_true
+        File.read("controlscripts/Oink.py").each_line { |line| line.match(/<%.*%>/).should be_nil }
+      end
+    end
+
+    it "error when controlscript name isn't provided" do
+      with_blank_project do |p|
+        stderr, stdout = execute("generate:controlscript")
+        stderr.should == <<-STDERR
+ !    Usage: mortar generate:controlscript SCRIPTNAME
+ !    Must specify SCRIPTNAME.
+ STDERR
+      end
+    end
+  end
+
   describe "generate:python_udf" do
     it "Generate a new python udf in a project" do
       with_blank_project do |p| 

data/spec/mortar/command/jobs_spec.rb

@@ -337,7 +337,6 @@ STDERR
     end
     
     context("status") do
-      
       it "gets status for a completed, successful job" do
         with_git_initialized_project do |p|
           job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"

data/spec/mortar/command/local_spec.rb

@@ -69,11 +69,16 @@ STDERR
       it "errors when the script doesn't exist" do
         with_git_initialized_project do |p|
           write_file(File.join(p.pigscripts_path, "my_other_script.pig"))
+          write_file(File.join(p.controlscripts_path, "my_control_script.py"))
           stderr, stdout = execute("local:run my_script", p)
           stderr.should == <<-STDERR
- !    Unable to find pigscript my_script
- !    Available scripts:
+ !    Unable to find a pigscript or controlscript for my_script
+ !    
+ !    Available pigscripts:
  !    my_other_script
+ !    
+ !    Available controlscripts:
+ !    my_control_script
 STDERR
         end
       end
@@ -132,6 +137,9 @@ STDERR
         any_instance_of(Mortar::Local::Python) do |j|
           mock(j).setup_project_python_environment.returns(true)
         end
+        any_instance_of(Mortar::Local::Jython) do |j|
+          mock(j).install_or_update.returns(true)
+        end
         stderr, stdout = execute("local:configure")
         stderr.should == ""
       end

data/spec/mortar/command/projects_spec.rb

@@ -104,7 +104,7 @@ STDOUT
         # if the git stuff doesn't work, the registration will fail, so we can pretend it does work here
         mock(@git).git("add .").returns(true)
         mock(@git).git("commit -m \"Mortar project scaffolding\"").returns(true)
-        mock(@git).git("push mortar master").returns(true)
+        mock(@git).push_master
 
         stderr, stdout = execute("projects:create #{project_name}", nil, @git)
         Dir.pwd.end_with?("some_new_project").should be_true
@@ -127,6 +127,8 @@ STDOUT
 \e[1;32m      create\e[0m  .gitignore
 \e[1;32m      create\e[0m  pigscripts
 \e[1;32m      create\e[0m  pigscripts/some_new_project.pig
+\e[1;32m      create\e[0m  controlscripts
+\e[1;32m      create\e[0m  controlscripts/.gitkeep
 \e[1;32m      create\e[0m  macros
 \e[1;32m      create\e[0m  macros/.gitkeep
 \e[1;32m      create\e[0m  fixtures
@@ -215,6 +217,7 @@ STDERR
         mock(@git).has_dot_git?().returns(true)
         mock(@git).remotes.with_any_args.returns({})
         mock(@git).remote_add("mortar", project_git_url)
+        mock(@git).push_master
 
         stderr, stdout = execute("projects:register #{project_name}  --polling_interval 0.05", nil, @git)
         stdout.should == <<-STDOUT
@@ -236,6 +239,7 @@ STDOUT
         mock(@git).has_dot_git?().returns(true)
         mock(@git).remotes.with_any_args.returns({})
         mock(@git).remote_add("mortar", project_git_url)
+        mock(@git).push_master
 
         stderr, stdout = execute("projects:register #{project_name}  --polling_interval 0.05", nil, @git)
         stdout.should == <<-STDOUT

data/spec/mortar/helpers_spec.rb

@@ -19,6 +19,7 @@
 
 require "spec_helper"
 require "mortar/helpers"
+require 'fakefs/spec_helpers'
 
 module Mortar
   describe Helpers do
@@ -78,5 +79,29 @@ OUT
         infile.close()
       end
     end
+
+    context "ensure_dir_exists" do
+
+      it "does not have an existing directory" do
+        FakeFS do
+          dir_name = "./foo-bar-dir"
+          expect(File.directory?(dir_name)).to be_false
+          ensure_dir_exists(dir_name)
+          expect(File.directory?(dir_name)).to be_true
+        end
+      end
+
+      it "does have an existing directory" do
+        FakeFS do
+          dir_name = "./foo-bar-dir"
+          FileUtils.mkdir_p(dir_name)
+          expect(File.directory?(dir_name)).to be_true
+          ensure_dir_exists(dir_name)
+          expect(File.directory?(dir_name)).to be_true
+        end
+      end
+
+    end
+
   end
 end

data/spec/mortar/local/installutil_spec.rb

@@ -49,6 +49,17 @@ module Mortar::Local
         end
       end
 
+      it "works with file created by note-install" do
+        install_file_path = @installutil.install_file_for("foo")
+        install_date = 1234568
+        stub(Time).now.returns(install_date)
+        FakeFS do
+          FileUtils.mkdir_p(File.dirname(install_file_path))
+          @installutil.note_install('foo')
+          expect(@installutil.install_date('foo')).to eq(install_date)
+        end
+      end
+
     end
 
     context("note-install") do
@@ -66,5 +77,50 @@ module Mortar::Local
 
     end
 
+    context "is_newer_version" do
+
+      it "is if remote file is newer" do
+        stub(@installutil).install_date.returns(1)
+        stub(@installutil).url_date.returns(2)
+        expect(@installutil.is_newer_version('foo', 'http://bar')).to be_true
+      end
+
+      it "is not if remote file is older" do
+        stub(@installutil).install_date.returns(2)
+        stub(@installutil).url_date.returns(1)
+        expect(@installutil.is_newer_version('foo', 'http://bar')).to be_false
+      end
+
+      it "if no version is present" do
+        install_file_path = @installutil.install_file_for("foo")
+        stub(@installutil).url_date.returns(1)
+        FakeFS do
+          FileUtils.rm_rf(File.dirname(install_file_path), :force => true)
+          expect(@installutil.is_newer_version('foo', 'http://bar')).to be_true
+        end
+      end
+
+    end
+
+    context "url_date" do
+
+      it "returns an epoch" do
+        excon_response = Excon::Response.new(:headers => {"Last-Modified" => "Mon, 11 Mar 2013 15:03:55 GMT"})
+        mock(Excon).head("http://foo/bar").returns(excon_response)
+        actual_epoch = @installutil.url_date("http://foo/bar")
+        expect(actual_epoch).to eq(1363014235)
+      end
+
+    end
+
+    context "parse_http_date" do
+
+      it "returns the appropriate epoch" do
+        epoch = @installutil.http_date_to_epoch("Mon, 11 Mar 2013 15:03:55 GMT")
+        expect(epoch).to eq(1363014235)
+      end
+
+    end
+
   end
 end

data/spec/mortar/local/jython_spec.rb

@@ -0,0 +1,68 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'spec_helper'
+require 'fakefs/spec_helpers'
+require 'mortar/local/jython'
+
+
+
+module Mortar::Local
+  describe Jython do
+
+    context("update") do
+
+      it "removes existing install and does install" do
+        install_dir = "/foo/bar/jython"
+        jython = Mortar::Local::Jython.new
+        mock(jython).jython_directory.returns(install_dir)
+        mock(jython).install
+        FakeFS do
+          FileUtils.mkdir_p(install_dir)
+          expect(File.directory?(install_dir)).to be_true
+          jython.update
+          expect(File.directory?(install_dir)).to be_false
+        end
+      end
+
+    end
+
+    context "should_install" do
+
+      it "is true if the directory does not exist" do
+        install_dir = "/foo/bar/jython"
+        jython = Mortar::Local::Jython.new
+        mock(jython).jython_directory.returns(install_dir)
+        FakeFS do
+          FileUtils.mkdir_p(install_dir)
+          expect(jython.should_install).to be_false
+        end
+      end
+
+      it "is false if the directory already exists" do
+        install_dir = "/foo/bar/jython"
+        jython = Mortar::Local::Jython.new
+        mock(jython).jython_directory.returns(install_dir)
+        FakeFS do
+          FileUtils.rm_rf(install_dir, :force => true)
+          expect(jython.should_install).to be_true
+        end
+      end
+
+    end
+
+  end
+end

data/spec/mortar/local/pig_spec.rb

@@ -30,24 +30,12 @@ module Mortar::Local
 
     context("install") do
 
-      it "does nothing if told not to" do
-        pig = Mortar::Local::Pig.new
-        mock(pig).should_do_pig_install?.returns(false)
-        FakeFS do
-          FileUtils.mkdir_p(File.dirname(pig.local_install_directory))
-          FileUtils.rm_rf(pig.local_install_directory, :force => true)
-          pig.install
-          expect(File.exists?(pig.local_install_directory)).to be_false
-        end
-      end
-
       it "handles necessary installation steps" do
         # creates the parent directory, downloads the tgz, extracts it,
         # chmods bin/pig, removes tgz, and notes the installation
         FakeFS do
           pig = Mortar::Local::Pig.new
           local_pig_archive = File.join(pig.local_install_directory, pig.pig_archive_file)
-          mock(pig).should_do_pig_install?.returns(true)
           mock(pig).download_file(pig.pig_archive_url, pig.local_install_directory) do
             # Simulate the tgz file being downloaded, this should be deleted
             # before the method finishes executing
@@ -67,6 +55,42 @@ module Mortar::Local
 
     end
 
+    context "install_or_update" do
+
+      it "does nothing if existing install and no update available" do
+        pig = Mortar::Local::Pig.new
+        mock(pig).should_do_pig_install?.returns(false)
+        mock(pig).should_do_pig_update?.returns(false)
+        FakeFS do
+          FileUtils.mkdir_p(File.dirname(pig.local_install_directory))
+          FileUtils.rm_rf(pig.local_install_directory, :force => true)
+          pig.install_or_update
+          expect(File.exists?(pig.local_install_directory)).to be_false
+        end
+      end
+
+      it "does install if none has been done before" do
+        pig = Mortar::Local::Pig.new
+        mock(pig).should_do_pig_install?.returns(true)
+        mock(pig).install
+        capture_stdout do
+          pig.install_or_update
+        end
+      end
+
+      it "does install if one was done before but there is an update" do
+        pig = Mortar::Local::Pig.new
+        mock(pig).should_do_pig_install?.returns(false)
+        mock(pig).should_do_pig_update?.returns(true)
+        mock(pig).install
+        capture_stdout do
+          pig.install_or_update
+        end
+      end
+
+    end
+
+
     context "show_illustrate_output" do
 
       it "takes a path to a json file, renders the html template, and opens it" do
@@ -153,5 +177,29 @@ module Mortar::Local
 
     end
 
+    context "illustrate_alias" do
+
+      it "displays html results if illustrate was successful" do
+        any_instance_of(Mortar::Project::PigScripts, :elements => nil, :path => "/foo/bar/baz.pig")
+        script = Mortar::Project::PigScripts.new('/foo/bar/baz.pig', 'baz', 'pig')
+        pig = Mortar::Local::Pig.new
+        mock(pig).run_pig_command.with_any_args.returns(true)
+        mock(pig).show_illustrate_output.with_any_args
+        stub(pig).make_pig_param_file.returns('param.file')
+        pig.illustrate_alias(script, 'my_alias', false, [])
+      end
+
+      it "skips results if illustrate was unsuccessful" do
+        any_instance_of(Mortar::Project::PigScripts, :elements => nil, :path => "/foo/bar/baz.pig")
+        script = Mortar::Project::PigScripts.new('/foo/bar/baz.pig', 'baz', 'pig')
+        pig = Mortar::Local::Pig.new
+        mock(pig).run_pig_command.with_any_args.returns(false)
+        mock(pig).show_illustrate_output.with_any_args.never
+        stub(pig).make_pig_param_file.returns('param.file')
+        pig.illustrate_alias(script, 'my_alias', false, [])
+      end
+
+    end
+
   end
 end

data/spec/mortar/local/python_spec.rb

@@ -0,0 +1,68 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'spec_helper'
+require 'fakefs/spec_helpers'
+require 'mortar/local/python'
+require 'launchy'
+
+
+module Mortar::Local
+  describe Python do
+
+    context("check_or_install") do
+
+      it "checks for python system requirements if not osx" do
+        python = Mortar::Local::Python.new
+        mock(python).osx?.returns(true)
+        mock(python).install_or_update_osx.returns(true)
+        python.check_or_install
+      end
+
+      it "installs python on osx" do
+        python = Mortar::Local::Python.new
+        mock(python).osx?.returns(false)
+        mock(python).check_system_python.returns(true)
+        python.check_or_install
+      end
+
+    end
+
+    context("install_or_update_osx") do
+
+      it "does install if none present" do
+        python = Mortar::Local::Python.new
+        mock(python).should_do_python_install?.returns(true)
+        mock(python).install_osx.returns(true)
+        capture_stdout do
+          python.install_or_update_osx
+        end
+      end
+
+      it "does install if an update is available" do
+        python = Mortar::Local::Python.new
+        mock(python).should_do_python_install?.returns(false)
+        mock(python).should_do_update?.returns(true)
+        mock(python).install_osx.returns(true)
+        capture_stdout do
+          python.install_or_update_osx
+        end
+      end
+
+    end
+
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mortar
 version: !ruby/object:Gem::Version 
-  hash: 1
+  hash: 7
   prerelease: 
   segments: 
   - 0
   - 7
-  - 1
-  version: 0.7.1
+  - 2
+  version: 0.7.2
 platform: ruby
 authors: 
 - Mortar Data
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-03-14 00:00:00 Z
+date: 2013-03-26 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: mortar-api-ruby
@@ -187,6 +187,7 @@ files:
 - lib/mortar/command/validate.rb
 - lib/mortar/command/version.rb
 - lib/mortar/errors.rb
+- lib/mortar/generators/controlscript_generator.rb
 - lib/mortar/generators/generator_base.rb
 - lib/mortar/generators/macro_generator.rb
 - lib/mortar/generators/pigscript_generator.rb
@@ -197,13 +198,16 @@ files:
 - lib/mortar/local/controller.rb
 - lib/mortar/local/installutil.rb
 - lib/mortar/local/java.rb
+- lib/mortar/local/jython.rb
 - lib/mortar/local/pig.rb
 - lib/mortar/local/python.rb
 - lib/mortar/project.rb
+- lib/mortar/templates/controlscript/controlscript.py
 - lib/mortar/templates/macro/macro.pig
 - lib/mortar/templates/pigscript/pigscript.pig
 - lib/mortar/templates/pigscript/python_udf.py
 - lib/mortar/templates/project/README.md
+- lib/mortar/templates/project/controlscripts/gitkeep
 - lib/mortar/templates/project/fixtures/gitkeep
 - lib/mortar/templates/project/gitignore
 - lib/mortar/templates/project/macros/gitkeep
@@ -235,7 +239,9 @@ files:
 - spec/mortar/local/controller_spec.rb
 - spec/mortar/local/installutil_spec.rb
 - spec/mortar/local/java_spec.rb
+- spec/mortar/local/jython_spec.rb
 - spec/mortar/local/pig_spec.rb
+- spec/mortar/local/python_spec.rb
 - spec/mortar/project_spec.rb
 - spec/mortar/updater_spec.rb
 - spec/spec.opts