mortar 0.14.1 → 0.15.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a48ef7a19f19d16af145ed38847a5c316c509361
+  data.tar.gz: 4d554dd57193f50f23b0f98b273695d6e7d1af2c
+SHA512:
+  metadata.gz: d8a722fdcc23c6312b8faac36dba45d9f2399d6bbb64c2187576abd600fead99d0c20dfd7073002d1f4903c9026409b57b219ba49ace184d5334f0b575ab7ba1
+  data.tar.gz: 868f456e447061aaeb90a941a00f2cdf0b0706fa7355fe5753a7554ec99af13d43501bcb553aa3246ff0aa9605382f6cc5f620733ae57327cbe521b1970a5b1c

data/lib/mortar/command/base.rb

@@ -399,7 +399,18 @@ protected
 
     pigscript or controlscript
   end
-  
+
+  def validate_luigiscript!(luigiscript_name)
+    shortened_script_name = File.basename(luigiscript_name, ".*")
+    unless luigiscript = project.luigiscripts[shortened_script_name]
+      available_scripts = project.luigiscripts.none? ? "No luigiscripts found" : "Available luigiscripts:\n#{project.luigiscripts.collect{|k,v| v.executable_path}.sort.join("\n")}"
+      error("Unable to find luigiscript #{shortened_script_name}\n#{available_scripts}")
+    end
+    #While validating we can load the defaults that are relevant to this script.
+    load_defaults(shortened_script_name)
+    luigiscript
+  end
+
   def validate_pigscript!(pigscript_name)
     shortened_pigscript_name = File.basename(pigscript_name, ".*")
     unless pigscript = project.pigscripts[shortened_pigscript_name]
@@ -467,6 +478,11 @@ protected
       elsif remotes.values.uniq.size == 1
         # take the only project in the remotes
         [remotes.first[1], remotes.first[0]]
+      elsif remotes.has_key? 'mortar'
+        # In some cases (like forking a public project in mortar-code)
+        # we'll have more than one possible remote.  We'll default to the
+        # one called mortar.
+        [remotes['mortar'], 'mortar']
       else
         raise(Mortar::Command::CommandFailed, "Multiple projects in folder and no project specified.\nSpecify which project to use with --project <project name>")
       end

data/lib/mortar/command/describe.rb

@@ -22,7 +22,7 @@ class Mortar::Command::Describe < Mortar::Command::Base
   
   include Mortar::Git
     
-  # describe [PIGSCRIPT] [ALIAS]
+  # describe PIGSCRIPT ALIAS
   #
   # Describe the schema of an alias and all of its ancestors.
   #

data/lib/mortar/command/illustrate.rb

@@ -40,14 +40,14 @@ class Mortar::Command::Illustrate < Mortar::Command::Base
   def index
     pigscript_name = shift_argument
     alias_name = shift_argument
-    skip_pruning = options[:skippruning] ||= false
-    
     validate_arguments!
-    pigscript = validate_script!(pigscript_name)
+
+    skip_pruning = options[:skippruning] ||= false
           
     unless pigscript_name
       error("Usage: mortar illustrate PIGSCRIPT [ALIAS]\nMust specify PIGSCRIPT.")
     end
+    pigscript = validate_script!(pigscript_name)
     
     if pigscript.is_a? Mortar::Project::ControlScript
       error "Currently Mortar does not support illustrating control scripts"

data/lib/mortar/command/jobs.rb

@@ -205,6 +205,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
     unless job_id
       error("Usage: mortar jobs:status JOB_ID\nMust specify JOB_ID.")
     end
+    validate_arguments!
     
     # Inner function to display the hash table when the job is complte
     def display_job_status(job_status)
@@ -313,6 +314,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
     unless job_id
       error("Usage: mortar jobs:stop JOB_ID\nMust specify JOB_ID.")
     end
+    validate_arguments!
 
     response = api.stop_job(job_id).body  
 

data/lib/mortar/command/local.rb

@@ -144,7 +144,10 @@ class Mortar::Command::Local < Mortar::Command::Base
   def illustrate
     pigscript_name = shift_argument
     alias_name = shift_argument
+    validate_arguments!
+
     skip_pruning = options[:skippruning] ||= false
+    no_browser = options[:no_browser] ||= false
 
     unless pigscript_name
       error("Usage: mortar local:illustrate PIGSCRIPT [ALIAS]\nMust specify PIGSCRIPT.")
@@ -157,11 +160,10 @@ class Mortar::Command::Local < Mortar::Command::Base
     end
     Dir.chdir(project_root)
 
-    validate_arguments!
     pigscript = validate_pigscript!(pigscript_name)
 
     ctrl = Mortar::Local::Controller.new
-    ctrl.illustrate(pigscript, alias_name, pig_version, pig_parameters, skip_pruning)
+    ctrl.illustrate(pigscript, alias_name, pig_version, pig_parameters, skip_pruning, no_browser)
   end
 
 
@@ -211,4 +213,39 @@ class Mortar::Command::Local < Mortar::Command::Base
     ctrl.repl(pig_version, pig_parameters)
   end
 
+
+  # local:luigi SCRIPT
+  #
+  # Run a luigi workflow on your local machine in local scheduler mode.
+  # Any additional command line arguments will be passed directly to the luigi script.
+  #
+  # -p, --parameter NAME=VALUE  # Set a pig parameter value in your script.
+  # -f, --param-file PARAMFILE  # Load pig parameter values from a file.
+  # --project-root PROJECTDIR   # The root directory of the project if not the CWD
+  #
+  #Examples:
+  #
+  #    Run the recsys luigi script with a parameter named date-interval
+  #        $ mortar local:luigi luigiscripts/recsys.py --date-interval 2012-04
+  def luigi
+    script_name = shift_argument
+    unless script_name
+      error("Usage: mortar local:luigi SCRIPT\nMust specify SCRIPT.")
+    end
+    validate_arguments!
+
+    # cd into the project root
+    project_root = options[:project_root] ||= Dir.getwd
+    unless File.directory?(project_root)
+      error("No such directory #{project_root}")
+    end
+    Dir.chdir(project_root)
+    script = validate_luigiscript!(script_name)
+    ctrl = Mortar::Local::Controller.new
+    luigi_params = pig_parameters.sort_by { |p| p['name'] }
+    luigi_params = luigi_params.map { |arg| ["--#{arg['name']}", "#{arg['value']}"] }.flatten
+    ctrl.run_luigi(script, luigi_params)
+  end
+
+
 end

data/lib/mortar/conf/luigi/logging.ini

@@ -0,0 +1,30 @@
+
+[loggers]
+keys=root,luigi-interface        
+
+[handlers]
+keys=console
+
+[formatters]
+keys=standard,plain        
+
+[logger_root]
+level=DEBUG
+handlers=console
+
+[logger_luigi-interface]
+level=DEBUG
+handlers=
+qualname=luigi-interface        
+
+[handler_console]
+level=DEBUG
+class=StreamHandler
+args=(sys.stderr,)
+formatter=standard        
+
+[formatter_standard]
+format=%(asctime)s [%(process)d:%(threadName)s] (%(filename)s:%(lineno)d)  %(levelname)-5.5s - %(message)s
+
+[formatter_plain]
+format=%(message)s

data/lib/mortar/local/controller.rb

@@ -181,11 +181,11 @@ EOF
   end
 
   # Main entry point for illustrating a pig alias
-  def illustrate(pig_script, pig_alias, pig_version, pig_parameters, skip_pruning)
+  def illustrate(pig_script, pig_alias, pig_version, pig_parameters, skip_pruning, no_browser)
     require_aws_keys
     install_and_configure(pig_version)
     pig = Mortar::Local::Pig.new()
-    pig.illustrate_alias(pig_script, pig_alias, skip_pruning, pig_version, pig_parameters)
+    pig.illustrate_alias(pig_script, pig_alias, skip_pruning, no_browser, pig_version, pig_parameters)
   end
 
   def validate(pig_script, pig_version, pig_parameters)
@@ -200,4 +200,10 @@ EOF
     pig.launch_repl(pig_version, pig_parameters)
   end
 
+  def run_luigi(luigi_script, user_script_args)
+    install_and_configure()
+    py = Mortar::Local::Python.new()
+    py.run_luigi_script(luigi_script, user_script_args)
+  end
+
 end

data/lib/mortar/local/installutil.rb

@@ -165,6 +165,7 @@ module Mortar
         else
           raise RuntimeError, "Unknown call type: #{call_func}"
         end
+
         case response.status
         when 300..303 then 
           make_call(response.headers['Location'], call_func, redirect_times+1, errors)
@@ -207,6 +208,59 @@ module Mortar
         return existing_install_date < remote_archive_date
       end
 
+      def run_templated_script(template, template_params)
+        # Insert standard template variables
+        template_params['project_home'] = File.expand_path("..", local_install_directory)
+        template_params['local_install_dir'] = local_install_directory
+
+        unset_hadoop_env_vars
+        reset_local_logs
+        # Generate the script for running the command, then
+        # write it to a temp script which will be exectued
+        script_text = render_script_template(template, template_params)
+        script = Tempfile.new("mortar-")
+        script.write(script_text)
+        script.close(false)
+        FileUtils.chmod(0755, script.path)
+        system(script.path)
+        script.unlink
+        return (0 == $?.to_i)
+      end
+
+      def render_script_template(template, template_params)
+        erb = ERB.new(File.read(template), 0, "%<>")
+        erb.result(BindingClazz.new(template_params).get_binding)
+      end
+
+      # so Pig doesn't try to load the wrong hadoop jar/configuration
+      # this doesn't mess up the env vars in the terminal, just this process (ruby)
+      def unset_hadoop_env_vars
+        ENV['HADOOP_HOME'] = ''
+        ENV['HADOOP_CONF_DIR'] = ''
+      end
+
+      def reset_local_logs
+        if File.directory? local_log_dir
+          FileUtils.rm_rf local_log_dir
+        end
+        Dir.mkdir local_log_dir
+        Dir.mkdir local_udf_log_dir
+      end
+
+
+      # Allows us to use a hash for template variables
+      class BindingClazz
+        def initialize(attrs)
+          attrs.each{ |k, v|
+        # set an instance variable with the key name so the binding will find it in scope
+            self.instance_variable_set("@#{k}".to_sym, v)
+          }
+        end
+        def get_binding()
+          binding
+        end
+      end
+
     end
   end
 end

data/lib/mortar/local/pig.rb

@@ -202,7 +202,7 @@ class Mortar::Local::Pig
     outfile.path
   end
 
-  # Given a file path, open it and decode the containing json
+  # Given a file path, open it and decode the containing text
   def decode_illustrate_input_file(illustrate_outpath)
     data_raw = File.read(illustrate_outpath)
     begin
@@ -212,10 +212,9 @@ class Mortar::Local::Pig
       ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
       data_encoded = ic.iconv(data_raw)
     end
-    json_decode(data_encoded)
   end
 
-  def show_illustrate_output(illustrate_outpath)
+  def show_illustrate_output_browser(illustrate_outpath)
     ensure_dir_exists("illustrate-output")
     ensure_dir_exists("illustrate-output/resources")
     ensure_dir_exists("illustrate-output/resources/css")
@@ -229,7 +228,8 @@ class Mortar::Local::Pig
     }
 
     # Pull in the dumped json file
-    illustrate_data = decode_illustrate_input_file(illustrate_outpath)
+    illustrate_data_json_text = decode_illustrate_input_file(illustrate_outpath)
+    illustrate_data = json_decode(illustrate_data_json_text)
 
     # Render a template using it's values
     template_params = create_illustrate_template_parameters(illustrate_data)
@@ -248,7 +248,6 @@ class Mortar::Local::Pig
       require "launchy"
       Launchy.open(File.expand_path(@resource_destinations["illustrate_html"]))
     end
-
   end
 
   def create_illustrate_template_parameters(illustrate_data)
@@ -258,7 +257,7 @@ class Mortar::Local::Pig
     return params
   end
 
-  def illustrate_alias(pig_script, pig_alias, skip_pruning, pig_version, pig_parameters)
+  def illustrate_alias(pig_script, pig_alias, skip_pruning, no_browser, pig_version, pig_parameters)
     cmd = "-e 'illustrate "
 
     # Parameters have to be entered with the illustrate command (as
@@ -275,7 +274,11 @@ class Mortar::Local::Pig
       cmd += " -skipPruning "
     end
 
-    cmd += " -json '"
+    if no_browser
+      cmd += " -str '"
+    else
+      cmd += " -json '"
+    end
 
     if pig_alias
       cmd += " #{pig_alias} "
@@ -283,7 +286,11 @@ class Mortar::Local::Pig
 
     result = run_pig_command(cmd, pig_version, [], false)
     if result
-      show_illustrate_output(illustrate_outpath)
+      if no_browser
+        display decode_illustrate_input_file(illustrate_outpath)
+      else
+        show_illustrate_output_browser(illustrate_outpath)
+      end
     end
   end
 
@@ -291,42 +298,9 @@ class Mortar::Local::Pig
   # can be appended to the command line invocation of Pig that will
   # get it to do something interesting, such as '-f some-file.pig'
   def run_pig_command(cmd, pig_version, parameters = nil, jython_output = true)
-    unset_hadoop_env_vars
-    reset_local_logs
-    # Generate the script for running the command, then
-    # write it to a temp script which will be exectued
-    script_text = script_for_command(cmd, pig_version, parameters)
-    script = Tempfile.new("mortar-")
-    script.write(script_text)
-    script.close(false)
-    FileUtils.chmod(0755, script.path)
-    system(script.path)
-    script.unlink
-    return (0 == $?.to_i)
-  end
-
-  # so Pig doesn't try to load the wrong hadoop jar/configuration
-  # this doesn't mess up the env vars in the terminal, just this process (ruby)
-  def unset_hadoop_env_vars
-    ENV['HADOOP_HOME'] = ''
-    ENV['HADOOP_CONF_DIR'] = ''
-  end
-
-  def reset_local_logs
-    if File.directory? local_log_dir
-      FileUtils.rm_rf local_log_dir
-    end
-    Dir.mkdir local_log_dir
-    Dir.mkdir local_udf_log_dir
-  end
-
-  # Generates a bash script which sets up the necessary environment and
-  # then runs the pig command
-  def script_for_command(cmd, pig_version, parameters, jython_output = true)
     template_params = pig_command_script_template_parameters(cmd, pig_version, parameters)
     template_params['pig_opts']['jython.output'] = jython_output
-    erb = ERB.new(File.read(pig_command_script_template_path), 0, "%<>")
-    erb.result(BindingClazz.new(template_params).get_binding)
+    return run_templated_script(pig_command_script_template_path, template_params)
   end
 
   # Path to the template which generates the bash script for running pig
@@ -355,8 +329,6 @@ class Mortar::Local::Pig
     template_params['pig_classpath'] = "#{pig_directory(pig_version)}/lib-local/*:#{lib_directory}/lib-local/*:#{pig_directory(pig_version)}/lib-pig/*:#{pig_directory(pig_version)}/lib-cluster/*:#{lib_directory}/lib-pig/*:#{lib_directory}/lib-cluster/*:#{jython_directory}/jython.jar"
     template_params['classpath'] = template_params_classpath
     template_params['log4j_conf'] = log4j_conf
-    template_params['project_home'] = File.expand_path("..", local_install_directory)
-    template_params['local_install_dir'] = local_install_directory
     template_params['pig_sub_command'] = cmd
     template_params['pig_opts'] = pig_options
     template_params
@@ -423,17 +395,4 @@ class Mortar::Local::Pig
     param_file.path
   end
 
-  # Allows us to use a hash for template variables
-  class BindingClazz
-    def initialize(attrs)
-      attrs.each{ |k, v|
-        # set an instance variable with the key name so the binding will find it in scope
-        self.instance_variable_set("@#{k}".to_sym, v)
-      }
-    end
-    def get_binding()
-      binding
-    end
-  end
-
 end

data/lib/mortar/local/python.rb

@@ -21,8 +21,9 @@ class Mortar::Local::Python
 
   PYTHON_OSX_TGZ_NAME = "mortar-python-osx.tgz"
   PYTHON_OSX_TGZ_DEFAULT_URL_PATH = "resource/python_osx"
+  PYPI_URL_PATH = "resource/mortar_pypi"
 
-  MORTAR_PYTHON_PACKAGES = ["luigi"]
+  MORTAR_PYTHON_PACKAGES = ["luigi", "mortar-luigi"]
 
   # Path to the python binary that should be used
   # for running UDFs
@@ -167,8 +168,11 @@ class Mortar::Local::Python
   end
 
   def has_valid_virtualenv?
-    `#{@command} -m virtualenv #{python_env_dir}`
+    output = `#{@command} -m virtualenv #{python_env_dir} 2>&1`
     if 0 != $?.to_i
+      File.open(virtualenv_error_log_path, 'w') { |f|
+        f.write(output)
+      }
       return false
     end
     return true
@@ -205,6 +209,10 @@ class Mortar::Local::Python
     return ENV.fetch('PIP_ERROR_LOG', "dependency_install.log")
   end
 
+  def virtualenv_error_log_path
+    return ENV.fetch('VE_ERROR_LOG', "virtualenv.log")
+  end
+
   # Whether or not we need to do a `pip install -r requirements.txt` because
   # we've never done one before or the dependencies have changed
   def should_do_requirements_install
@@ -230,7 +238,9 @@ class Mortar::Local::Python
   end
 
   def mortar_package_url(package)
-    return "http://s3.amazonaws.com/mortar-pypi/#{package}/#{package}.tar.gz";
+    full_host  = (host =~ /^http/) ? host : "https://api.#{host}"
+    default_url = full_host + "/" + PYPI_URL_PATH
+    "#{ENV.fetch('MORTAR_PACKAGE_URL', default_url)}/#{package}"
   end
 
   def update_mortar_package?(package)
@@ -266,9 +276,27 @@ class Mortar::Local::Python
     return true
   end
 
+  def local_activate_path
+    return "#{python_env_dir}/bin/activate"
+  end
+
+  def local_python_bin
+    return "#{python_env_dir}/bin/python"
+  end
+
+  def local_pip_bin
+    return "#{python_env_dir}/bin/pip"
+  end
+
   def pip_install package_url
-    pip_output = `. #{python_env_dir}/bin/activate &&
-          #{python_env_dir}/bin/pip install  #{package_url} --use-mirrors`
+    # Note that we're executing pip by passing it as a script for python to execute, this is
+    # explicitly done to deal with this command breaking due to the maximum size of the path
+    # to the interpreter in a shebang.  Since the containing virtualenv is already buried
+    # several layers deep in the .mortar-local directory we're very likely to (read: have) hit
+    # this limit.  This unfortunately leads to very vague errors about pip not existing when
+    # in fact it is the truncated path to the interpreter that does not exist.  I would now
+    # like the last day of my life back.
+    pip_output = `. #{local_activate_path} && #{local_python_bin} #{local_pip_bin} install  #{package_url} --use-mirrors;`
     if 0 != $?.to_i
       File.open(pip_error_log_path, 'w') { |f|
         f.write(pip_output)
@@ -287,4 +315,31 @@ class Mortar::Local::Python
     note_install mortar_package_dir(package_name)
   end
 
+  def run_luigi_script(luigi_script, user_script_args)
+    template_params = luigi_command_template_parameters(luigi_script, user_script_args)
+    return run_templated_script(python_command_script_template_path, template_params)
+  end
+
+  # Path to the template which generates the bash script for running python
+  def python_command_script_template_path
+    File.expand_path("../../templates/script/runpython.sh", __FILE__)
+  end
+
+  def luigi_logging_config_file_path
+    File.expand_path("../../conf/luigi/logging.ini", __FILE__)
+  end
+
+  def luigi_command_template_parameters(luigi_script, user_script_args)
+    script_args = [
+      "--local-scheduler",
+      "--logging-conf-file #{luigi_logging_config_file_path}",
+      user_script_args.join(" "),
+    ]
+    return {
+      :python_arugments => "",
+      :python_script => luigi_script.executable_path(),
+      :script_arguments => script_args.join(" ")
+    }
+  end
+
 end

data/lib/mortar/project.rb

@@ -70,7 +70,20 @@ module Mortar
           :optional => true)
         @controlscripts
       end
-      
+
+      def luigiscripts_path
+        File.join(@root_path, "luigiscripts")
+      end
+
+      def luigiscripts
+        @luigiscripts ||= LuigiScripts.new(
+          luigiscripts_path,
+          "luigiscripts",
+          ".py",
+          :optional => true)
+        @luigiscripts
+      end
+
       def tmp_path
         path = File.join(@root_path, "tmp")
         unless File.directory? path
@@ -153,6 +166,12 @@ module Mortar
       end
     end
 
+    class LuigiScripts < ProjectEntity
+      def element(name, path)
+        LuigiScript.new(name, path)
+      end
+    end
+
     class PythonUDFs < ProjectEntity
       def element(name, path)
         Script.new(name, path)
@@ -180,6 +199,14 @@ module Mortar
         code
       end
     end
+
+    class LuigiScript < Script
+      
+      def executable_path
+        "luigiscripts/#{self.name}.py"
+      end
+      
+    end
     
     class ControlScript < Script
       

data/lib/mortar/templates/script/runpython.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+# Setup python environment
+source <%= @local_install_dir %>/pythonenv/bin/activate
+
+# Run Python
+<%= @local_install_dir %>/pythonenv/bin/python \
+    <%= @python_arugments %> \
+    <%= @python_script %> \
+    <%= @script_arguments %>

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.14.1"
+  VERSION = "0.15.0"
 end

data/spec/mortar/command/base_spec.rb

@@ -69,6 +69,13 @@ other\tgit@github.com:other.git (push)
         @base.project.name.should == 'myproject'
       end
 
+      it "gets the project from remotes when there's two projects but one mortar remote" do
+        stub(@base.git).has_dot_git? {true}
+        stub(@base.git).remotes {{ 'mortar' => 'myproject', 'base' => 'my-base-project'}}
+        mock(@base.git).git("config mortar.remote", false).returns("")
+        @base.project.name.should == 'myproject'
+      end
+
       it "accepts a --remote argument to choose the project from the remote name" do
         stub(@base.git).has_dot_git?.returns(true)
         stub(@base.git).remotes.returns({ 'staging' => 'myproject-staging', 'production' => 'myproject' })

data/spec/mortar/command/local_spec.rb

@@ -44,7 +44,7 @@ STDERR
           pigscript = Mortar::Project::PigScript.new(script_name, script_path)
           mock(Mortar::Project::PigScript).new(script_name, script_path).returns(pigscript)
           any_instance_of(Mortar::Local::Controller) do |u|
-            mock(u).illustrate(pigscript, "some_alias", is_a(Mortar::PigVersion::Pig09), [], false).returns(nil)
+            mock(u).illustrate(pigscript, "some_alias", is_a(Mortar::PigVersion::Pig09), [], false, false).returns(nil)
           end
           stderr, stdout = execute("local:illustrate #{script_name} some_alias", p)
           stderr.should == ""
@@ -59,7 +59,7 @@ STDERR
           pigscript = Mortar::Project::PigScript.new(script_name, script_path)
           mock(Mortar::Project::PigScript).new(script_name, script_path).returns(pigscript)
           any_instance_of(Mortar::Local::Controller) do |u|
-            mock(u).illustrate(pigscript, nil, is_a(Mortar::PigVersion::Pig012), [], false).returns(nil)
+            mock(u).illustrate(pigscript, nil, is_a(Mortar::PigVersion::Pig012), [], false, false).returns(nil)
           end
           stderr, stdout = execute("local:illustrate #{script_name} -g 0.12", p)
           stderr.should == ""
@@ -228,6 +228,50 @@ PARAMS
 
     end
 
+    context "local:luigi" do
+
+      it "Exits with error if no script specified" do
+        with_git_initialized_project do |p|
+          stderr, stdout = execute("local:luigi", p)
+          stderr.should == <<-STDERR
+ !    Usage: mortar local:luigi SCRIPT
+ !    Must specify SCRIPT.
+STDERR
+        end
+      end
+
+      it "Exits with error if script doesn't exist" do
+        with_git_initialized_project do |p|
+          stderr, stdout = execute("local:luigi foobarbaz", p)
+          stderr.should == <<-STDERR
+ !    Unable to find luigiscript foobarbaz
+ !    No luigiscripts found
+STDERR
+        end
+      end
+
+      it "Runs script forwarding options to luigi script" do
+        with_git_initialized_project do |p|
+          script_name = "some_luigi_script"
+          script_path = File.join(p.luigiscripts_path, "#{script_name}.py")
+          write_file(script_path)
+          luigi_script = Mortar::Project::LuigiScript.new(script_name, script_path)
+          mock(Mortar::Project::LuigiScript).new(script_name, script_path).returns(luigi_script)
+          any_instance_of(Mortar::Local::Python) do |u|
+            mock(u).run_luigi_script(luigi_script, %W{--myoption 2 --myotheroption 3})
+          end
+          any_instance_of(Mortar::Local::Controller) do |u|
+            mock(u).install_and_configure
+          end
+          stderr, stdout = execute("local:luigi some_luigi_script -p myoption=2 -p myotheroption=3", p)
+          stderr.should == ""
+        end
+      end
+
+
+    end
+
+
   end
 end
 

data/spec/mortar/local/controller_spec.rb

@@ -175,11 +175,12 @@ module Mortar::Local
         test_script = "foobar-script"
         script_alias = "some_alias"
         prune = false
+        no_browser = false
         the_parameters = []
         any_instance_of(Mortar::Local::Pig) do |p|
-          mock(p).illustrate_alias(test_script, script_alias, prune, "0.9", the_parameters)
+          mock(p).illustrate_alias(test_script, script_alias, prune, no_browser, "0.9", the_parameters)
         end
-        c.illustrate(test_script, script_alias, "0.9", the_parameters, prune)
+        c.illustrate(test_script, script_alias, "0.9", the_parameters, prune, no_browser)
       end
     end
 

data/spec/mortar/local/pig_spec.rb

@@ -19,6 +19,7 @@ require 'fakefs/spec_helpers'
 require 'mortar/local/pig'
 require 'mortar/auth'
 require 'launchy'
+require 'mortar/pigversion'
 
 class Mortar::Local::Pig
   attr_reader :command
@@ -131,7 +132,9 @@ module Mortar::Local
         template_location = pig.resource_locations["illustrate_template"]
         template_contents = File.read(template_location)
         output_path = pig.resource_destinations["illustrate_html"]
-        mock(pig).decode_illustrate_input_file("foo/bar/file.json").returns(fake_illustrate_data)
+        temp_text = "This is temporary text"
+        mock(pig).decode_illustrate_input_file("foo/bar/file.json").returns(temp_text)
+        mock(pig).json_decode(temp_text).returns(fake_illustrate_data)
 
         # TODO: test that these files are copied
         ["illustrate_css", 
@@ -149,7 +152,7 @@ module Mortar::Local
           File.open(template_location, 'w') { |f| f.write(template_contents) }
           begin
             previous_stdout, $stdout = $stdout, StringIO.new
-            pig.show_illustrate_output("foo/bar/file.json")
+            pig.show_illustrate_output_browser("foo/bar/file.json")
           ensure
             $stdout = previous_stdout
           end
@@ -172,7 +175,8 @@ module Mortar::Local
         illustrate_output_file = 'illustrate-output.json'
         FakeFS do
           File.open(illustrate_output_file, 'w') { |f| f.write(json) }
-          actual_data = pig.decode_illustrate_input_file(illustrate_output_file)
+          actual_data_raw = pig.decode_illustrate_input_file(illustrate_output_file)
+          actual_data = pig.json_decode(actual_data_raw)
           expect(actual_data).to eq(expected_data)
         end
       end
@@ -187,7 +191,8 @@ module Mortar::Local
         illustrate_output_file = 'illustrate-output.json'
         FakeFS do
           File.open(illustrate_output_file, 'w') { |f| f.write(json) }
-          actual_data = pig.decode_illustrate_input_file(illustrate_output_file)
+          actual_data_raw = pig.decode_illustrate_input_file(illustrate_output_file)
+          actual_data = pig.json_decode(actual_data_raw)
           expect(actual_data).to eq(expected_data)
         end
       end
@@ -201,9 +206,19 @@ module Mortar::Local
         script = Mortar::Project::PigScripts.new('/foo/bar/baz.pig', 'baz', 'pig')
         pig = Mortar::Local::Pig.new
         mock(pig).run_pig_command.with_any_args.returns(true)
-        mock(pig).show_illustrate_output.with_any_args
+        mock(pig).show_illustrate_output_browser.with_any_args
         stub(pig).make_pig_param_file.returns('param.file')
-        pig.illustrate_alias(script, 'my_alias', false, "0.9", [])
+        pig.illustrate_alias(script, 'my_alias', false, false, "0.9", [])
+      end
+
+      it "displays text results if illustrate was successful with no_browser" do
+        any_instance_of(Mortar::Project::PigScripts, :elements => nil, :path => "/foo/bar/baz.pig")
+        script = Mortar::Project::PigScripts.new('/foo/bar/baz.pig', 'baz', 'pig')
+        pig = Mortar::Local::Pig.new
+        stub(pig).run_pig_command.with_any_args.returns(true)
+        mock(pig).display.with_any_args
+        stub(pig).make_pig_param_file.returns('param.file')
+        pig.illustrate_alias(script, 'my_alias', false, true, "0.9", [])
       end
 
       it "skips results if illustrate was unsuccessful" do
@@ -211,9 +226,9 @@ module Mortar::Local
         script = Mortar::Project::PigScripts.new('/foo/bar/baz.pig', 'baz', 'pig')
         pig = Mortar::Local::Pig.new
         mock(pig).run_pig_command.with_any_args.returns(false)
-        mock(pig).show_illustrate_output.with_any_args.never
+        mock(pig).show_illustrate_output_browser.with_any_args.never
         stub(pig).make_pig_param_file.returns('param.file')
-        pig.illustrate_alias(script, 'my_alias', false, "0.9", [])
+        pig.illustrate_alias(script, 'my_alias', false, false, "0.9", [])
       end
 
       it "does not require login credentials for illustration" do
@@ -222,8 +237,8 @@ module Mortar::Local
         pig = Mortar::Local::Pig.new
         mock(Mortar::Auth).user_s3_safe(true).returns('notloggedin-user-org')
         mock(pig).run_pig_command.with_any_args.returns(true)
-        mock(pig).show_illustrate_output.with_any_args
-        pig.illustrate_alias(script, 'my_alias', false, "0.9", [])
+        mock(pig).show_illustrate_output_browser.with_any_args
+        pig.illustrate_alias(script, 'my_alias', false, false, "0.9", [])
       end
 
     end

data/spec/mortar/local/python_spec.rb

@@ -128,5 +128,26 @@ module Mortar::Local
 
     end
 
+    context "running python commands" do
+
+      it "Generates the appropriate tempate variables" do
+        with_git_initialized_project do |p|
+          script_name = "some_luigi_script"
+          script_path = File.join(p.luigiscripts_path, "#{script_name}.py")
+          write_file(script_path)
+          luigi_script = Mortar::Project::LuigiScript.new(script_name, script_path)
+          args = %W{--myoption 2 --myotheroption 3}
+          py = Mortar::Local::Python.new
+          expected_hash = {
+            :python_arugments => "",
+            :python_script => "luigiscripts/#{script_name}.py",
+            :script_arguments => "--local-scheduler --logging-conf-file #{py.luigi_logging_config_file_path} --myoption 2 --myotheroption 3",
+          }
+          expect(py.luigi_command_template_parameters(luigi_script, args)).to eq(expected_hash)
+        end
+      end
+
+    end
+
   end
 end

metadata

@@ -1,36 +1,32 @@
 --- !ruby/object:Gem::Specification
 name: mortar
 version: !ruby/object:Gem::Version
-  version: 0.14.1
-  prerelease: 
+  version: 0.15.0
 platform: ruby
 authors:
 - Mortar Data
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-18 00:00:00.000000000 Z
+date: 2014-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 4.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 4.0.0
 - !ruby/object:Gem::Dependency
   name: mortar-api-ruby
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -46,7 +41,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: netrc
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -54,7 +48,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -62,7 +55,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: launchy
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -70,7 +62,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -78,7 +69,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: parseconfig
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -86,7 +76,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -94,7 +83,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: excon
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -102,7 +90,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -110,7 +97,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: fakefs
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -118,7 +104,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -126,65 +111,57 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: gem-release
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rr
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: Client library and command-line tool to interact with the Mortar service.
@@ -224,6 +201,7 @@ files:
 - lib/mortar/command/projects.rb
 - lib/mortar/command/validate.rb
 - lib/mortar/command/version.rb
+- lib/mortar/conf/luigi/logging.ini
 - lib/mortar/errors.rb
 - lib/mortar/generators/characterize_generator.rb
 - lib/mortar/generators/controlscript_generator.rb
@@ -283,6 +261,7 @@ files:
 - lib/mortar/templates/project/vendor/udfs/python/gitkeep
 - lib/mortar/templates/report/illustrate-report.html
 - lib/mortar/templates/script/runpig.sh
+- lib/mortar/templates/script/runpython.sh
 - lib/mortar/templates/udf/python_udf.py
 - lib/mortar/updater.rb
 - lib/mortar/version.rb
@@ -320,26 +299,25 @@ files:
 - spec/support/display_message_matcher.rb
 homepage: http://mortardata.com/
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: 1.8.7
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.0.6
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Client library and CLI to interact with the Mortar service.
 test_files: []