mortar 0.15.26 → 0.15.27

data/lib/mortar/local/python.rb

@@ -15,15 +15,17 @@
 #
 
 require "mortar/local/installutil"
+require "mortar/local/params"
 
 class Mortar::Local::Python
   include Mortar::Local::InstallUtil
+  include Mortar::Local::Params
 
   PYTHON_OSX_TGZ_NAME = "mortar-python-osx.tgz"
   PYTHON_OSX_TGZ_DEFAULT_URL_PATH = "resource/python_osx"
   PYPI_URL_PATH = "resource/mortar_pypi"
 
-  MORTAR_PYTHON_PACKAGES = ["luigi", "mortar-luigi"]
+  MORTAR_PYTHON_PACKAGES = ["luigi", "mortar-luigi", "stillson"]
 
   # Path to the python binary that should be used
   # for running UDFs
@@ -318,7 +320,19 @@ class Mortar::Local::Python
 
   def run_luigi_script(luigi_script, user_script_args)
     template_params = luigi_command_template_parameters(luigi_script, user_script_args)
-    return run_templated_script(python_command_script_template_path, template_params)
+    run_templated_script(python_command_script_template_path, template_params)
+  end
+
+  def run_stillson_luigi_client_cfg_expansion(luigi_script, project_config_parameters)
+    # combine automatic mortar parameters with 
+    # parameters provided in the project config
+    auto_params = automatic_parameters()
+    parameters = merge_parameters(auto_params, project_config_parameters)
+    stillson_template_params = {
+      :parameters => parameters,
+      :luigi_script => luigi_script.executable_path()
+    }
+    run_templated_script(stillson_command_script_template_path, stillson_template_params)
   end
 
   # Path to the template which generates the bash script for running python
@@ -326,6 +340,11 @@ class Mortar::Local::Python
     File.expand_path("../../templates/script/runpython.sh", __FILE__)
   end
 
+  # Path to the template which generates the bash script for running stillson
+  def stillson_command_script_template_path
+    File.expand_path("../../templates/script/runstillson.sh", __FILE__)
+  end
+
   def luigi_logging_config_file_path
     File.expand_path("../../conf/luigi/logging.ini", __FILE__)
   end

data/lib/mortar/templates/project/gitignore

@@ -2,6 +2,7 @@ Gemfile.lock
 *.pyc
 *.class
 *.log
+luigiscripts/client.cfg
 tmp
 .mortar-fork
 .mortar-local

data/lib/mortar/templates/project/luigiscripts/client.cfg.template

@@ -0,0 +1,33 @@
+# 
+# This is the configuration file your Luigi pipelines.
+#
+# When you run "mortar local:luigi" or "mortar luigi", Mortar will expand
+# the variables in this file (e.g. MORTAR_EMAIL) to their actual values
+# and store the result in "luigiscripts/client.cfg". You should not check
+# "luigiscripts/client.cfg" into source control, as it will be generated
+# on each new run.
+# 
+# In this file, you can reference any internal Mortar configuration variables 
+# (e.g. MORTAR_EMAIL, AWS_ACCESS_KEY_ID), or any
+# project configuration variables you've set with "mortar config:set".
+# 
+# Read more at https://help.mortardata.com/technologies/luigi/luigi_configuration
+#
+
+
+# 
+# Mortar credentials to run jobs and query for
+# state from the Mortar API.
+#
+[mortar]
+email: ${MORTAR_EMAIL}
+api_key: ${MORTAR_API_KEY}
+host: api.mortardata.com
+
+# 
+# AWS credentials to check for completed output
+# and store job completion tokens in S3 from Luigi.
+#
+[s3]
+aws_access_key_id: ${AWS_ACCESS_KEY_ID}
+aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY}

data/lib/mortar/templates/project/luigiscripts/luigiscript.py

@@ -0,0 +1,134 @@
+import luigi
+from luigi import configuration
+from luigi.s3 import S3Target, S3PathTask
+
+from mortar.luigi import mortartask
+
+"""
+  Luigi is a powerful, easy-to-use framework for building data pipelines.
+
+  This is an example Luigi script to get you started. This script has a 
+  'fill in the blank' interaction. Feel free to alter it to build your pipeline.
+
+  In this example we will run a Pig script, and then shutdown any clusters associated
+  with that script.  We will do that by running the ShutdownClusters Task,
+  which is dependent on RunMyExamplePigScript Task. This means the cluster will only 
+  shutdown after RunMyExamplePigScript (where the data transformation happens) 
+  Task is completed.
+
+  For full tutorials and in-depth Luigi documentation, visit:
+  https://help.mortardata.com/technologies/luigi
+
+  To Run:
+    mortar luigi luigiscripts/<%= project_name_alias %>_luigi.py \
+      --output-base-path "s3://mortar-example-output-data/<your_username_here>/<%= project_name_alias %>"
+"""
+
+MORTAR_PROJECT = '<%= project_name_alias %>'
+
+# helper function
+def create_full_path(base_path, sub_path):
+  return '%s/%s' % (base_path, sub_path)
+
+class RunMyExamplePigScript(mortartask.MortarProjectPigscriptTask):
+  """
+  This is a Luigi Task that extends MortarProjectPigscriptTask to run a Pig
+  script on Mortar.
+  """
+
+  """
+  The location in S3 where the output of the Mortar job will be written.
+  """
+  output_base_path = luigi.Parameter() 
+
+  """
+  Default cluster size to use for running Mortar jobs.  A cluster size of 0
+  will run in Mortar's local mode.  This is a fast (and free!) way to run jobs
+  on small data samples.  Cluster sizes >= 2 will run on a Hadoop cluster.
+  """
+  cluster_size = luigi.IntParameter(default=0)
+
+  """
+  Path to input data being analyzed using the Pig script.
+  """
+  input_path = luigi.Parameter(default ='s3://mortar-example-data/tutorial/excite.log.bz2')
+
+
+  def requires(self):
+    """
+    The requires method specifies a list of other Tasks that must be complete
+    for this Task to run. In this case, we want to require that our input
+    exists on S3 before we run the script. S3PathTask validates that the
+    specified file or directory exists on S3.
+    """
+    return [S3PathTask(self.input_path)]
+
+  def project(self):
+    """
+    Name of Mortar Project to run.
+    """
+    return MORTAR_PROJECT
+
+  def script_output(self):
+    """
+    The script_output method is how you define where the output from this task
+    will be stored. Luigi will check this output location before starting any
+    tasks that depend on this task.
+    """
+    return[S3Target(self.output_base_path + '/pigoutput')]
+  
+  def token_path(self):
+    """
+    Luigi manages dependencies between tasks by checking for the existence of
+    files.  When one task finishes it writes out a 'token' file that will
+    trigger the next task in the dependency graph.  This is the base path for
+    where those tokens will be written.
+    """
+    return self.output_base_path
+  
+  def parameters(self):
+    """
+    Parameters for this pig job.
+    """
+    return {'INPUT_PATH': self.input_path,
+            'OUTPUT_PATH': self.output_base_path + '/pigoutput'}
+  
+  def script(self):
+    """
+    Name of Pig script to run.
+    """
+    return '<%= project_name_alias %>'
+
+
+class ShutdownClusters(mortartask.MortarClusterShutdownTask):
+  """
+  When the pipeline is completed, this task shuts down all active clusters not
+  currently running jobs. As this task is only shutting down clusters and not
+  generating any output data, this S3 location is used to store a 'token' file
+  indicating when the task has been completed.
+  """
+  output_base_path = luigi.Parameter() 
+
+  def requires(self):
+    """
+    The ShutdownClusters task is dependent on RunMyExamplePigScript because a
+    cluster should not shut down until all the tasks are completed. You can
+    think of this as saying 'shut down my cluster after running my task'.
+    """
+    return RunMyExamplePigScript(output_base_path = self.output_base_path)
+
+  def output(self):
+    """
+    This output statement is needed because ShutdownClusters has no actual
+    output. We write a token with the class name to S3 to know that this task
+    has completed and it does not need to be run again.
+    """
+    return [S3Target((create_full_path(self.output_base_path, 'ShutdownClusters')))]
+
+if __name__ == "__main__":
+  """
+  The final task in your pipeline, which will in turn pull in any dependencies
+  need to be run should be called in the main method. In this case ShutdownClusters
+  is being called.
+  """
+  luigi.run(main_task_cls= ShutdownClusters)

data/lib/mortar/templates/project/project.manifest

@@ -7,6 +7,7 @@
 # for more information.
 
 lib
+luigiscripts
 macros
 pigscripts
 udfs

data/lib/mortar/templates/script/runstillson.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -e
+
+# Setup python environment
+source <%= @local_install_dir %>/pythonenv/bin/activate
+
+# template config file
+export LUIGI_CONFIG_TEMPLATE_PATH=`pwd`/`dirname <%= @luigi_script %>`/client.cfg.template
+
+# expanded config file
+export LUIGI_CONFIG_PATH=`pwd`/`dirname <%= @luigi_script %>`/client.cfg
+
+# Setup parameters in environment variables
+<% @parameters.each do |p| %>
+export <%= p['name'] %>="<%= p['value'] %>";
+<% end %>
+
+# Run stillson to expand the configuration template
+if [ -f "$LUIGI_CONFIG_TEMPLATE_PATH" ] 
+then 
+    stillson "$LUIGI_CONFIG_TEMPLATE_PATH" -o $LUIGI_CONFIG_PATH
+else
+    echo "No luigi client configuration template found in expected location $LUIGI_CONFIG_TEMPLATE_PATH. Not expanding."
+fi

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.15.26"
+  VERSION = "0.15.27"
 end

data/spec/mortar/auth_spec.rb

@@ -182,5 +182,13 @@ module Mortar
       end
     end
 
+    it "does not prompt for username in local mode if user not logged in" do
+      @cli.user(true).should == 'notloggedin@user.org'
+      @cli.user_s3_safe(true).should == 'notloggedin-user-org'
+    end
+
+    it "does not prompt for password in local mode if user not logged in" do
+      @cli.password(true).should == 'notloggedin'
+    end
   end
 end

data/spec/mortar/command/generate_spec.rb

@@ -39,6 +39,7 @@ describe Mortar::Command::Generate do
       File.exists?("Test/macros/.gitkeep").should be_true
       File.exists?("Test/pigscripts/Test.pig").should be_true
       File.exists?("Test/udfs/python/Test.py").should be_true
+      File.exists?("Test/luigiscripts/README").should be_true
 
       File.read("Test/pigscripts/Test.pig").each_line { |line| line.match(/<%.*%>/).should be_nil }
     end

data/spec/mortar/command/jobs_spec.rb

@@ -46,7 +46,7 @@ module Mortar::Command
           job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
           cluster_size = 5
 
-          mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
+          mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :pig_version => "0.9",
             :project_script_path => be_a_kind_of(String),
             :parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]), 
@@ -82,7 +82,7 @@ STDOUT
           job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
           cluster_size = 5
 
-          mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String),cluster_size,
+          mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String),cluster_size,
                                                       :pig_version => "0.9", 
                                                       :project_script_path => be_a_kind_of(String),
                                                       :parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
@@ -142,7 +142,7 @@ STDERR
           job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
           cluster_size = 5
 
-          mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String),cluster_size, 
+          mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String),cluster_size, 
             :pig_version => "0.9", 
             :project_script_path => be_a_kind_of(String),
             :parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]), 
@@ -178,7 +178,7 @@ STDOUT
           job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
           cluster_size = 5
 
-          mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String),cluster_size, 
+          mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String),cluster_size, 
             :pig_version => "0.9", 
             :project_script_path => be_a_kind_of(String),
             :parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]), 
@@ -214,7 +214,7 @@ STDOUT
           job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
           cluster_id = "e2790e7e8c7d48e39157238d58191346"
 
-          mock(Mortar::Auth.api).post_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id,
+          mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id,
               :pig_version => "0.9", 
               :project_script_path => be_a_kind_of(String),
               :parameters => [],
@@ -248,7 +248,7 @@ STDOUT
           job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
           cluster_id = "e2790e7e8c7d48e39157238d58191346"
 
-          mock(Mortar::Auth.api).post_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id,
+          mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id,
               :pig_version => "0.9", 
               :project_script_path => be_a_kind_of(String),
               :parameters => [],
@@ -282,7 +282,7 @@ STDOUT
           cluster_size = 2
 
           mock(Mortar::Auth.api).get_clusters() {Excon::Response.new(:body => {'clusters' => []})}
-          mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
+          mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :pig_version => "0.9", 
             :project_script_path => be_a_kind_of(String),
             :parameters => [], 
@@ -320,7 +320,7 @@ STDOUT
           cluster_size = 2
 
           mock(Mortar::Auth.api).get_clusters() {Excon::Response.new(:body => {'clusters' => []})}
-          mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
+          mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :pig_version => "0.9", 
             :project_script_path => be_a_kind_of(String),
             :parameters => [], 
@@ -358,7 +358,7 @@ STDOUT
           job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
           cluster_id = "e2790e7e8c7d48e39157238d58191346"
 
-          mock(Mortar::Auth.api).post_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id, :pig_version => "0.9", :project_script_path => be_a_kind_of(String), :parameters => [], :notify_on_job_finish => false, :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
+          mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id, :pig_version => "0.9", :project_script_path => be_a_kind_of(String), :parameters => [], :notify_on_job_finish => false, :is_control_script=>false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
 
           write_file(File.join(p.pigscripts_path, "my_script.pig"))
           stderr, stdout = execute("jobs:run pigscripts/my_script.pig --clusterid e2790e7e8c7d48e39157238d58191346 -d", p, @git)
@@ -410,7 +410,7 @@ STDOUT
                        'job_name' => "", 'start_timestamp' => ""} ], 'status_code' => huge_busy_cluster_status  }
               ]})
           }
-          mock(Mortar::Auth.api).post_job_existing_cluster("myproject", "my_script", is_a(String), large_cluster_id, 
+          mock(Mortar::Auth.api).post_pig_job_existing_cluster("myproject", "my_script", is_a(String), large_cluster_id, 
             :pig_version => "0.9", 
             :project_script_path => be_a_kind_of(String),
             :parameters => [], 
@@ -444,7 +444,7 @@ STDOUT
           job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
           cluster_size = 5
 
-          mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
+          mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :pig_version => "0.9",
             :project_script_path => be_a_kind_of(String), 
             :parameters => match_array([{"name" => "FIRST", "value" => "FOO"}, {"name" => "SECOND", "value" => "BAR"}, {"name" => "THIRD", "value" => "BEAR"}]), 
@@ -472,7 +472,7 @@ PARAMS
           job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
           cluster_size = 5
 
-          mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
+          mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :pig_version => "0.9", 
             :project_script_path => be_a_kind_of(String),
             :parameters => match_array([{"name" => "FIRST", "value" => "FOO"}, {"name" => "SECOND", "value" => "BAR"}, {"name" => "THIRD", "value" => "BEAR"}]), 
@@ -528,7 +528,7 @@ STDERR
 
           mock(@git).sync_embedded_project.with_any_args.times(1) { "somewhere_over_the_rainbow" }
 
-          mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
+          mock(Mortar::Auth.api).post_pig_job_new_cluster("myproject", "my_script", is_a(String), cluster_size, 
             :pig_version => "0.9", 
             :project_script_path => be_a_kind_of(String),
             :parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]), 

data/spec/mortar/command/local_spec.rb

@@ -289,29 +289,67 @@ STDERR
         end
       end
 
-      it "Runs script forwarding options to luigi script" do
+      it "Runs script with old pig-style parameters passed to luigi script" do
         with_git_initialized_project do |p|
+          ENV['MORTAR_LUIGI_GIT_REF'] = nil
           script_name = "some_luigi_script"
           script_path = File.join(p.luigiscripts_path, "#{script_name}.py")
           write_file(script_path)
           luigi_script = Mortar::Project::LuigiScript.new(script_name, script_path)
+          config_parameters = []
           mock(Mortar::Project::LuigiScript).new(script_name, script_path).returns(luigi_script)
+          any_instance_of(Mortar::Command::Local) do |u|
+            mock(u).config_parameters.returns(config_parameters)
+          end
           any_instance_of(Mortar::Local::Python) do |u|
             mock(u).run_luigi_script(luigi_script, %W{--myoption 2 --myotheroption 3})
+            mock(u).run_stillson_luigi_client_cfg_expansion(luigi_script, config_parameters).returns(true)
           end
           any_instance_of(Mortar::Local::Controller) do |u|
             mock(u).install_and_configure(is_a(Mortar::PigVersion::Pig09),'luigi')
+            mock(u).require_aws_keys()
           end
           any_instance_of(Mortar::Command::Local) do |u|
             mock(u).sync_code_with_cloud().returns("some-git-ref")
           end
           ENV['MORTAR_LUIGI_GIT_REF'].should be_nil
           stderr, stdout = execute("local:luigi some_luigi_script -p myoption=2 -p myotheroption=3", p)
-          stderr.should == ""
-          ENV['MORTAR_LUIGI_GIT_REF'].should == 'some-git-ref'
+          stderr.should == <<-STDERR
+[DEPRECATION] Passing luigi parameters with -p is deprecated. Please pass them directly (e.g. --myparam myvalue)
+STDERR
+          ENV['MORTAR_LUIGI_GIT_REF'].should == "some-git-ref"
         end
       end
 
+    it "Runs script with new luigi-style parameters passed to luigi script" do
+        with_git_initialized_project do |p|
+          ENV['MORTAR_LUIGI_GIT_REF'] = nil
+          script_name = "some_luigi_script"
+          script_path = File.join(p.luigiscripts_path, "#{script_name}.py")
+          write_file(script_path)
+          luigi_script = Mortar::Project::LuigiScript.new(script_name, script_path)
+          config_parameters = []
+          mock(Mortar::Project::LuigiScript).new(script_name, script_path).returns(luigi_script)
+          any_instance_of(Mortar::Command::Local) do |u|
+            mock(u).config_parameters.returns(config_parameters)
+          end
+          any_instance_of(Mortar::Local::Python) do |u|
+            mock(u).run_stillson_luigi_client_cfg_expansion(luigi_script, config_parameters).returns(true)
+            mock(u).run_luigi_script(luigi_script, %W{--myoption 2 --myotheroption 3})
+          end
+          any_instance_of(Mortar::Local::Controller) do |u|
+            mock(u).install_and_configure(is_a(Mortar::PigVersion::Pig09),'luigi')
+            mock(u).require_aws_keys()
+          end
+          any_instance_of(Mortar::Command::Local) do |u|
+            mock(u).sync_code_with_cloud().returns("some-git-ref")
+          end
+          ENV['MORTAR_LUIGI_GIT_REF'].should be_nil
+          stderr, stdout = execute("local:luigi some_luigi_script --myoption 2 --myotheroption 3", p)
+          stderr.should == ""
+          ENV['MORTAR_LUIGI_GIT_REF'].should == "some-git-ref"
+        end
+      end
 
     end