mortar 0.15.26 → 0.15.27

data/lib/mortar/auth.rb

@@ -72,16 +72,24 @@ class Mortar::Auth
       @credentials = ask_for_and_save_credentials
     end
 
-    def user    # :nodoc:
-      get_credentials[0]
+    def user(local=false)    # :nodoc:
+      if (local && !has_credentials)
+        "notloggedin@user.org"
+      else
+        get_credentials[0]
+      end
     end
 
-    def password    # :nodoc:
-      get_credentials[1]
+    def password(local=false)    # :nodoc:
+      if (local && !has_credentials)
+        "notloggedin"
+      else
+        get_credentials[1]
+      end
     end
 
     def user_s3_safe(local = false)
-      user_email = (local && !has_credentials) ? "notloggedin@user.org" : user
+      user_email = user(local)
       return user_email.gsub(/[^0-9a-zA-Z]/i, '-')
     end
 

data/lib/mortar/command/base.rb

@@ -104,6 +104,26 @@ class Mortar::Command::Base
     param_list
   end
   
+  def luigi_parameters
+    parameters = []
+    invalid_arguments.each_slice(2) do |arg_pair|
+      name_with_dashes = arg_pair[0]
+      unless (name_with_dashes.start_with? "--") && (name_with_dashes.length > 2)
+        error("Luigi parameter #{name_with_dashes} must begin with --")
+      end
+
+      unless arg_pair.length == 2
+        error("No value provided for luigi parameter #{name_with_dashes}")
+      end
+
+      name = name_with_dashes[2..name_with_dashes.length-1]
+      value = arg_pair[1]
+      parameters << {"name" => name, "value" => value}
+    end
+
+    parameters
+  end
+
   def pig_parameters
     paramfile_params = {}
     if options[:param_file]

data/lib/mortar/command/help.rb

@@ -23,7 +23,7 @@ require "mortar/command/base"
 #
 class Mortar::Command::Help < Mortar::Command::Base
 
-  PRIMARY_NAMESPACES = %w( auth clusters generate illustrate jobs pigscripts projects )
+  PRIMARY_NAMESPACES = %w( auth config clusters generate jobs local luigi projects )
 
   # help [COMMAND]
   #

data/lib/mortar/command/jobs.rb

@@ -164,7 +164,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
           cluster_type = CLUSTER_TYPE__PERMANENT
         end
         use_spot_instances = options[:spot] || false
-        api.post_job_new_cluster(project_name, script_name, git_ref, cluster_size, 
+        api.post_pig_job_new_cluster(project_name, script_name, git_ref, cluster_size, 
           :pig_version => pig_version.version, 
           :project_script_path => script.rel_path,
           :parameters => pig_parameters,
@@ -174,7 +174,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
           :use_spot_instances => use_spot_instances).body
       else
         cluster_id = options[:clusterid]
-        api.post_job_existing_cluster(project_name, script_name, git_ref, cluster_id,
+        api.post_pig_job_existing_cluster(project_name, script_name, git_ref, cluster_id,
           :pig_version => pig_version.version, 
           :project_script_path => script.rel_path,
           :parameters => pig_parameters,

data/lib/mortar/command/local.rb

@@ -225,12 +225,12 @@ class Mortar::Command::Local < Mortar::Command::Base
 
   # local:luigi SCRIPT
   #
-  # Run a luigi workflow on your local machine in local scheduler mode.
+  # Run a luigi pipeline script on your local machine in local scheduler mode.
   # Any additional command line arguments will be passed directly to the luigi script.
   #
-  # -p, --parameter NAME=VALUE  # Set a pig parameter value in your script.
-  # -f, --param-file PARAMFILE  # Load pig parameter values from a file.
   # --project-root PROJECTDIR   # The root directory of the project if not the CWD
+  # -p, --parameter NAME=VALUE  # [deprecated] Instead, pass luigi parameters directly as options (see below)
+  # -f, --param-file PARAMFILE  # [deprecated] Instead, pass luigi parameters directly as options (see below)
   #
   #Examples:
   #
@@ -241,8 +241,7 @@ class Mortar::Command::Local < Mortar::Command::Base
     unless script_name
       error("Usage: mortar local:luigi SCRIPT\nMust specify SCRIPT.")
     end
-    validate_arguments!
-
+    
     # cd into the project root
     project_root = options[:project_root] ||= Dir.getwd
     unless File.directory?(project_root)
@@ -257,10 +256,24 @@ class Mortar::Command::Local < Mortar::Command::Base
     git_ref = sync_code_with_cloud()
     ENV['MORTAR_LUIGI_GIT_REF'] = git_ref
 
+    # pick up standard luigi-style params provided by the user
+    luigi_cli_parameters = luigi_parameters()
+
+    # pick up old pig-style parameters (included for backwards compatibility)
+    pig_style_parameters = pig_parameters()
+    if pig_style_parameters.length > 0
+      warn "[DEPRECATION] Passing luigi parameters with -p is deprecated. Please pass them directly (e.g. --myparam myvalue)"
+    end
+
+    luigi_cli_parameters.concat(pig_style_parameters)
+    cli_parameters = \
+      luigi_cli_parameters.sort_by { |p| p['name'] }.map { |arg| ["--#{arg['name']}", "#{arg['value']}"] }.flatten
+
+    # get project configuration parameters
+    project_config_params = config_parameters()
+
     ctrl = Mortar::Local::Controller.new
-    luigi_params = pig_parameters.sort_by { |p| p['name'] }
-    luigi_params = luigi_params.map { |arg| ["--#{arg['name']}", "#{arg['value']}"] }.flatten
-    ctrl.run_luigi(pig_version, script, luigi_params)
+    ctrl.run_luigi(pig_version, script, cli_parameters, project_config_params)
   end
 
   # local:sqoop_table dbtype database-name table s3-destination

data/lib/mortar/command/luigi.rb

@@ -0,0 +1,86 @@
+#
+# Copyright 2014 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "mortar/command/base"
+require "time"
+
+# run luigi pipeline jobs
+#
+class Mortar::Command::Luigi < Mortar::Command::Base
+
+  include Mortar::Git
+
+  # luigi SCRIPT
+  #
+  # Run a luigi pipeline.
+  #
+  # -P, --project PROJECTNAME   # Use a project that is not checked out in the current directory.  Runs code from project's master branch in GitHub rather than snapshotting local code.
+  # -B, --branch BRANCHNAME     # Used with --project to specify a non-master branch
+  #
+  # Examples:
+  #
+  #    Run the nightly_rollup luigiscript:
+  #        $ mortar luigi luigiscripts/nightly_rollup.py
+  #
+  #    Run the nightly_rollup luigiscript with two parameters:
+  #        $ mortar luigi luigiscripts/nightly_rollup.py --data-date 2012-02-01 --my-param myval
+  #
+  def index
+    script_name = shift_argument
+    unless script_name
+      error("Usage: mortar luigi SCRIPT\nMust specify SCRIPT.")
+    end
+    
+    if options[:project]
+      project_name = options[:project]
+      if File.extname(script_name) == ".py"
+        script_name = File.basename(script_name, ".*")
+      end
+    else
+      project_name = project.name
+      script = validate_luigiscript!(script_name)
+      script_name = script.name
+    end
+
+    parameters = luigi_parameters()
+
+    if options[:project]
+      if options[:branch]
+        git_ref = options[:branch]
+      else
+        git_ref = "master"
+      end
+    else
+      git_ref = sync_code_with_cloud()
+    end
+    
+    # post job to API 
+    response = action("Requesting job execution") do      
+      api.post_luigi_job(project_name, script_name, git_ref,
+        :project_script_path => script.rel_path,
+        :parameters => parameters).body
+    end
+    
+    display("job_id: #{response['job_id']}")
+    display
+    display("Job status can be viewed on the web at:\n\n #{response['web_job_url']}")
+    display
+
+    response['job_id']
+  end
+  
+  alias_command "luigi:run", "luigi"
+end

data/lib/mortar/generators/project_generator.rb

@@ -72,6 +72,8 @@ module Mortar
 
           inside "luigiscripts" do
             copy_file "README", "README"
+            generate_file "luigiscript.py", "#{project_name}_luigi.py"
+            generate_file "client.cfg.template", "client.cfg.template"
           end
 
           mkdir "lib"

data/lib/mortar/git.rb

@@ -160,6 +160,19 @@ module Mortar
         manifest_pathlist
       end
 
+      def add_entry_to_mortar_project_manifest(path, entry)
+        contents = File.open(path, "r") do |manifest|
+          manifest.read.strip
+        end
+
+        if contents && (! contents.include? entry)
+          new_contents = "#{contents}\n#{entry}\n"
+          File.open(path, "w") do |manifest|
+            manifest.write new_contents
+          end
+        end
+      end
+
       def add_newline_to_file(path)
         File.open(path, "r+") do |manifest|
           contents = manifest.read()
@@ -189,12 +202,25 @@ module Mortar
       #
       def ensure_valid_mortar_project_manifest()
         if File.exists? project_manifest_name
+          ensure_luigiscripts_in_project_manifest()
           add_newline_to_file(project_manifest_name)
         else
           create_mortar_project_manifest('.')
         end
       end
 
+      #
+      # Ensure that the luigiscripts directory,
+      # which was added after some project manifests were
+      # created, is in the manifest (if luigiscripts exists).
+      #
+      def ensure_luigiscripts_in_project_manifest
+        luigiscripts_path = "luigiscripts"
+        if File.directory? luigiscripts_path
+          add_entry_to_mortar_project_manifest(project_manifest_name, luigiscripts_path)
+        end
+      end
+
       #
       # Create a project manifest file
       #
@@ -207,6 +233,10 @@ module Mortar
           if File.directory? "#{path}/lib"
             manifest.puts "lib"
           end
+
+          if File.directory? "#{path}/luigiscripts"
+            manifest.puts "luigiscripts"
+          end
         end
       end
     

data/lib/mortar/local/controller.rb

@@ -230,10 +230,14 @@ README
     pig.launch_repl(pig_version, pig_parameters)
   end
 
-  def run_luigi(pig_version, luigi_script, user_script_args)
+  def run_luigi(pig_version, luigi_script, luigi_script_parameters, project_config_parameters)
+    require_aws_keys
     install_and_configure(pig_version, 'luigi')
     py = Mortar::Local::Python.new()
-    py.run_luigi_script(luigi_script, user_script_args)
+    unless py.run_stillson_luigi_client_cfg_expansion(luigi_script, project_config_parameters)
+      error("Unable to expand your configuration template [luigiscripts/client.cfg.template] to [luigiscripts/client.cfg]")
+    end
+    py.run_luigi_script(luigi_script, luigi_script_parameters)
   end
 
   def sqoop_export_table(pig_version, connstr, dbtable, s3dest, options)

data/lib/mortar/local/installutil.rb

@@ -198,7 +198,12 @@ module Mortar
 
       def url_date(url, command=nil)
         result = head_resource(url, command)
-        http_date_to_epoch(result.get_header('Last-Modified'))
+        last_modified = result.get_header('Last-Modified')
+        if last_modified
+          http_date_to_epoch(last_modified)
+        else
+          nil
+        end
       end
 
       # Given a subdirectory where we have installed some software
@@ -211,6 +216,9 @@ module Mortar
           return true
         end
         remote_archive_date = url_date(url, command)
+        if not remote_archive_date
+          return false
+        end
         return existing_install_date < remote_archive_date
       end
 

data/lib/mortar/local/params.rb

@@ -0,0 +1,68 @@
+#
+# Copyright 2014 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'set'
+require 'mortar/auth'
+
+module Mortar
+  module Local
+    module Params
+
+      # Job parameters that are supplied automatically from Mortar when
+      # running on the server side.  We duplicate these here.
+      def automatic_parameters()
+        params = {}
+
+        params['MORTAR_EMAIL']   = Mortar::Auth.user(true)
+        params['MORTAR_API_KEY'] = Mortar::Auth.password(true)
+        
+        if ENV['MORTAR_EMAIL_S3_ESCAPED']
+          params['MORTAR_EMAIL_S3_ESCAPED'] = ENV['MORTAR_EMAIL_S3_ESCAPED']
+        else
+          params['MORTAR_EMAIL_S3_ESCAPED'] = Mortar::Auth.user_s3_safe(true)
+        end
+
+        if ENV['MORTAR_PROJECT_ROOT']
+          params['MORTAR_PROJECT_ROOT'] = ENV['MORTAR_PROJECT_ROOT']
+        else
+          params['MORTAR_PROJECT_ROOT'] = project_root
+          ENV['MORTAR_PROJECT_ROOT'] = params['MORTAR_PROJECT_ROOT']
+        end
+
+        params['AWS_ACCESS_KEY']    = ENV['AWS_ACCESS_KEY']
+        params['AWS_ACCESS_KEY_ID'] = ENV['AWS_ACCESS_KEY']
+        params['aws_access_key_id'] = ENV['AWS_ACCESS_KEY']
+
+        params['AWS_SECRET_KEY'] = ENV['AWS_SECRET_KEY']
+        params['AWS_SECRET_ACCESS_KEY'] = ENV['AWS_SECRET_KEY']
+        params['aws_secret_access_key'] = ENV['AWS_SECRET_KEY']
+        
+        param_list = params.map do |k,v|
+          {"name" => k, "value" => v}
+        end
+      end
+
+      # Merge two lists of parameters, removing dupes.
+      # Parameters in param_list_1 override those in param_list_2
+      def merge_parameters(param_list_0, param_list_1)
+        param_list_1_keys = Set.new(param_list_1.map{|item| item["name"]})
+        merged = param_list_1.clone
+        merged.concat(param_list_0.select{|item| (! param_list_1_keys.include? item["name"]) })
+        merged
+      end
+    end
+  end
+end

data/lib/mortar/local/pig.rb

@@ -18,9 +18,11 @@ require "erb"
 require 'tempfile'
 require "mortar/helpers"
 require "mortar/local/installutil"
+require "mortar/local/params"
 
 class Mortar::Local::Pig
   include Mortar::Local::InstallUtil
+  include Mortar::Local::Params
 
   PIG_LOG_FORMAT = "humanreadable"
   LIB_TGZ_NAME = "lib-common.tar.gz"
@@ -399,39 +401,11 @@ class Mortar::Local::Pig
     return opts
   end
 
-  # Pig Paramenters that are supplied directly from Mortar when
-  # running on the server side.  We duplicate these here.
-  def automatic_pig_parameters
-    params = {}
-
-    if ENV['MORTAR_EMAIL_S3_ESCAPED']
-      params['MORTAR_EMAIL_S3_ESCAPED'] = ENV['MORTAR_EMAIL_S3_ESCAPED']
-    else
-      params['MORTAR_EMAIL_S3_ESCAPED'] = Mortar::Auth.user_s3_safe(true)
-    end
-
-    if ENV['MORTAR_PROJECT_ROOT']
-      params['MORTAR_PROJECT_ROOT'] = ENV['MORTAR_PROJECT_ROOT']
-    else
-      params['MORTAR_PROJECT_ROOT'] = project_root
-      ENV['MORTAR_PROJECT_ROOT'] = params['MORTAR_PROJECT_ROOT']
-    end
-
-
-    # Coerce into the same format as pig parameters that were
-    # passed in via the command line or a parameter file
-    param_list = []
-    params.each{ |k,v|
-      param_list.push({"name" => k, "value" => v})
-    }
-    return param_list
-  end
-
   # Given a set of user specified pig parameters, combine with the
   # automatic mortar parameters and write out to a tempfile, returning
   # it's path so it may be referenced later in the process
   def make_pig_param_file(pig_parameters)
-    mortar_pig_params = automatic_pig_parameters
+    mortar_pig_params = automatic_parameters()
     all_parameters = mortar_pig_params.concat(pig_parameters)
     param_file = Tempfile.new("mortar-pig-parameters")
     all_parameters.each { |p|
@@ -447,4 +421,9 @@ class Mortar::Local::Pig
     param_file.path
   end
 
+  def automatic_pig_parameters
+    warn "[DEPRECATION] Please call automatic_parameters instead"
+    automatic_parameters
+  end
+  
 end