mortar 0.15.36 → 0.15.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8038226d4444ee657c0f34ef9af8ed05deec35b5
-  data.tar.gz: 05086ca35f9a1ce136d0f679fe6d3ea000ee9aca
+  metadata.gz: 15cff7085bdc1d7475ce5de63a224f1e5f9af981
+  data.tar.gz: e273bd6668c0752d89cef8546c1c43cdc69f1d92
 SHA512:
-  metadata.gz: 4e6dcc9dee6926957ddce2e1b6f804d46dffd1be318659ff3c65e9cdb17695c986e6aaea3f037979fce108293afd3a75adab9addc5f6cf87648669565fd43797
-  data.tar.gz: 32dc2c1c42b58400465496b8169c21689cbac62873c8c16474027edb523a602178fab6633065beee99152ad3e70fa3467cfbe0fae8b1f6f22c3bdff62992df02
+  metadata.gz: 3c52e4a5fb9d3921626417c54aa3622d8494b81bb481155862918c05a9d1a69c3f1ceab0ac4485409533f383d2c1d1043b9c7d89293c35c27f12ade5e2d49ae7
+  data.tar.gz: c31c3f2135b0e592630cbf622d9815d7ccb28eb741826e203df3b11be0d427b5c936ec67e23543a5494ff1444ec1bee0eb91c990234ffe9a8d9c32d49845be84

data/lib/mortar/command/base.rb

@@ -296,7 +296,7 @@ protected
 
   def self.replace_templates(help)
     help.each do |line|
-      line.gsub!("<PIG_VERSION_OPTIONS>", "0.9 (default) and 0.12")
+      line.gsub!("<PIG_VERSION_OPTIONS>", "0.12 and 0.12-Hadoop-2")
     end
   end
 
@@ -608,7 +608,7 @@ protected
   end
 
   def pig_version
-    pig_version_str = options[:pigversion] || '0.9'
+    pig_version_str = options[:pigversion] || '0.12'
     pig_version = Mortar::PigVersion.from_string(pig_version_str)
   end
 

data/lib/mortar/command/config.rb

@@ -27,21 +27,27 @@ class Mortar::Command::Config < Mortar::Command::Base
   #
   # Display the config vars for a project.
   #
-  # -s, --shell  # output config vars in shell format
+  # -s, --shell  # output config vars in shell format.
   #
   #
   # $ mortar config
   # A: one
+  # B: three
+  #
+  # == Project config settings overriden by user config
   # B: two
   #
+  #
   # $ mortar config --shell
   # A=one
-  # B=two
+  # B=three
   #
   def index
     validate_arguments!
     project_name = options[:project] || project.name
-    vars = api.get_config_vars(project_name).body['config']
+    config_body = api.get_config_vars(project_name).body
+    vars = config_body['config']
+    shadow_vars = config_body['shadow_config'] || {}
     if vars.empty?
       display("#{project_name} has no config vars.")
     else
@@ -53,6 +59,12 @@ class Mortar::Command::Config < Mortar::Command::Base
       else
         styled_header("#{project_name} Config Vars")
         styled_hash(vars)
+        unless shadow_vars.empty?
+          shadow_vars.each {|key, value| shadow_vars[key] = value.to_s}
+          display("\n")
+          styled_header("Project config settings overriden by user config")
+          styled_hash(shadow_vars)
+        end
       end
     end
   end
@@ -86,6 +98,10 @@ class Mortar::Command::Config < Mortar::Command::Base
   #
   # Set one or more config vars
   #
+  # -u, --user  # Set the config value for your user.  Only visible to your user
+  #             # and will apply to all projects.  Will overwrite a project config
+  #             # variable with the same key.
+  #
   #Example:
   #
   # $ mortar config:set A=one
@@ -97,6 +113,11 @@ class Mortar::Command::Config < Mortar::Command::Base
   # A: one
   # B: two
   #
+  # $ mortar config:set B=three -u
+  # Setting config vars... done.
+  # A: one
+  # B: three
+  #
   def set
     unless args.size > 0 and args.all? { |a| a.include?('=') }
       error("Usage: mortar config:set KEY1=VALUE1 [KEY2=VALUE2 ...]\nMust specify KEY and VALUE to set.")
@@ -108,10 +129,15 @@ class Mortar::Command::Config < Mortar::Command::Base
       vars
     end
 
-    project_name = options[:project] || project.name
-
-    action("Setting config vars for project #{project_name}") do
-      api.put_config_vars(project_name, vars)
+    if options[:user]
+      action("Setting config vars for your user") do
+        api.put_user_config_vars(vars)
+      end
+    else
+      project_name = options[:project] || project.name
+      action("Setting config vars for project #{project_name}") do
+        api.put_config_vars(project_name, vars)
+      end
     end
 
     vars.each {|key, value| vars[key] = value.to_s}
@@ -125,6 +151,9 @@ class Mortar::Command::Config < Mortar::Command::Base
   #
   # unset one or more config vars
   #
+  # -u, --user  # Unset the config value for your user.  Only affecsts your user
+  #             # and will apply to all projects.
+  #
   # $ mortar config:unset A
   # Unsetting A... done
   #
@@ -132,14 +161,26 @@ class Mortar::Command::Config < Mortar::Command::Base
   # Unsetting A... done
   # Unsetting B... done
   #
+  # $ mortar config:unset A B -u
+  # Unsetting A... done
+  # Unsetting B... done
+  #
   def unset
     if args.empty?
       error("Usage: mortar config:unset KEY1 [KEY2 ...]\nMust specify KEY to unset.")
     end
-    project_name = options[:project] || project.name
-    args.each do |key|
-      action("Unsetting #{key} for project #{project_name}") do
-        api.delete_config_var(project_name, key)
+    if options[:user]
+      args.each do |key|
+        action("Unsetting #{key} for your user") do
+          api.delete_user_config_var(key)
+        end
+      end
+    else
+      project_name = options[:project] || project.name
+      args.each do |key|
+        action("Unsetting #{key} for project #{project_name}") do
+          api.delete_config_var(project_name, key)
+        end
       end
     end
   end

data/lib/mortar/command/jobs.rb

@@ -114,7 +114,7 @@ class Mortar::Command::Jobs < Mortar::Command::Base
     end
     
     unless options[:clusterid] || options[:clustersize]
-      clusters = api.get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_1).body['clusters']
+      clusters = api.get_clusters(pig_version.cluster_backend).body['clusters']
 
       largest_free_cluster = clusters.select{ |c| \
         c['running_jobs'].length == 0 && c['status_code'] == Mortar::API::Clusters::STATUS_RUNNING }.

data/lib/mortar/command/local.rb

@@ -41,7 +41,7 @@ class Mortar::Command::Local < Mortar::Command::Base
     Dir.chdir(project_root)
 
     ctrl = Mortar::Local::Controller.new
-    ctrl.install_and_configure(pig_version, nil, true)
+    ctrl.install_and_configure(pig_version, nil)
   end
 
   # local:run SCRIPT
@@ -277,139 +277,4 @@ class Mortar::Command::Local < Mortar::Command::Base
     ctrl.run_luigi(pig_version, script, cli_parameters, project_config_params)
   end
 
-  # local:sqoop_table dbtype database-name table s3-destination
-  #
-  # Export data from an RDBMS table to S3.
-  #
-  # -h, --host HOSTNAME # Database host, localhost assumed if not specified
-  # -u, --username USERNAME # User to log into the database with
-  # -p, --password PASSWORD # Password to log into the database
-  # -j, --jdbcdriver COM.DRIVER.BAR # Name of the JDBC driver class
-  # -d, --direct # Use a direct import path
-  # -r, --driverjar JARFILE # Path to the jar containing the jdbc driver
-  # -c, --jdbcconnectionstring CONNECTION_STRING # The JDBC connection string override
-  #
-  #Examples:
-  #
-  #    Export from a postgres database
-  #        $ mortar local:sqoop_table postgres mydb mytable s3://com.dbhost01/archive -u steve -p stevespassword
-  def sqoop_table
-    dbtype = shift_argument
-    unless dbtype
-      error("Usage: mortar local:sqoop_table dbtype database-name table s3-destination\nMust specify database type.")
-    end
-    physdb = shift_argument
-    unless physdb
-      error("Usage: mortar local:sqoop_table dbtype database-name table s3-destination\nMust specify database name.")
-    end
-    dbtable = shift_argument
-    unless dbtable
-      error("Usage: mortar local:sqoop_table dbtype database-name table s3-destination\nMust specify database table.")
-    end
-    s3dest = shift_argument
-    unless s3dest
-      error("Usage: mortar local:sqoop_table dbtype database-name table s3-destination\nMust specify s3 destination.")
-    end
-    validate_arguments!
-
-    dbhost = options[:host] || "localhost"
-    connstr = Shellwords.escape(options[:jdbcconnectionstring] || jdbc_conn(dbtype, dbhost, physdb))
-
-    ctrl = Mortar::Local::Controller.new
-    ctrl.sqoop_export_table(pig_version, connstr, dbtable, s3dest, options)
-  end
-
-  # local:sqoop_query dbtype database-name query s3-destination
-  #
-  # Export the result of an SQL query to S3.
-  #
-  # -h, --host HOSTNAME # Database host, localhost assumed if not specified
-  # -u, --username USERNAME # User to log into the database with
-  # -p, --password PASSWORD # Password to log into the database
-  # -j, --jdbcdriver COM.DRIVER.BAR # Name of the JDBC driver class
-  # -d, --direct # Use a direct import path
-  # -r, --driverjar JARFILE # Path to the jar containing the jdbc driver
-  # -c, --jdbcconnectionstring CONNECTION_STRING # The JDBC connection string override
-  #
-  #Examples:
-  #
-  #    Export from a postgres database
-  #        $ mortar local:sqoop_query postgres mydb "select user_name, id from users" s3://com.dbhost01/archive
-  def sqoop_query
-    dbtype = shift_argument
-    unless dbtype
-      error("Usage: mortar local:sqoop_query dbtype database-name query s3-destination\nMust specify database type.")
-    end
-    physdb = shift_argument
-    unless physdb
-      error("Usage: mortar local:sqoop_query dbtype database-name query s3-destination\nMust specify database name.")
-    end
-    query = shift_argument
-    unless query
-      error("Usage: mortar local:sqoop_query dbtype database-name query s3-destination\nMust specify sql query.")
-    end
-    s3dest = shift_argument
-    unless s3dest
-      error("Usage: mortar local:sqoop_query dbtype database-name query s3-destination\nMust specify s3 destination.")
-    end
-    validate_arguments!
-
-    dbhost = options[:host] || "localhost"
-    connstr = Shellwords.escape(options[:jdbcconnectionstring] || jdbc_conn(dbtype, dbhost, physdb))
-
-    ctrl = Mortar::Local::Controller.new
-    ctrl.sqoop_export_query(pig_version, connstr, query, s3dest, options)
-  end
-
-  # local:sqoop_incremental dbtype database-name table column value s3-destination
-  #
-  # Export all records where column is > value
-  #
-  # -h, --host HOSTNAME # Database host, localhost assumed if not specified
-  # -u, --username USERNAME # User to log into the database with
-  # -p, --password PASSWORD # Password to log into the database
-  # -j, --jdbcdriver COM.DRIVER.BAR # Name of the JDBC driver class
-  # -d, --direct # Use a direct import path
-  # -r, --driverjar JARFILE # Path to the jar containing the jdbc driver
-  # -c, --jdbcconnectionstring CONNECTION_STRING # The JDBC connection string override
-  #
-  #Examples:
-  #
-  #    Export from the newest users
-  #        $ mortar local:sqoop_incremental postgres mydb users user_id 12345 s3://com.dbhost01/archive
-  def sqoop_incremental
-    dbtype = shift_argument
-    unless dbtype
-      error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify database type.")
-    end
-    physdb = shift_argument
-    unless physdb
-      error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify database name.")
-    end
-    table = shift_argument
-    unless table
-      error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify database table.")
-    end
-    column = shift_argument
-    unless column
-      error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify column.")
-    end
-    max_value = shift_argument
-    unless max_value
-      error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify value.")
-    end
-    s3dest = shift_argument
-    unless s3dest
-      error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify s3 destination.")
-    end
-    validate_arguments!
-
-    dbhost = options[:host] || "localhost"
-    connstr = Shellwords.escape(options[:jdbcconnectionstring] || jdbc_conn(dbtype, dbhost, physdb))
-
-    ctrl = Mortar::Local::Controller.new
-    ctrl.sqoop_export_incremental(pig_version, connstr, table, column, max_value, s3dest, options)
-  end
-
-
 end

data/lib/mortar/command/spark.rb

@@ -17,56 +17,44 @@
 require "mortar/command/base"
 require "time"
 
-# run spark jobs
+# run spark jobs using Spark Job Server
 #
 class Mortar::Command::Spark < Mortar::Command::Base
 
   include Mortar::Git
 
-  # spark SCRIPT
+  # spark CLASS_NAME
   #
-  # Run a spark job.
+  # Run a spark job on a spark jobserver.
   #
-  # -c, --clusterid CLUSTERID   # Run job on an existing cluster with ID of CLUSTERID (optional)
-  # -s, --clustersize NUMNODES  # Run job on a new cluster, with NUMNODES nodes (optional; must be >= 2 if provided)
-  # -1, --singlejobcluster      # Stop the cluster after job completes.  (Default: false--cluster can be used for other jobs, and will shut down after 1 hour of inactivity)
-  # -2, --permanentcluster      # Don't automatically stop the cluster after it has been idle for an hour (Default: false--cluster will be shut down after 1 hour of inactivity)
-  # -3, --spot                  # Use spot instances for this cluster (Default: false, only applicable to new clusters)
+  # -c, --clusterid CLUSTERID   # Run job on an existing cluster with ID of CLUSTERID (Default: runs on an existing available cluster)
+  # -s, --clustersize NUMNODES  # Run job with NUMNODES nodes (optional; must be >= 2 if provided)
+  # -3, --spot                  # Use spot instances for this cluster (Default: true)
   # -P, --project PROJECTNAME   # Use a project that is not checked out in the current directory.  Runs code from project's master branch in GitHub rather than snapshotting local code.
   # -B, --branch BRANCHNAME     # Used with --project to specify a non-master branch
   #
   # Examples:
   #
-  #    Run the classify_text sparkscript:
-  #        $ mortar spark sparkscripts/classify_text.py
+  #    Run a spark job:
+  #        $ mortar spark com.datadog.some.Job
   #
-  #    Run the classify_text sparkscript with 3 script arguments (input location, output location, tuning parameter):
-  #        $ mortar spark sparkscripts/classify_text.py s3://your-bucket/input s3://your-bucket/output 100
+  #    Run a spark job with some arguments:
+  #        $ mortar spark com.datadog.some.Job --env prod s3://your-bucket/input s3://your-bucket/output 100
   #
   def index
-    script_name = shift_argument
-    unless script_name
-      error("Usage: mortar spark SCRIPT\nMust specify SCRIPT.")
+    class_name = shift_argument
+    unless class_name
+      error("Usage: mortar spark CLASS_NAME\nMust specify CLASS_NAME.")
     end
-    
+
     if options[:project]
       project_name = options[:project]
     else
       project_name = project.name
-      script = validate_sparkscript!(script_name)
-      script_name = script.name
     end
 
     script_arguments = spark_script_arguments()
 
-    if options[:clusterid]
-      [:clustersize, :singlejobcluster, :permanentcluster].each do |opt|
-        unless options[opt].nil?
-          error("Option #{opt.to_s} cannot be set when running a job on an existing cluster (with --clusterid option)")
-        end
-      end
-    end
-
     if options[:project]
       if options[:branch]
         git_ref = options[:branch]
@@ -78,46 +66,27 @@ class Mortar::Command::Spark < Mortar::Command::Base
     end
 
     unless options[:clusterid] || options[:clustersize]
-      clusters = api.get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_HADOOP_2).body['clusters']
+      clusters = api.get_clusters(Mortar::API::Jobs::CLUSTER_BACKEND__EMR_SPARK_JOBSERVER).body['clusters']
 
       largest_free_cluster = clusters.select{ |c| \
         c['running_jobs'].length == 0 && c['status_code'] == Mortar::API::Clusters::STATUS_RUNNING }.
         max_by{|c| c['size']}
-
+      
       if largest_free_cluster.nil?
-        options[:clustersize] = 2
-        display("Defaulting to running job on new cluster of size 2")
-      else
-        options[:clusterid] = largest_free_cluster['cluster_id']
-        display("Defaulting to running job on largest existing free cluster, id = " + 
-                largest_free_cluster['cluster_id'] + ", size = " + largest_free_cluster['size'].to_s)
+        error('No running clusters with Spark Job Server detected, please, launch a SparkJobServer cluster first')
       end
+
+      options[:clusterid] = largest_free_cluster['cluster_id']
+      display("Defaulting to running job on largest existing free cluster, id = " +
+              largest_free_cluster['cluster_id'] + ", size = " + largest_free_cluster['size'].to_s)
     end
 
     response = action("Requesting job execution") do
-      if options[:clustersize]
-        if options[:singlejobcluster] && options[:permanentcluster]
-          error("Cannot declare cluster as both --singlejobcluster and --permanentcluster")
-        end
-        cluster_size = options[:clustersize].to_i
-        cluster_type = Mortar::API::Jobs::CLUSTER_TYPE__PERSISTENT
-        if options[:singlejobcluster]
-          cluster_type = Mortar::API::Jobs::CLUSTER_TYPE__SINGLE_JOB
-        elsif options[:permanentcluster]
-          cluster_type = Mortar::API::Jobs::CLUSTER_TYPE__PERMANENT
-        end
-        use_spot_instances = options[:spot] || false
-        api.post_spark_job_new_cluster(project_name, script_name, git_ref, cluster_size, 
-          :project_script_path => script.rel_path,
-          :script_arguments => script_arguments,
-          :cluster_type => cluster_type,
-          :use_spot_instances => use_spot_instances).body
-      else
-        cluster_id = options[:clusterid]
-        api.post_spark_job_existing_cluster(project_name, script_name, git_ref, cluster_id,
-          :project_script_path => script.rel_path,
-          :script_arguments => script_arguments).body
-      end
+      cluster_id = options[:clusterid]
+      api.post_spark_job_on_jobserver(project_name, class_name, git_ref, cluster_id,
+        :script_arguments => script_arguments,
+        :clustersize => options[:clustersize]
+      ).body
     end
     
     display("job_id: #{response['job_id']}")

data/lib/mortar/local/controller.rb

@@ -19,7 +19,6 @@ require "mortar/local/pig"
 require "mortar/local/java"
 require "mortar/local/python"
 require "mortar/local/jython"
-require "mortar/local/sqoop"
 
 
 class Mortar::Local::Controller
@@ -118,14 +117,14 @@ EOF
   end
   # Main entry point to perform installation and configuration necessary
   # to run pig on the users local machine
-  def install_and_configure(pig_version=nil, command=nil, install_sqoop=false)
+  def install_and_configure(pig_version=nil, command=nil)
     #To support old watchtower plugins we'll accept nil pig_version
     base = Mortar::Command::Base.new
     set_project_name(base)
     require_aws_keys()
 
     if pig_version.nil?
-      pig_version = Mortar::PigVersion::Pig09.new
+      pig_version = Mortar::PigVersion::Pig012.new
     end
 
     java = Mortar::Local::Java.new()
@@ -154,11 +153,6 @@ EOF
     jy = Mortar::Local::Jython.new()
     jy.install_or_update()
 
-    if install_sqoop
-      sqoop = Mortar::Local::Sqoop.new()
-      sqoop.install_or_update()
-    end
-
     write_local_readme
 
     ensure_local_install_dirs_in_gitignore
@@ -245,27 +239,4 @@ README
     py.run_luigi_script(luigi_script, luigi_script_parameters)
   end
 
-  def sqoop_export_table(pig_version, connstr, dbtable, s3dest, options)
-    install_and_configure(pig_version, 'sqoop', true)
-    sqoop = Mortar::Local::Sqoop.new()
-    options[:dbtable] = dbtable
-    sqoop.export(connstr, s3dest, options)
-  end
-
-  def sqoop_export_query(pig_version, connstr, query, s3dest, options)
-    install_and_configure(pig_version, 'sqoop', true)
-    sqoop = Mortar::Local::Sqoop.new()
-    options[:sqlquery] = sqoop.prep_query(query)
-    sqoop.export(connstr, s3dest, options)
-  end
-
-  def sqoop_export_incremental(pig_version, connstr, dbtable, column, max_value, s3dest, options)
-    install_and_configure(pig_version, 'sqoop', true)
-    sqoop = Mortar::Local::Sqoop.new()
-    options[:dbtable] = dbtable
-    options[:inc_column] = column
-    options[:inc_value] = max_value
-    sqoop.export(connstr, s3dest, options)
-  end
-
 end