mortar 0.15.31 → 0.15.32

data/lib/mortar/command/local.rb

@@ -17,6 +17,7 @@
 require "mortar/local/controller"
 require "mortar/command/base"
 require "mortar/generators/characterize_generator"
+require "shellwords"
 
 # run select pig commands on your local machine
 #
@@ -78,10 +79,10 @@ class Mortar::Command::Local < Mortar::Command::Base
 
   # local:characterize -f PARAMFILE
   #
-  # Characterize will inspect your input data, inferring a schema and 
+  # Characterize will inspect your input data, inferring a schema and
   #    generating keys, if needed. It will output CSV containing various
   #    statistics about your data (most common values, percent null, etc.)
-  # 
+  #
   # -f, --param-file PARAMFILE # Load pig parameter values from a file
   # -g, --pigversion PIG_VERSION  # Set pig version.  Options are <PIG_VERSION_OPTIONS>.
   #
@@ -217,7 +218,7 @@ class Mortar::Command::Local < Mortar::Command::Base
     validate_arguments!
 
     params = config_parameters.concat(pig_parameters)
-    
+
     ctrl = Mortar::Local::Controller.new
     ctrl.repl(pig_version, params)
   end
@@ -241,7 +242,7 @@ class Mortar::Command::Local < Mortar::Command::Base
     unless script_name
       error("Usage: mortar local:luigi SCRIPT\nMust specify SCRIPT.")
     end
-    
+
     # cd into the project root
     project_root = options[:project_root] ||= Dir.getwd
     unless File.directory?(project_root)
@@ -250,7 +251,7 @@ class Mortar::Command::Local < Mortar::Command::Base
     Dir.chdir(project_root)
 
     script = validate_luigiscript!(script_name)
-    
+
     #Set git ref as environment variable for mortar-luigi to use when
     #running a MortarTask
     git_ref = sync_code_with_cloud()
@@ -286,6 +287,7 @@ class Mortar::Command::Local < Mortar::Command::Base
   # -j, --jdbcdriver COM.DRIVER.BAR # Name of the JDBC driver class
   # -d, --direct # Use a direct import path
   # -r, --driverjar JARFILE # Path to the jar containing the jdbc driver
+  # -c, --jdbcconnectionstring CONNECTION_STRING # The JDBC connection string override
   #
   #Examples:
   #
@@ -311,7 +313,7 @@ class Mortar::Command::Local < Mortar::Command::Base
     validate_arguments!
 
     dbhost = options[:host] || "localhost"
-    connstr = jdbc_conn(dbtype, dbhost, physdb)
+    connstr = Shellwords.escape(options[:jdbcconnectionstring] || jdbc_conn(dbtype, dbhost, physdb))
 
     ctrl = Mortar::Local::Controller.new
     ctrl.sqoop_export_table(pig_version, connstr, dbtable, s3dest, options)
@@ -327,6 +329,7 @@ class Mortar::Command::Local < Mortar::Command::Base
   # -j, --jdbcdriver COM.DRIVER.BAR # Name of the JDBC driver class
   # -d, --direct # Use a direct import path
   # -r, --driverjar JARFILE # Path to the jar containing the jdbc driver
+  # -c, --jdbcconnectionstring CONNECTION_STRING # The JDBC connection string override
   #
   #Examples:
   #
@@ -352,7 +355,7 @@ class Mortar::Command::Local < Mortar::Command::Base
     validate_arguments!
 
     dbhost = options[:host] || "localhost"
-    connstr = jdbc_conn(dbtype, dbhost, physdb)
+    connstr = Shellwords.escape(options[:jdbcconnectionstring] || jdbc_conn(dbtype, dbhost, physdb))
 
     ctrl = Mortar::Local::Controller.new
     ctrl.sqoop_export_query(pig_version, connstr, query, s3dest, options)
@@ -368,6 +371,7 @@ class Mortar::Command::Local < Mortar::Command::Base
   # -j, --jdbcdriver COM.DRIVER.BAR # Name of the JDBC driver class
   # -d, --direct # Use a direct import path
   # -r, --driverjar JARFILE # Path to the jar containing the jdbc driver
+  # -c, --jdbcconnectionstring CONNECTION_STRING # The JDBC connection string override
   #
   #Examples:
   #
@@ -401,7 +405,7 @@ class Mortar::Command::Local < Mortar::Command::Base
     validate_arguments!
 
     dbhost = options[:host] || "localhost"
-    connstr = jdbc_conn(dbtype, dbhost, physdb)
+    connstr = Shellwords.escape(options[:jdbcconnectionstring] || jdbc_conn(dbtype, dbhost, physdb))
 
     ctrl = Mortar::Local::Controller.new
     ctrl.sqoop_export_incremental(pig_version, connstr, table, column, max_value, s3dest, options)

data/lib/mortar/templates/project/luigiscripts/luigiscript.py

@@ -26,8 +26,6 @@ mortar luigi luigiscripts/<%= project_name %>_luigi.py \
     --output-base-path "s3://mortar-example-output-data/<your_username_here>/<%= project_name %>"
 """
 
-MORTAR_PROJECT = '<%= project_name %>'
-
 """
 This logger outputs logs to Mortar Logs. An example of it's usage can be seen
 in the ShutdownClusters function.
@@ -71,12 +69,6 @@ class RunMyExamplePigScript(mortartask.MortarProjectPigscriptTask):
         """
         return [S3PathTask(self.input_path)]
 
-    def project(self):
-        """
-        Name of Mortar Project to run.
-        """
-        return MORTAR_PROJECT
-
     def script_output(self):
         """
         The script_output method is how you define where the output from this task

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.15.31"
+  VERSION = "0.15.32"
 end

data/spec/mortar/command/local_spec.rb

@@ -19,6 +19,7 @@ require 'fakefs/spec_helpers'
 require 'mortar/command/local'
 require 'launchy'
 require 'fileutils'
+require 'shellwords'
 
 module Mortar::Command
   describe Local do
@@ -155,7 +156,7 @@ PARAMS
         File.exists?("Test/macros/characterize_macro.pig").should be_false
         File.exists?("Test/udfs/jython/top_5_tuple.py").should be_false
         File.exists?("Test/controlscripts/lib/characterize_control.py").should be_false
-        File.delete("test.params")           
+        File.delete("test.params")
       end
 
     end
@@ -397,7 +398,7 @@ STDERR
       end
 
       it "sends everything to the controller" do
-        connstr = "jdbc:mysql://foobar.com/mydb?zeroDateTimeBehavior=convertToNull"
+        connstr = Shellwords.escape("jdbc:mysql://foobar.com/mydb?zeroDateTimeBehavior=convertToNull")
         dbtable = "customers"
         s3dest = "s3n://a-bucket/a-directory"
         any_instance_of(Mortar::Local::Controller) do |c|
@@ -407,7 +408,7 @@ STDERR
       end
 
       it "defaults to 'localhost' if no host specified" do
-        connstr = "jdbc:mysql://localhost/mydb?zeroDateTimeBehavior=convertToNull"
+        connstr = Shellwords.escape("jdbc:mysql://localhost/mydb?zeroDateTimeBehavior=convertToNull")
         dbtable = "customers"
         s3dest = "s3n://a-bucket/a-directory"
         any_instance_of(Mortar::Local::Controller) do |c|
@@ -416,6 +417,16 @@ STDERR
         stderr, stdout = execute "local:sqoop_table mysql mydb #{dbtable} #{s3dest}"
       end
 
+      it "overrides the JDBC connection string if one is specified" do
+        connstr = Shellwords.escape("jdbc:mysql://localhost/mydbother?zeroDateTimeBehavior=convertToNull")
+        dbtable = "customers"
+        s3dest = "s3n://a-bucket/a-directory"
+        any_instance_of(Mortar::Local::Controller) do |c|
+          mock(c).sqoop_export_table(is_a(Mortar::PigVersion::Pig09), connstr, dbtable, s3dest, {})
+        end
+        stderr, stdout = execute "local:sqoop_table mysql mydb #{dbtable} #{s3dest} -c jdbc:mysql://localhost/mydbother?zeroDateTimeBehavior=convertToNull"
+      end
+
     end
 
     context "local:sqoop_query" do
@@ -452,7 +463,7 @@ STDERR
       end
 
       it "sends everything to the controller" do
-        connstr = "jdbc:mysql://foobar.com/mydb?zeroDateTimeBehavior=convertToNull"
+        connstr = Shellwords.escape("jdbc:mysql://foobar.com/mydb?zeroDateTimeBehavior=convertToNull")
         query = "select_*_from_customers"
         s3dest = "s3n://a-bucket/a-directory"
         any_instance_of(Mortar::Local::Controller) do |c|
@@ -463,7 +474,7 @@ STDERR
       end
 
       it "defaults to 'localhost' if no host specified" do
-        connstr = "jdbc:mysql://localhost/mydb?zeroDateTimeBehavior=convertToNull"
+        connstr = Shellwords.escape("jdbc:mysql://localhost/mydb?zeroDateTimeBehavior=convertToNull")
         query = "select_*_from_customers"
         s3dest = "s3n://a-bucket/a-directory"
         any_instance_of(Mortar::Local::Controller) do |c|
@@ -473,6 +484,17 @@ STDERR
         stderr.should == ''
       end
 
+      it "overrides the JDBC connection string if one is specified" do
+        connstr = Shellwords.escape("jdbc:mysql://localhost/mydbother?zeroDateTimeBehavior=convertToNull")
+        query = "select_*_from_customers"
+        s3dest = "s3n://a-bucket/a-directory"
+        any_instance_of(Mortar::Local::Controller) do |c|
+          mock(c).sqoop_export_query(is_a(Mortar::PigVersion::Pig09), connstr, query, s3dest, {})
+        end
+        stderr, stdout = execute "local:sqoop_query mysql mydb #{query} #{s3dest} -c jdbc:mysql://localhost/mydbother?zeroDateTimeBehavior=convertToNull"
+        stderr.should == ''
+      end
+
     end
 
     context "local:sqoop_incremental" do
@@ -525,7 +547,7 @@ STDERR
       end
 
       it "sends everything to the controller" do
-        connstr = "jdbc:mysql://foobar.com/mydb?zeroDateTimeBehavior=convertToNull"
+        connstr = Shellwords.escape("jdbc:mysql://foobar.com/mydb?zeroDateTimeBehavior=convertToNull")
         dbtable = "customers"
         column = "customer_id"
         column_value = "12345"
@@ -537,7 +559,7 @@ STDERR
       end
 
       it "defaults to 'localhost' if no host specified" do
-        connstr = "jdbc:mysql://localhost/mydb?zeroDateTimeBehavior=convertToNull"
+        connstr = Shellwords.escape("jdbc:mysql://localhost/mydb?zeroDateTimeBehavior=convertToNull")
         dbtable = "customers"
         column = "customer_id"
         column_value = "12345"
@@ -548,10 +570,21 @@ STDERR
         stderr, stdout = execute "local:sqoop_incremental mysql mydb #{dbtable} #{column} #{column_value} #{s3dest}"
       end
 
+      it "overrides the JDBC connection string if one is specified" do
+        connstr = Shellwords.escape("jdbc:mysql://localhost/mydbother?zeroDateTimeBehavior=convertToNull")
+        dbtable = "customers"
+        column = "customer_id"
+        column_value = "12345"
+        s3dest = "s3n://a-bucket/a-directory"
+        any_instance_of(Mortar::Local::Controller) do |c|
+          mock(c).sqoop_export_incremental(is_a(Mortar::PigVersion::Pig09), connstr, dbtable, column, column_value, s3dest, {})
+        end
+        stderr, stdout = execute "local:sqoop_incremental mysql mydb #{dbtable} #{column} #{column_value} #{s3dest} -c jdbc:mysql://localhost/mydbother?zeroDateTimeBehavior=convertToNull"
+      end
+
     end
 
 
 
   end
 end
-

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mortar
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 99
   prerelease: 
   segments: 
   - 0
   - 15
-  - 31
-  version: 0.15.31
+  - 32
+  version: 0.15.32
 platform: ruby
 authors: 
 - Mortar Data
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2014-10-24 00:00:00 Z
+date: 2014-11-11 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rdoc