mortar 0.15.7 → 0.15.11

data/lib/mortar/command/base.rb

@@ -556,6 +556,10 @@ protected
     return Mortar::Auth.user_s3_safe + "-base"
   end
 
+  def jdbc_conn(dbtype, host, dbname)
+    "jdbc:#{dbtype}://#{host}/#{dbname}?zeroDateTimeBehavior=convertToNull"
+  end
+
 end
 
 module Mortar::Command

data/lib/mortar/command/local.rb

@@ -22,7 +22,6 @@ require "mortar/generators/characterize_generator"
 #
 class Mortar::Command::Local < Mortar::Command::Base
 
-
   # local:configure
   #
   # Install dependencies for running this mortar project locally - other mortar:local commands will also perform this step automatically.
@@ -257,5 +256,136 @@ class Mortar::Command::Local < Mortar::Command::Base
     ctrl.run_luigi(script, luigi_params)
   end
 
+  # local:sqoop_table dbtype database-name table s3-destination
+  #
+  # Export data from an RDBMS table to S3.
+  #
+  # -h, --host HOSTNAME # Database host, localhost assumed if not specified
+  # -u, --username USERNAME # User to log into the database with
+  # -p, --password PASSWORD # Password to log into the database
+  # -j, --jdbcdriver COM.DRIVER.BAR # Name of the JDBC driver class
+  # -d, --direct # Use a direct import path
+  # -r, --driverjar JARFILE # Path to the jar containing the jdbc driver
+  #
+  #Examples:
+  #
+  #    Export from a postgres database
+  #        $ mortar local:sqoop_table postgres mydb mytable s3://com.dbhost01/archive -u steve -p stevespassword
+  def sqoop_table
+    dbtype = shift_argument
+    unless dbtype
+      error("Usage: mortar local:sqoop_table dbtype database-name table s3-destination\nMust specify database type.")
+    end
+    physdb = shift_argument
+    unless physdb
+      error("Usage: mortar local:sqoop_table dbtype database-name table s3-destination\nMust specify database name.")
+    end
+    dbtable = shift_argument
+    unless dbtable
+      error("Usage: mortar local:sqoop_table dbtype database-name table s3-destination\nMust specify database table.")
+    end
+    s3dest = shift_argument
+    unless s3dest
+      error("Usage: mortar local:sqoop_table dbtype database-name table s3-destination\nMust specify s3 destination.")
+    end
+    validate_arguments!
+
+    dbhost = options[:host] || "localhost"
+    connstr = jdbc_conn(dbtype, dbhost, physdb)
+
+    ctrl = Mortar::Local::Controller.new
+    ctrl.sqoop_export_table(connstr, dbtable, s3dest, options)
+  end
+
+  # local:sqoop_query dbtype database-name query s3-destination
+  #
+  # Export the result of an SQL query to S3.
+  #
+  # -h, --host HOSTNAME # Database host, localhost assumed if not specified
+  # -u, --username USERNAME # User to log into the database with
+  # -p, --password PASSWORD # Password to log into the database
+  # -j, --jdbcdriver COM.DRIVER.BAR # Name of the JDBC driver class
+  # -d, --direct # Use a direct import path
+  # -r, --driverjar JARFILE # Path to the jar containing the jdbc driver
+  #
+  #Examples:
+  #
+  #    Export from a postgres database
+  #        $ mortar local:sqoop_query postgres mydb "select user_name, id from users" s3://com.dbhost01/archive
+  def sqoop_query
+    dbtype = shift_argument
+    unless dbtype
+      error("Usage: mortar local:sqoop_query dbtype database-name query s3-destination\nMust specify database type.")
+    end
+    physdb = shift_argument
+    unless physdb
+      error("Usage: mortar local:sqoop_query dbtype database-name query s3-destination\nMust specify database name.")
+    end
+    query = shift_argument
+    unless query
+      error("Usage: mortar local:sqoop_query dbtype database-name query s3-destination\nMust specify sql query.")
+    end
+    s3dest = shift_argument
+    unless s3dest
+      error("Usage: mortar local:sqoop_query dbtype database-name query s3-destination\nMust specify s3 destination.")
+    end
+    validate_arguments!
+
+    dbhost = options[:host] || "localhost"
+    connstr = jdbc_conn(dbtype, dbhost, physdb)
+
+    ctrl = Mortar::Local::Controller.new
+    ctrl.sqoop_export_query(connstr, query, s3dest, options)
+  end
+
+  # local:sqoop_incremental dbtype database-name table column value s3-destination
+  #
+  # Export all records where column is > value
+  #
+  # -h, --host HOSTNAME # Database host, localhost assumed if not specified
+  # -u, --username USERNAME # User to log into the database with
+  # -p, --password PASSWORD # Password to log into the database
+  # -j, --jdbcdriver COM.DRIVER.BAR # Name of the JDBC driver class
+  # -d, --direct # Use a direct import path
+  # -r, --driverjar JARFILE # Path to the jar containing the jdbc driver
+  #
+  #Examples:
+  #
+  #    Export from the newest users
+  #        $ mortar local:sqoop_incremental postgres mydb users user_id 12345 s3://com.dbhost01/archive
+  def sqoop_incremental
+    dbtype = shift_argument
+    unless dbtype
+      error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify database type.")
+    end
+    physdb = shift_argument
+    unless physdb
+      error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify database name.")
+    end
+    table = shift_argument
+    unless table
+      error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify database table.")
+    end
+    column = shift_argument
+    unless column
+      error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify column.")
+    end
+    max_value = shift_argument
+    unless max_value
+      error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify value.")
+    end
+    s3dest = shift_argument
+    unless s3dest
+      error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify s3 destination.")
+    end
+    validate_arguments!
+
+    dbhost = options[:host] || "localhost"
+    connstr = jdbc_conn(dbtype, dbhost, physdb)
+
+    ctrl = Mortar::Local::Controller.new
+    ctrl.sqoop_export_incremental(connstr, table, column, max_value, s3dest, options)
+  end
+
 
 end

data/lib/mortar/local/controller.rb

@@ -19,6 +19,7 @@ require "mortar/local/pig"
 require "mortar/local/java"
 require "mortar/local/python"
 require "mortar/local/jython"
+require "mortar/local/sqoop"
 
 
 class Mortar::Local::Controller
@@ -144,6 +145,10 @@ EOF
     jy = Mortar::Local::Jython.new()
     jy.install_or_update()
 
+
+    sqoop = Mortar::Local::Sqoop.new()
+    sqoop.install_or_update()
+
     ensure_local_install_dirs_in_gitignore
   end
 
@@ -206,4 +211,27 @@ EOF
     py.run_luigi_script(luigi_script, user_script_args)
   end
 
+  def sqoop_export_table(connstr, dbtable, s3dest, options)
+    install_and_configure(nil, 'sqoop')
+    sqoop = Mortar::Local::Sqoop.new()
+    options[:dbtable] = dbtable
+    sqoop.export(connstr, s3dest, options)
+  end
+
+  def sqoop_export_query(connstr, query, s3dest, options)
+    install_and_configure(nil, 'sqoop')
+    sqoop = Mortar::Local::Sqoop.new()
+    options[:sqlquery] = sqoop.prep_query(query)
+    sqoop.export(connstr, s3dest, options)
+  end
+
+  def sqoop_export_incremental(connstr, dbtable, column, max_value, s3dest, options)
+    install_and_configure(nil, 'sqoop')
+    sqoop = Mortar::Local::Sqoop.new()
+    options[:dbtable] = dbtable
+    options[:inc_column] = column
+    options[:inc_value] = max_value
+    sqoop.export(connstr, s3dest, options)
+  end
+
 end

data/lib/mortar/local/sqoop.rb

@@ -0,0 +1,159 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "erb"
+require 'tempfile'
+require "mortar/helpers"
+require "mortar/local/installutil"
+
+class Mortar::Local::Sqoop
+  include Mortar::Local::InstallUtil
+
+  def install_or_update
+    @command = "#{local_install_directory}/python/bin/python"
+    if should_do_install?
+      action "Installing sqoop to #{local_install_directory_name}" do
+        do_install
+      end
+    elsif should_do_update?
+      action "Updating to latest sqoop in #{local_install_directory_name}" do
+        do_install
+      end
+    end
+    true
+  end
+
+  def sqoop_url
+    full_host  = (host =~ /^http/) ? host : "https://api.#{host}"
+    default_url = full_host + "/" + "resource/sqoop"
+    return ENV.fetch('SQOOP_DISTRO_URL', default_url)
+  end
+
+  def should_do_install?
+    return (not (File.exists?(sqoop_directory)))
+  end
+
+  def should_do_update?
+    return is_newer_version('sqoop', sqoop_url)
+  end
+
+  def sqoop_directory
+    return "#{local_install_directory}/sqoop"
+  end
+
+  def sqoop_dir_in_tgz
+    "sqoop-1.4.4-mortar"
+  end
+
+  def do_install
+    local_tgz = File.join(local_install_directory, "sqoop-1.4.4-mortar.tar.gz")
+    if File.exists?(local_tgz)
+      FileUtils.rm(local_tgz)
+    end
+    download_file(sqoop_url, local_tgz)
+
+    if File.exists?(sqoop_directory)
+      FileUtils.rm_rf(sqoop_directory)
+    end
+
+    extract_tgz(local_tgz, local_install_directory)
+
+    FileUtils.mv(File.join(local_install_directory, sqoop_dir_in_tgz), sqoop_directory)
+
+    # This has been seening coming out of the tgz w/o +x so we do
+    # here to be sure it has the necessary permissions
+    FileUtils.chmod(0755, "#{sqoop_directory}/bin/sqoop")
+    FileUtils.chmod(0755, "#{sqoop_directory}/hadoop/bin/hadoop")
+
+    File.delete(local_tgz)
+    note_install("sqoop")
+  end
+
+  def sqoop_command_script_template_path
+    File.expand_path("../../templates/script/sqoop.sh", __FILE__)
+  end
+
+  def hadoop_home
+    "#{sqoop_directory}/hadoop"
+  end
+
+  def export(connstr, s3dest, options)
+    template_params = sqoop_export_template_parameters(connstr, s3dest, options)
+    return run_templated_script(sqoop_command_script_template_path, template_params)
+  end
+
+  def sqoop_export_template_parameters(connstr, s3dest, options)
+    pig = Mortar::Local::Pig.new()
+    parameters = {
+      "sqoop_dir" => sqoop_directory,
+      "jdb_conn_string" => connstr,
+      "destination" => s3dest,
+      "hadoop_home" => hadoop_home,
+      "classpath" => pig.template_params_classpath,
+      "jdbc_conn" => connstr,
+      "s3dest" => s3dest,
+      "project_root" => project_root,
+      "sqoop_opts" => sqoop_java_options
+    }
+    parameters["dbtable"] = options[:dbtable] if options[:dbtable]
+    parameters["sqlquery"] = options[:sqlquery] if options[:sqlquery]
+    parameters["inc_column"] = options[:inc_column] if options[:inc_column]
+    parameters["inc_value"] = options[:inc_value] if options[:inc_value]
+    if options[:inc_value] and 0 == options[:inc_value].to_i
+      parameters[:inc_mode] = "lastmodified"
+    elsif options[:inc_value]
+      parameters[:inc_mode] = "append"
+    end
+    parameters["dbuser"] = options[:username] if options[:username]
+    parameters["dbpass"] = options[:password] if options[:password]
+    parameters["jdbcdriver"] = options[:jdbcdriver] if options[:jdbcdriver]
+    parameters["driverjar"] = options[:driverjar] if options[:driverjar]
+    parameters["direct_import"] = true if options[:direct]
+    return parameters
+  end
+
+  def sqoop_java_options
+    opts = {}
+    opts['fs.s3n.awsAccessKeyId'] = ENV['AWS_ACCESS_KEY']
+    opts['fs.s3n.awsSecretAccessKey'] = ENV['AWS_SECRET_KEY']
+    opts['fs.s3.awsAccessKeyId'] = ENV['AWS_ACCESS_KEY']
+    opts['fs.s3.awsSecretAccessKey'] = ENV['AWS_SECRET_KEY']
+    return opts
+  end
+
+  # So this part kind of sucks.  In order to partition a query across multiple map
+  # reduce tasks sqoop does a query to to find the range of identifiying values,
+  # divides this range across the number of tasks to be executed and then modifies
+  # the query for each m/r task. To do this Sqoop needs to know at what point in the
+  # query that it should place its portion of the where clause. This is done via the
+  # $CONDITIONS marker. So that's well and good when you're running sqoop on a cluster
+  # but our users will be running on their own machine and don't know or care for this
+  # parrallel queries stuff.  So to make their lives easier we make a best effort to
+  # add the clause for them in a safe way.
+  def prep_query(original_query)
+    if original_query.include? "$CONDITIONS"
+      return original_query
+    elsif original_query.downcase.include? "where"
+      idxwhere = original_query.downcase.index("where")
+      select_where = original_query[0..idxwhere+"where".length-1]
+      clause = original_query[idxwhere+"where".length+1..original_query.length]
+      return "#{select_where} (#{clause}) AND \$CONDITIONS"
+    else
+      return "#{original_query} WHERE \$CONDITIONS"
+    end
+  end
+
+end

data/lib/mortar/templates/script/sqoop.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -e
+
+export HADOOP_CLASSPATH="<%= @project_root %>/lib/*"
+<% if @driverjar %>
+export HADOOP_CLASSPATH="$HADOOP_CLASSPATH:<%= @driverjar %>"
+<% end %>
+export HADOOP_COMMON_HOME="<%= @hadoop_home %>"
+export HADOOP_MAPRED_HOME="<%= @hadoop_home %>"
+export HADOOP_HOME_WARN_SUPPRESS="t"
+
+# Only setting these to get rid of warnings that sqoop is showing
+export HCAT_HOME="<%= @hadoop_home %>"
+export HBASE_HOME="<%= @hadoop_home %>"
+
+SQOOP_OPTS="<% @sqoop_opts.each do |k,v| %>-D<%= k %>=<%= v %> <% end %>"
+OPTARGS='<%= "--driver #{@jdbcdriver}" if @jdbcdriver %>'
+OPTARGS="$OPTARGS <%= "--username #{@dbuser}" if @dbuser %>"
+OPTARGS="$OPTARGS <%= "--password #{@dbpass}" if @dbpass %>"
+OPTARGS="$OPTARGS <%= "--direct" if @direct_import %>"
+<% if @inc_column and @inc_value %>
+OPTARGS="$OPTARGS --incremental <%= @inc_mode %>"
+OPTARGS="$OPTARGS --check-column <%= @inc_column %> "
+SQOOP_OPTS="$SQOOP_OPTS -Dsqoop.test.import.rootDir=<%= @s3dest %>/.tmp"
+<% end %>
+
+export HADOOP_OPTS="$SQOOP_OPTS";
+
+<%= @sqoop_dir %>/bin/sqoop \
+    import \
+    $SQOOP_OPTS \
+    <%= "--table #{@dbtable}" if @dbtable %> \
+    <%= "--query '#{@sqlquery}'" if @sqlquery %> \
+    -m 1 \
+    --connect <%= @jdbc_conn %> \
+    --target-dir <%= @s3dest %> \
+    $OPTARGS \
+    <% if @inc_column and @inc_value %>--last-value '<%= @inc_value %>'<% end %>
+

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.15.7"
+  VERSION = "0.15.11"
 end

data/spec/mortar/command/local_spec.rb

@@ -206,6 +206,9 @@ PARAMS
         any_instance_of(Mortar::Local::Jython) do |j|
           mock(j).install_or_update.returns(true)
         end
+        any_instance_of(Mortar::Local::Sqoop) do |j|
+          mock(j).install_or_update.returns(true)
+        end
         any_instance_of(Mortar::Local::Controller) do |j|
           mock(j).ensure_local_install_dirs_in_gitignore.returns(true)
         end
@@ -306,6 +309,194 @@ STDERR
 
     end
 
+    context "local:sqoop_table" do
+      it "requires a db type" do
+          stderr, stdout = execute "local:sqoop_table"
+          stderr.should == <<-STDERR
+ !    Usage: mortar local:sqoop_table dbtype database-name table s3-destination
+ !    Must specify database type.
+STDERR
+      end
+
+      it "requires the physical db name" do
+          stderr, stdout = execute "local:sqoop_table mysql"
+          stderr.should == <<-STDERR
+ !    Usage: mortar local:sqoop_table dbtype database-name table s3-destination
+ !    Must specify database name.
+STDERR
+      end
+
+      it "requires the table name" do
+          stderr, stdout = execute "local:sqoop_table mysql myappdb"
+          stderr.should == <<-STDERR
+ !    Usage: mortar local:sqoop_table dbtype database-name table s3-destination
+ !    Must specify database table.
+STDERR
+      end
+
+      it "requires the s3 destination" do
+          stderr, stdout = execute "local:sqoop_table mysql myappdb customers"
+          stderr.should == <<-STDERR
+ !    Usage: mortar local:sqoop_table dbtype database-name table s3-destination
+ !    Must specify s3 destination.
+STDERR
+      end
+
+      it "sends everything to the controller" do
+        connstr = "jdbc:mysql://foobar.com/mydb?zeroDateTimeBehavior=convertToNull"
+        dbtable = "customers"
+        s3dest = "s3n://a-bucket/a-directory"
+        any_instance_of(Mortar::Local::Controller) do |c|
+          mock(c).sqoop_export_table(connstr, dbtable, s3dest, {})
+        end
+        stderr, stdout = execute "local:sqoop_table mysql mydb #{dbtable} #{s3dest} --host foobar.com"
+      end
+
+      it "defaults to 'localhost' if no host specified" do
+        connstr = "jdbc:mysql://localhost/mydb?zeroDateTimeBehavior=convertToNull"
+        dbtable = "customers"
+        s3dest = "s3n://a-bucket/a-directory"
+        any_instance_of(Mortar::Local::Controller) do |c|
+          mock(c).sqoop_export_table(connstr, dbtable, s3dest, {})
+        end
+        stderr, stdout = execute "local:sqoop_table mysql mydb #{dbtable} #{s3dest}"
+      end
+
+    end
+
+    context "local:sqoop_query" do
+      it "requires a db type" do
+          stderr, stdout = execute "local:sqoop_query"
+          stderr.should == <<-STDERR
+ !    Usage: mortar local:sqoop_query dbtype database-name query s3-destination
+ !    Must specify database type.
+STDERR
+      end
+
+      it "requires the physical db name" do
+          stderr, stdout = execute "local:sqoop_query mysql"
+          stderr.should == <<-STDERR
+ !    Usage: mortar local:sqoop_query dbtype database-name query s3-destination
+ !    Must specify database name.
+STDERR
+      end
+
+      it "requires the table name" do
+          stderr, stdout = execute "local:sqoop_query mysql myappdb"
+          stderr.should == <<-STDERR
+ !    Usage: mortar local:sqoop_query dbtype database-name query s3-destination
+ !    Must specify sql query.
+STDERR
+      end
+
+      it "requires the s3 destination" do
+          stderr, stdout = execute "local:sqoop_query mysql myappdb customers"
+          stderr.should == <<-STDERR
+ !    Usage: mortar local:sqoop_query dbtype database-name query s3-destination
+ !    Must specify s3 destination.
+STDERR
+      end
+
+      it "sends everything to the controller" do
+        connstr = "jdbc:mysql://foobar.com/mydb?zeroDateTimeBehavior=convertToNull"
+        query = "select_*_from_customers"
+        s3dest = "s3n://a-bucket/a-directory"
+        any_instance_of(Mortar::Local::Controller) do |c|
+          mock(c).sqoop_export_query(connstr, query, s3dest, {})
+        end
+        stderr, stdout = execute "local:sqoop_query mysql mydb #{query} #{s3dest} --host foobar.com"
+        stderr.should == ''
+      end
+
+      it "defaults to 'localhost' if no host specified" do
+        connstr = "jdbc:mysql://localhost/mydb?zeroDateTimeBehavior=convertToNull"
+        query = "select_*_from_customers"
+        s3dest = "s3n://a-bucket/a-directory"
+        any_instance_of(Mortar::Local::Controller) do |c|
+          mock(c).sqoop_export_query(connstr, query, s3dest, {})
+        end
+        stderr, stdout = execute "local:sqoop_query mysql mydb #{query} #{s3dest}"
+        stderr.should == ''
+      end
+
+    end
+
+    context "local:sqoop_incremental" do
+      it "requires a db type" do
+          stderr, stdout = execute "local:sqoop_incremental"
+          stderr.should == <<-STDERR
+ !    Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
+ !    Must specify database type.
+STDERR
+      end
+
+      it "requires the physical db name" do
+          stderr, stdout = execute "local:sqoop_incremental mysql"
+          stderr.should == <<-STDERR
+ !    Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
+ !    Must specify database name.
+STDERR
+      end
+
+      it "requires the table name" do
+          stderr, stdout = execute "local:sqoop_incremental mysql myappdb"
+          stderr.should == <<-STDERR
+ !    Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
+ !    Must specify database table.
+STDERR
+      end
+
+      it "requires the column name" do
+          stderr, stdout = execute "local:sqoop_incremental mysql myappdb mytable"
+          stderr.should == <<-STDERR
+ !    Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
+ !    Must specify column.
+STDERR
+      end
+
+      it "requires the column value" do
+          stderr, stdout = execute "local:sqoop_incremental mysql myappdb mytable mycolumn"
+          stderr.should == <<-STDERR
+ !    Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
+ !    Must specify value.
+STDERR
+      end
+
+      it "requires the s3 destination" do
+          stderr, stdout = execute "local:sqoop_incremental mysql myappdb mytable mycolumn customers"
+          stderr.should == <<-STDERR
+ !    Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
+ !    Must specify s3 destination.
+STDERR
+      end
+
+      it "sends everything to the controller" do
+        connstr = "jdbc:mysql://foobar.com/mydb?zeroDateTimeBehavior=convertToNull"
+        dbtable = "customers"
+        column = "customer_id"
+        column_value = "12345"
+        s3dest = "s3n://a-bucket/a-directory"
+        any_instance_of(Mortar::Local::Controller) do |c|
+          mock(c).sqoop_export_incremental(connstr, dbtable, column, column_value, s3dest, {})
+        end
+        stderr, stdout = execute "local:sqoop_incremental mysql mydb #{dbtable} #{column} #{column_value} #{s3dest} --host foobar.com"
+      end
+
+      it "defaults to 'localhost' if no host specified" do
+        connstr = "jdbc:mysql://localhost/mydb?zeroDateTimeBehavior=convertToNull"
+        dbtable = "customers"
+        column = "customer_id"
+        column_value = "12345"
+        s3dest = "s3n://a-bucket/a-directory"
+        any_instance_of(Mortar::Local::Controller) do |c|
+          mock(c).sqoop_export_incremental(connstr, dbtable, column, column_value, s3dest, {})
+        end
+        stderr, stdout = execute "local:sqoop_incremental mysql mydb #{dbtable} #{column} #{column_value} #{s3dest}"
+      end
+
+    end
+
+
 
   end
 end

data/spec/mortar/local/controller_spec.rb

@@ -146,6 +146,10 @@ module Mortar::Local
         any_instance_of(Mortar::Local::Jython) do |j|
           mock(j).install_or_update
         end
+        any_instance_of(Mortar::Local::Sqoop) do |s|
+          mock(s).install_or_update
+        end
+
         mock(ctrl).ensure_local_install_dirs_in_gitignore
         ctrl.install_and_configure
       end

data/spec/mortar/local/sqoop_spec.rb

@@ -0,0 +1,57 @@
+#
+# Copyright 2012 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'spec_helper'
+require 'fakefs/spec_helpers'
+require 'mortar/local/sqoop'
+require 'launchy'
+
+
+module Mortar::Local
+  describe Sqoop do
+
+    context "prepare query" do
+
+      it "adds a where clause if none exists" do
+        sqoop = Mortar::Local::Sqoop.new
+        expect(sqoop.prep_query("SELECT * FROM customers")).to eq("SELECT * FROM customers WHERE \$CONDITIONS")
+      end
+
+      it "wraps existing where clause and appends condition" do
+        original = "SELECT * FROM customers WHERE customer_id = 1"
+        expected = "SELECT * FROM customers WHERE (customer_id = 1) AND \$CONDITIONS"
+        sqoop = Mortar::Local::Sqoop.new
+        expect(sqoop.prep_query(original)).to eq(expected)
+      end
+
+      it "wraps a complex where clause and appends condition" do
+        original = "SELECT * FROM customers WHERE (customer_id = 1 and customer_name = 'tom') or customer_id=2"
+        expected = "SELECT * FROM customers WHERE ((customer_id = 1 and customer_name = 'tom') or customer_id=2) AND \$CONDITIONS"
+        sqoop = Mortar::Local::Sqoop.new
+        expect(sqoop.prep_query(original)).to eq(expected)
+      end
+
+      it "does nothing if the user was polite enough to supply the clause themselves" do
+        query = "SELECT * FROM customers WHERE (customer_id = 1) AND \$CONDITIONS"
+        sqoop = Mortar::Local::Sqoop.new
+        expect(sqoop.prep_query(query)).to eq(query)
+      end
+
+
+    end
+
+  end
+end

metadata

@@ -1,204 +1,229 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: mortar
-version: !ruby/object:Gem::Version
-  version: 0.15.7
+version: !ruby/object:Gem::Version 
+  hash: 53
+  prerelease: 
+  segments: 
+  - 0
+  - 15
+  - 11
+  version: 0.15.11
 platform: ruby
-authors:
+authors: 
 - Mortar Data
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-14 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
+
+date: 2014-04-22 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
   name: rdoc
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 4.0.0
-  type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 63
+        segments: 
+        - 4
+        - 0
+        - 0
         version: 4.0.0
-- !ruby/object:Gem::Dependency
-  name: mortar-api-ruby
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 0.8.1
   type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: mortar-api-ruby
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
-        version: 0.8.1
-- !ruby/object:Gem::Dependency
-  name: netrc
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.7'
+      - !ruby/object:Gem::Version 
+        hash: 57
+        segments: 
+        - 0
+        - 8
+        - 3
+        version: 0.8.3
   type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: netrc
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.7'
-- !ruby/object:Gem::Dependency
-  name: launchy
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
-        version: '2.1'
+      - !ruby/object:Gem::Version 
+        hash: 5
+        segments: 
+        - 0
+        - 7
+        version: "0.7"
   type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: launchy
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '2.1'
-- !ruby/object:Gem::Dependency
-  name: parseconfig
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.0.2
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 2
+        - 1
+        version: "2.1"
   type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: parseconfig
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
+      - !ruby/object:Gem::Version 
+        hash: 19
+        segments: 
+        - 1
+        - 0
+        - 2
         version: 1.0.2
-- !ruby/object:Gem::Dependency
-  name: aws-sdk
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
   type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: aws-sdk
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
-- !ruby/object:Gem::Dependency
-  name: nokogiri
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.5.0
+      - !ruby/object:Gem::Version 
+        hash: 15
+        segments: 
+        - 1
+        - 0
+        version: "1.0"
   type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 1
+        - 5
+        - 0
         version: 1.5.0
-- !ruby/object:Gem::Dependency
+  type: :runtime
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
   name: excon
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.28'
-  type: :development
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.28'
-- !ruby/object:Gem::Dependency
-  name: fakefs
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 0.4.2
+      - !ruby/object:Gem::Version 
+        hash: 51
+        segments: 
+        - 0
+        - 28
+        version: "0.28"
   type: :development
+  version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  name: fakefs
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
+  requirement: &id009 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
+      - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 0
+        - 4
+        - 2
         version: 0.4.2
-- !ruby/object:Gem::Dependency
-  name: gem-release
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
   type: :development
+  version_requirements: *id009
+- !ruby/object:Gem::Dependency 
+  name: gem-release
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 10.1.1
+  requirement: &id010 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
   type: :development
+  version_requirements: *id010
+- !ruby/object:Gem::Dependency 
+  name: rake
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
+  requirement: &id011 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
+      - !ruby/object:Gem::Version 
+        hash: 73
+        segments: 
+        - 10
+        - 1
+        - 1
         version: 10.1.1
-- !ruby/object:Gem::Dependency
-  name: rr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
   type: :development
+  version_requirements: *id011
+- !ruby/object:Gem::Dependency 
+  name: rr
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  requirement: &id012 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
   type: :development
+  version_requirements: *id012
+- !ruby/object:Gem::Dependency 
+  name: rspec
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  requirement: &id013 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id013
 description: Client library and command-line tool to interact with the Mortar service.
 email: support@mortardata.com
-executables:
+executables: 
 - mortar
 extensions: []
+
 extra_rdoc_files: []
-files:
+
+files: 
 - README.md
 - bin/mortar
 - css/illustrate.css
@@ -247,6 +272,7 @@ files:
 - lib/mortar/local/jython.rb
 - lib/mortar/local/pig.rb
 - lib/mortar/local/python.rb
+- lib/mortar/local/sqoop.rb
 - lib/mortar/pigversion.rb
 - lib/mortar/plugin.rb
 - lib/mortar/project.rb
@@ -293,6 +319,7 @@ files:
 - lib/mortar/templates/report/illustrate-report.html
 - lib/mortar/templates/script/runpig.sh
 - lib/mortar/templates/script/runpython.sh
+- lib/mortar/templates/script/sqoop.sh
 - lib/mortar/templates/udf/python_udf.py
 - lib/mortar/updater.rb
 - lib/mortar/version.rb
@@ -323,6 +350,7 @@ files:
 - spec/mortar/local/jython_spec.rb
 - spec/mortar/local/pig_spec.rb
 - spec/mortar/local/python_spec.rb
+- spec/mortar/local/sqoop_spec.rb
 - spec/mortar/plugin_spec.rb
 - spec/mortar/project_spec.rb
 - spec/mortar/s3_spec.rb
@@ -333,25 +361,38 @@ files:
 - spec/support/display_message_matcher.rb
 homepage: http://mortardata.com/
 licenses: []
-metadata: {}
+
 post_install_message: 
 rdoc_options: []
-require_paths:
+
+require_paths: 
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 57
+      segments: 
+      - 1
+      - 8
+      - 7
       version: 1.8.7
-required_rubygems_version: !ruby/object:Gem::Requirement
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
 requirements: []
+
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 1.8.24
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Client library and CLI to interact with the Mortar service.
 test_files: []
+

checksums.yaml

@@ -1,15 +0,0 @@
----
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    YjQyM2RmYTBiYjJjNjI3NzA2NDkyMWM0ZmNjZmFkZTgwMGQxNDRkZQ==
-  data.tar.gz: !binary |-
-    YjY1ZDAzMjlmZjEzNTQyZDcwNjQ3ZjAyYmI0MDNlY2ZjNmQxMmY5MQ==
-SHA512:
-  metadata.gz: !binary |-
-    ODc3YWI0NDkwOGUyOTRlYjI0MzQ3MzgyYmY1MjA4MzBkNjEyZTM1YmUwZmM0
-    MTFjOTNlMmJhOGYxNTllMDhmODQ5MmE4N2ZjM2RjNDY2ZDY4ZTU5MGYxNjYz
-    ZWNjNzYyMTQ5NDc2ODRkM2QyMDE3MjAwOGNjYjg3MzhkYzQyYzA=
-  data.tar.gz: !binary |-
-    ODU5MzE3YTRhYmZmMzEzNWVlMjdlZGViNmY3Mzk4MTJiMjE1MTBhNjNiYjUy
-    ZDUwNjM0NTM1OGY1MzgzM2QwODRlYWRhMmFhMWE4NjQxYTQ0Y2RiMTJjN2Fi
-    Zjg0MjQzNjNkZmE5Mjk1ODdjZDE3ZjJjNzYzMDMyOWUxNWY4YzY=