mortar 0.15.36 → 0.15.37

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mortar
 version: !ruby/object:Gem::Version
-  version: 0.15.36
+  version: 0.15.37
 platform: ruby
 authors:
 - Mortar Data
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-27 00:00:00.000000000 Z
+date: 2016-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rdoc
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.13
+        version: 0.8.14
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.13
+        version: 0.8.14
 - !ruby/object:Gem::Dependency
   name: netrc
   requirement: !ruby/object:Gem::Requirement
@@ -249,7 +249,6 @@ files:
 - lib/mortar/local/params.rb
 - lib/mortar/local/pig.rb
 - lib/mortar/local/python.rb
-- lib/mortar/local/sqoop.rb
 - lib/mortar/pigversion.rb
 - lib/mortar/plugin.rb
 - lib/mortar/project.rb
@@ -304,7 +303,6 @@ files:
 - lib/mortar/templates/script/runpig.sh
 - lib/mortar/templates/script/runpython.sh
 - lib/mortar/templates/script/runstillson.sh
-- lib/mortar/templates/script/sqoop.sh
 - lib/mortar/templates/udf/python_udf.py
 - lib/mortar/updater.rb
 - lib/mortar/version.rb
@@ -337,7 +335,6 @@ files:
 - spec/mortar/local/params_spec.rb
 - spec/mortar/local/pig_spec.rb
 - spec/mortar/local/python_spec.rb
-- spec/mortar/local/sqoop_spec.rb
 - spec/mortar/plugin_spec.rb
 - spec/mortar/project_spec.rb
 - spec/mortar/s3_spec.rb
@@ -365,7 +362,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.4
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Client library and CLI to interact with the Mortar service.

data/lib/mortar/local/sqoop.rb

@@ -1,162 +0,0 @@
-#
-# Copyright 2012 Mortar Data Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-require "erb"
-require 'tempfile'
-require "mortar/helpers"
-require "mortar/local/installutil"
-
-class Mortar::Local::Sqoop
-  include Mortar::Local::InstallUtil
-  include Mortar::Helpers
-
-  def install_or_update
-    @command = "#{local_install_directory}/python/bin/python"
-    if should_do_install?
-      action "Installing sqoop to #{local_install_directory_name}" do
-        do_install
-      end
-    elsif should_do_update?
-      action "Updating to latest sqoop in #{local_install_directory_name}" do
-        do_install
-      end
-    end
-    true
-  end
-
-  def sqoop_url
-    default_url = full_host + "/" + "resource/sqoop"
-    return ENV.fetch('SQOOP_DISTRO_URL', default_url)
-  end
-
-  def should_do_install?
-    return (not (File.exists?(sqoop_directory)))
-  end
-
-  def should_do_update?
-    return is_newer_version('sqoop', sqoop_url)
-  end
-
-  def sqoop_directory
-    return "#{local_install_directory}/sqoop"
-  end
-
-  def sqoop_dir_in_tgz
-    "sqoop-1.4.4-mortar"
-  end
-
-  def do_install
-    local_tgz = File.join(local_install_directory, "sqoop-1.4.4-mortar.tar.gz")
-    if File.exists?(local_tgz)
-      FileUtils.rm(local_tgz)
-    end
-    download_file(sqoop_url, local_tgz)
-
-    if File.exists?(sqoop_directory)
-      FileUtils.rm_rf(sqoop_directory)
-    end
-
-    extract_tgz(local_tgz, local_install_directory)
-
-    FileUtils.mv(File.join(local_install_directory, sqoop_dir_in_tgz), sqoop_directory)
-
-    # This has been seening coming out of the tgz w/o +x so we do
-    # here to be sure it has the necessary permissions
-    FileUtils.chmod(0755, "#{sqoop_directory}/bin/sqoop")
-    FileUtils.chmod(0755, "#{sqoop_directory}/hadoop/bin/hadoop")
-
-    File.delete(local_tgz)
-    note_install("sqoop")
-  end
-
-  def sqoop_command_script_template_path
-    File.expand_path("../../templates/script/sqoop.sh", __FILE__)
-  end
-
-  def hadoop_home
-    "#{sqoop_directory}/hadoop"
-  end
-
-  def export(connstr, s3dest, options)
-    template_params = sqoop_export_template_parameters(connstr, s3dest, options)
-    return run_templated_script(sqoop_command_script_template_path, template_params)
-  end
-
-  def sqoop_export_template_parameters(connstr, s3dest, options)
-    pig = Mortar::Local::Pig.new()
-    parameters = {
-      "sqoop_dir" => sqoop_directory,
-      "jdb_conn_string" => connstr,
-      "destination" => s3dest,
-      "hadoop_home" => hadoop_home,
-      "classpath" => pig.template_params_classpath,
-      "postgres_jar" => "#{pig.lib_directory}/lib-cluster/postgresql.jar",
-      "jdbc_conn" => connstr,
-      "s3dest" => s3dest,
-      "project_root" => project_root,
-      "sqoop_opts" => sqoop_java_options
-    }
-    parameters["dbtable"] = options[:dbtable] if options[:dbtable]
-    parameters["sqlquery"] = options[:sqlquery] if options[:sqlquery]
-    parameters["inc_column"] = options[:inc_column] if options[:inc_column]
-    parameters["inc_value"] = options[:inc_value] if options[:inc_value]
-    if options[:inc_value] and 0 == options[:inc_value].to_i
-      parameters[:inc_mode] = "lastmodified"
-    elsif options[:inc_value]
-      parameters[:inc_mode] = "append"
-    end
-    parameters["dbuser"] = options[:username] if options[:username]
-    parameters["dbpass"] = options[:password] if options[:password]
-    parameters["jdbcdriver"] = options[:jdbcdriver] if options[:jdbcdriver]
-    parameters["driverjar"] = options[:driverjar] if options[:driverjar]
-    parameters["direct_import"] = true if options[:direct]
-    return parameters
-  end
-
-  def sqoop_java_options
-    opts = {}
-    opts['fs.s3n.awsAccessKeyId'] = ENV['AWS_ACCESS_KEY']
-    opts['fs.s3n.awsSecretAccessKey'] = ENV['AWS_SECRET_KEY']
-    opts['fs.s3.awsAccessKeyId'] = ENV['AWS_ACCESS_KEY']
-    opts['fs.s3.awsSecretAccessKey'] = ENV['AWS_SECRET_KEY']
-    opts['fs.s3.impl'] = 'org.apache.hadoop.fs.s3native.NativeS3FileSystem'
-    opts['fs.s3n.impl'] = 'org.apache.hadoop.fs.s3native.NativeS3FileSystem'
-    return opts
-  end
-
-  # So this part kind of sucks.  In order to partition a query across multiple map
-  # reduce tasks sqoop does a query to to find the range of identifiying values,
-  # divides this range across the number of tasks to be executed and then modifies
-  # the query for each m/r task. To do this Sqoop needs to know at what point in the
-  # query that it should place its portion of the where clause. This is done via the
-  # $CONDITIONS marker. So that's well and good when you're running sqoop on a cluster
-  # but our users will be running on their own machine and don't know or care for this
-  # parrallel queries stuff.  So to make their lives easier we make a best effort to
-  # add the clause for them in a safe way.
-  def prep_query(original_query)
-    if original_query.include? "$CONDITIONS"
-      return original_query
-    elsif original_query.downcase.include? "where"
-      idxwhere = original_query.downcase.index("where")
-      select_where = original_query[0..idxwhere+"where".length-1]
-      clause = original_query[idxwhere+"where".length+1..original_query.length]
-      return "#{select_where} (#{clause}) AND \$CONDITIONS"
-    else
-      return "#{original_query} WHERE \$CONDITIONS"
-    end
-  end
-
-end

data/lib/mortar/templates/script/sqoop.sh

@@ -1,40 +0,0 @@
-#!/bin/bash
-
-set -e
-
-export HADOOP_CLASSPATH="<%= @project_root %>/lib/*:<%= @postgres_jar %>"
-<% if @driverjar %>
-export HADOOP_CLASSPATH="$HADOOP_CLASSPATH:<%= @driverjar %>"
-<% end %>
-export HADOOP_COMMON_HOME="<%= @hadoop_home %>"
-export HADOOP_MAPRED_HOME="<%= @hadoop_home %>"
-export HADOOP_HOME_WARN_SUPPRESS="t"
-
-# Only setting these to get rid of warnings that sqoop is showing
-export HCAT_HOME="<%= @hadoop_home %>"
-export HBASE_HOME="<%= @hadoop_home %>"
-
-SQOOP_OPTS="<% @sqoop_opts.each do |k,v| %>-D<%= k %>=<%= v %> <% end %>"
-OPTARGS='<%= "--driver #{@jdbcdriver}" if @jdbcdriver %>'
-OPTARGS="$OPTARGS <%= "--username #{@dbuser}" if @dbuser %>"
-OPTARGS="$OPTARGS <%= "--password #{@dbpass}" if @dbpass %>"
-OPTARGS="$OPTARGS <%= "--direct" if @direct_import %>"
-<% if @inc_column and @inc_value %>
-OPTARGS="$OPTARGS --incremental <%= @inc_mode %>"
-OPTARGS="$OPTARGS --check-column <%= @inc_column %> "
-SQOOP_OPTS="$SQOOP_OPTS -Dsqoop.test.import.rootDir=<%= @s3dest %>/.tmp"
-<% end %>
-
-export HADOOP_OPTS="$SQOOP_OPTS";
-
-<%= @sqoop_dir %>/bin/sqoop \
-    import \
-    $SQOOP_OPTS \
-    <%= "--table #{@dbtable}" if @dbtable %> \
-    <%= "--query '#{@sqlquery}'" if @sqlquery %> \
-    -m 1 \
-    --connect <%= @jdbc_conn %> \
-    --target-dir <%= @s3dest %> \
-    $OPTARGS \
-    <% if @inc_column and @inc_value %>--last-value '<%= @inc_value %>'<% end %>
-

data/spec/mortar/local/sqoop_spec.rb

@@ -1,57 +0,0 @@
-#
-# Copyright 2012 Mortar Data Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-require 'spec_helper'
-require 'fakefs/spec_helpers'
-require 'mortar/local/sqoop'
-require 'launchy'
-
-
-module Mortar::Local
-  describe Sqoop do
-
-    context "prepare query" do
-
-      it "adds a where clause if none exists" do
-        sqoop = Mortar::Local::Sqoop.new
-        expect(sqoop.prep_query("SELECT * FROM customers")).to eq("SELECT * FROM customers WHERE \$CONDITIONS")
-      end
-
-      it "wraps existing where clause and appends condition" do
-        original = "SELECT * FROM customers WHERE customer_id = 1"
-        expected = "SELECT * FROM customers WHERE (customer_id = 1) AND \$CONDITIONS"
-        sqoop = Mortar::Local::Sqoop.new
-        expect(sqoop.prep_query(original)).to eq(expected)
-      end
-
-      it "wraps a complex where clause and appends condition" do
-        original = "SELECT * FROM customers WHERE (customer_id = 1 and customer_name = 'tom') or customer_id=2"
-        expected = "SELECT * FROM customers WHERE ((customer_id = 1 and customer_name = 'tom') or customer_id=2) AND \$CONDITIONS"
-        sqoop = Mortar::Local::Sqoop.new
-        expect(sqoop.prep_query(original)).to eq(expected)
-      end
-
-      it "does nothing if the user was polite enough to supply the clause themselves" do
-        query = "SELECT * FROM customers WHERE (customer_id = 1) AND \$CONDITIONS"
-        sqoop = Mortar::Local::Sqoop.new
-        expect(sqoop.prep_query(query)).to eq(query)
-      end
-
-
-    end
-
-  end
-end