rspec-hive 0.1.0

data/lib/rspec/hive/configuration.rb

@@ -0,0 +1,86 @@
+require 'tmpdir'
+
+module RSpec
+  module Hive
+    class Configuration
+      attr_accessor :host,
+                    :port,
+                    :host_shared_directory_path,
+                    :docker_shared_directory_path,
+                    :logger,
+                    :hive_version,
+                    :connection_timeout,
+                    :hive_options
+
+      def initialize(path_to_config_file = nil)
+        if path_to_config_file.nil?
+          load_default_variables
+        else
+          interpolated = ERB.new(File.read(path_to_config_file)).result
+          config = YAML.load(interpolated)['hive']
+          load_variables_from_config(config)
+        end
+        @logger = Logger.new(STDOUT)
+      end
+
+      private
+
+      def load_default_variables
+        @host = platform_specific_host
+        @port = 10000
+        @host_shared_directory_path = platform_specific_host_shared_dir_path
+        @docker_shared_directory_path = '/tmp/spec-tmp-files'
+        @hive_version = default_version
+        @connection_timeout = default_timeout
+        @hive_options = default_hive_options
+      end
+
+      def load_variables_from_config(config)
+        @host = config['host']
+        @port = config['port']
+        @host_shared_directory_path = config['host_shared_directory_path']
+        @docker_shared_directory_path = config['docker_shared_directory_path']
+        @hive_version = (config['hive_version'] || default_version).to_i
+        @connection_timeout = (config['timeout'] || default_timeout).to_i
+        @hive_options = merge_config_options(default_hive_options, config)
+      end
+
+      def merge_config_options(hash, config)
+        hash.merge(config['hive_options'].to_h)
+      end
+
+      def mac?
+        host_os = RbConfig::CONFIG['host_os']
+        host_os =~ /darwin|mac os/
+      end
+
+      def platform_specific_host
+        mac? ? '192.168.99.100' : '127.0.0.1'
+      end
+
+      def platform_specific_host_shared_dir_path
+        if mac?
+          File.join(Dir.mktmpdir(nil, '/Users/Shared'), 'spec-tmp-files')
+        else
+          File.join(Dir.mktmpdir, 'spec-tmp-files')
+        end
+      end
+
+      def default_timeout
+        1800
+      end
+
+      def default_version
+        10
+      end
+
+      def default_hive_options
+        {'hive.exec.dynamic.partition' => 'true',
+         'hive.exec.dynamic.partition.mode' => 'nonstrict',
+         'hive.exec.max.dynamic.partitions.pernodexi' => '100000',
+         'hive.exec.max.dynamic.partitions' => '100000',
+         'mapred.child.java.opts' => '-Xmx2048m'}
+      end
+    end
+  end
+end

data/lib/rspec/hive/connection_delegator.rb

@@ -0,0 +1,88 @@
+require 'delegate'
+require 'tempfile'
+
+module RSpec
+  module Hive
+    class ConnectionDelegator < SimpleDelegator
+      def initialize(connection, config)
+        super(connection)
+        @config = config
+      end
+
+      def create_table(table_schema)
+        table_schema = table_schema.dup
+        table_schema.instance_variable_set(:@location, nil)
+        execute(table_schema.create_table_statement)
+      end
+
+      def load_partitions(table_name, partitions)
+        partitions = partition_clause(partitions)
+        query = "ALTER TABLE #{table_name} ADD #{partitions}"
+        execute(query)
+      end
+
+      def load_into_table(table_schema, values, partitions = nil)
+        table_name = table_schema.name
+        Tempfile.open(table_name, @config.host_shared_directory_path) do |file|
+          write_values_to_file(file, values, table_schema.instance_variable_get(:@field_sep))
+          partition_query = partition_clause(partitions) if partitions
+          load_file_to_hive_table(table_name, docker_path(file), partition_query)
+        end
+      end
+
+      def show_tables
+        fetch('SHOW TABLES')
+      end
+
+      def create_database(name)
+        execute("CREATE DATABASE IF NOT EXISTS `#{name}`")
+      end
+
+      def use_database(name)
+        execute("USE `#{name}`")
+      end
+
+      def drop_database(name)
+        execute("DROP DATABASE `#{name}`")
+      end
+
+      def show_databases
+        fetch('SHOW DATABASES')
+      end
+
+      def switch_database(db_name)
+        create_database(db_name)
+        use_database(db_name)
+      end
+
+      private
+
+      def partition_clause(partitions)
+        if partitions.is_a?(Array)
+          partitions.collect { |x| to_partition_clause(x) }.join(' ')
+        else
+          to_partition_clause(partitions)
+        end
+      end
+
+      def to_partition_clause(partition)
+        "PARTITION(#{partition.map { |k, v| "#{k}='#{v}'" }.join(',')})"
+      end
+
+      def load_file_to_hive_table(table_name, path, partition_clause = nil)
+        request_txt = "load data local inpath '#{path}' into table #{table_name}"
+        request_txt << " #{partition_clause}" unless partition_clause.nil?
+        execute(request_txt)
+      end
+
+      def docker_path(file)
+        File.join(@config.docker_shared_directory_path, File.basename(file.path))
+      end
+
+      def write_values_to_file(file, values, delimiter=';')
+        values.each { |value| file.puts(value.join(delimiter)) }
+        file.flush
+      end
+    end
+  end
+end

data/lib/rspec/hive/connector.rb

@@ -0,0 +1,82 @@
+require 'rbhive'
+require 'tempfile'
+require 'yaml'
+require 'pry'
+
+module RSpec
+  module Hive
+    class Connector
+      attr_reader :result, :config
+
+      def initialize(configuration)
+        @config = configuration
+        # transport: :sasl, sasl_params: {username: 'hive', password: ''},
+      end
+
+      def start_connection(db_name = DbName.random_name)
+        connection = open_connection
+        connection.switch_database(db_name)
+        @config.hive_options.each do |key, value|
+          connection.execute("SET #{key}=#{value};")
+        end
+
+        connection
+
+      rescue Thrift::ApplicationException => e
+        config.logger.fatal('An exception was thrown during start connection')
+        config.logger.fatal(e)
+        stop_connection(connection)
+        connection
+      end
+
+      def stop_connection(connection)
+        connection.close_session if connection.session
+        connection.close
+      rescue IOError => e
+        config.logger.fatal('An exception was thrown during close connection')
+        config.logger.fatal(e)
+      end
+
+      def tlcli_connect
+        RBHive.tcli_connect(@config.host,
+                            @config.port,
+                            connection_options) do |connection|
+          yield ConnectionDelegator.new(connection, @config)
+        end
+      end
+
+      private
+
+      def log_connection_params
+        @config.logger.info('Opening connection.')
+        @config.logger.info("Connection options: #{connection_options}")
+        @config.logger.info("Config #{@config.inspect}")
+      end
+
+      def open_connection
+        log_connection_params
+
+        connection = RBHive::TCLIConnection.new(
+          @config.host,
+          @config.port,
+          connection_options
+        )
+        connection = ConnectionDelegator.new(connection, @config)
+
+        connection.open
+        connection.open_session
+        connection
+      end
+
+      def connection_options
+        {
+          hive_version: @config.hive_version,
+          transport: :sasl,
+          sasl_params: {},
+          logger: @config.logger,
+          timeout: @config.connection_timeout
+        }
+      end
+    end
+  end
+end

data/lib/rspec/hive/db_name.rb

@@ -0,0 +1,23 @@
+require 'securerandom'
+
+module RSpec
+  module Hive
+    class DbName
+      class << self
+        def random_name
+          "#{timestamp}_#{random_key}"
+        end
+
+        private
+
+        def timestamp
+          Time.now.getutc.to_i.to_s
+        end
+
+        def random_key
+          SecureRandom.uuid.delete('-')
+        end
+      end
+    end
+  end
+end

data/lib/rspec/hive/version.rb

@@ -0,0 +1,5 @@
+module RSpec
+  module Hive
+    VERSION = '0.1.0'
+  end
+end

data/lib/rspec/hive/with_hive_connection.rb

@@ -0,0 +1,27 @@
+module RSpec
+  module Hive
+    module WithHiveConnection
+      def hive
+        Hive.connector
+      end
+
+      def connection
+        @connection ||= hive.start_connection
+      end
+
+      def self.included(mod)
+        mod.before(:all) do
+          connection
+        end
+
+        mod.before(:each) do
+          connection.switch_database(DbName.random_name)
+        end
+
+        mod.after(:all) do
+          hive.stop_connection(connection) unless hive && connection
+        end
+      end
+    end
+  end
+end

data/lib/rspec/rake_tasks/docker.rake

@@ -0,0 +1,76 @@
+require 'yaml'
+require 'colorize'
+require 'tmpdir'
+require 'rspec/hive'
+
+namespace :spec do
+  namespace :hive do
+    namespace :config do
+      desc 'Generates example config file. Accepts directory to file.'
+      task :generate_default do
+        require 'rbconfig'
+
+        default_config = RSpec::Hive::Configuration.new
+
+        default_values = {
+          'hive' =>
+            {
+              'host' => ENV['HOST'] || default_config.host,
+              'port' => ENV['PORT'] || default_config.port,
+              'host_shared_directory_path' =>
+                ENV['HOST_SHARED_DIR'] || default_config.host_shared_directory_path,
+              'docker_shared_directory_path' =>
+                ENV['DOCKER_SHARED_DIR'] || default_config.docker_shared_directory_path,
+              'hive_version' =>
+                ENV['HIVE_VERSION'] || default_config.hive_version
+            }
+        }
+        system 'mkdir', '-p', default_values['hive']['host_shared_directory_path']
+        file_path = File.join(
+          ENV['CONFIG_FILE_DIR'] || '.',
+          ENV['CONFIG_FILE_NAME'] || 'rspec-hive.yml'
+        )
+        File.open(file_path, 'w+') do |f|
+          f.write default_values.to_yaml
+          puts "Default config written to #{f.path}".green
+        end
+      end
+    end
+
+    namespace :docker do
+      desc 'Runs docker using hive config file.'\
+            ' It assumes your docker-machine is running.'
+      task :run do
+        puts 'Command `docker` not found.'.red unless system('which docker')
+
+        config_filepath = ENV['CONFIG_FILE'] || 'rspec-hive.yml'
+        docker_image_name = ENV['DOCKER_IMAGE_NAME'] || 'nielsensocial/hive'
+        unless File.exist? config_filepath
+          puts "There's no config file #{config_filepath} please generate default or provide custom config.".red
+          raise Errno::ENOENT.new config_filepath unless File.exist? config_filepath
+        end
+
+        interpolated = ERB.new(File.read(config_filepath)).result
+        config = YAML.load(interpolated)['hive']
+
+        cmd = "docker run -v #{config['host_shared_directory_path']}:"\
+              "#{config['docker_shared_directory_path']}"\
+              " -d -p #{config['port']}:10000 #{docker_image_name}"
+
+        puts "Running `#{cmd}`...".green
+        system(cmd)
+      end
+
+      desc 'Downloads docker image from dockerhub.'
+      task :download_image do
+        puts 'Command `docker` not found.'.red unless system('which docker')
+
+        docker_image_name = ENV['DOCKER_IMAGE_NAME'] || 'nielsensocial/hive'
+
+        cmd = "docker pull #{docker_image_name}"
+        puts "Running `#{cmd}`...".green
+        system(cmd)
+      end
+    end
+  end
+end

data/lib/rspec/rake_tasks/railtie.rb

@@ -0,0 +1,11 @@
+require 'rails'
+
+module RSpec
+  module Hive
+    class Railtie < Rails::Railtie
+      rake_tasks do
+        load 'rspec/rake_tasks/docker.rake'
+      end
+    end
+  end
+end

data/rspec-hive.gemspec

@@ -0,0 +1,37 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rspec/hive/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'rspec-hive'
+  spec.version       = RSpec::Hive::VERSION
+  spec.authors       = ['Wojtek Mielczarek', 'Mikołaj Nowak']
+  spec.email         = %w(wojtek.mielczarek@u2i.com mikolaj.nowak@u2i.com)
+  spec.summary       = 'RSpec addition to test hive queries'
+  spec.description   = 'RSpecHive let you test your hive queries
+                        connecting to hive instance installed on docker'
+  spec.homepage      = 'https://github.com/u2i/ns-rspec-hive'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'rake', '~> 10.0'
+  spec.add_dependency 'colorize', '~> 0.7'
+
+  spec.add_development_dependency 'bundler', '~> 1.7'
+  spec.add_development_dependency 'rspec', '~> 3.4'
+  spec.add_development_dependency 'rspec-its', '~> 1.2'
+  spec.add_development_dependency 'rbhive', '~> 0.6.0'
+  spec.add_development_dependency 'rubocop', '~> 0.34'
+  spec.add_development_dependency 'rubocop-rspec', '~> 1.3'
+  spec.add_development_dependency 'guard', '~> 2.6'
+  spec.add_development_dependency 'guard-rspec', '~> 4.3'
+  spec.add_development_dependency 'guard-rubocop', '~> 1.2'
+  spec.add_development_dependency 'codeclimate-test-reporter', '~> 0.4'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'pry-byebug'
+end