rspec-hive 0.4.2 → 0.6.0

data/lib/rspec/hive/connection_delegator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'delegate'
 require 'tempfile'
 
@@ -15,9 +17,9 @@ module RSpec
         execute(table_schema.create_table_statement)
       end
 
-      def load_partitions(table_name, partitions)
-        partitions = partition_clause(partitions)
-        query = "ALTER TABLE #{table_name} ADD #{partitions}"
+      def load_partitions(table_schema, partitions)
+        partitions = partition_clause(table_schema, partitions)
+        query = "ALTER TABLE #{table_schema.name} ADD #{partitions}"
         execute(query)
       end
 
@@ -29,7 +31,7 @@ module RSpec
             values,
             table_schema.instance_variable_get(:@field_sep)
           )
-          partition_query = partition_clause(partitions) if partitions
+          partition_query = partition_clause(table_schema, partitions) if partitions
           load_file_to_hive_table(
             table_name,
             docker_path(file),
@@ -65,23 +67,27 @@ module RSpec
 
       private
 
-      def partition_clause(partitions)
+      def partition_clause(table_schema, partitions)
         if partitions.is_a?(Array)
-          partitions.collect { |x| to_partition_clause(x) }.join(' ')
+          partitions.collect { |x| to_partition_clause(table_schema, x) }.join(' ')
         else
-          to_partition_clause(partitions)
+          to_partition_clause(table_schema, partitions)
         end
       end
 
-      def to_partition_clause(partition)
-        "PARTITION(#{partition.map { |k, v| "#{k}='#{v}'" }.join(',')})"
+      def to_partition_clause(table_schema, partition)
+        "PARTITION(#{partition.map { |k, v| "#{k}=#{partition_value(table_schema, k, v)}" }.join(',')})"
+      end
+
+      def partition_value(table_schema, key, value)
+        return value if table_schema.partitions.detect { |x| x.name == key && x.type == :int }
+        "'#{value}'"
       end
 
       def load_file_to_hive_table(table_name, path, partition_clause = nil)
-        request_txt =
-          "load data local inpath '#{path}' into table #{table_name}"
-        request_txt << " #{partition_clause}" unless partition_clause.nil?
-        execute(request_txt)
+        request_txt = "load data local inpath '#{path}' into table #{table_name}"
+        return execute(request_txt) if partition_clause.nil?
+        execute("#{request_txt} #{partition_clause}")
       end
 
       def docker_path(file)

data/lib/rspec/hive/connector.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rbhive'
 require 'tempfile'
 require 'yaml'
@@ -16,11 +18,10 @@ module RSpec
         connection = open_connection
         connection.switch_database(db_name)
         @config.hive_options.each do |key, value|
-          connection.execute("SET #{key}=#{value};")
+          connection.execute("SET #{key}=#{value}")
         end
 
         connection
-
       rescue Thrift::ApplicationException => e
         config.logger.fatal('An exception was thrown during start connection')
         config.logger.fatal(e)

data/lib/rspec/hive/db_name.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'securerandom'
 
 module RSpec

data/lib/rspec/hive/exponential_backoff.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'retryable'
 
 module RSpec

data/lib/rspec/hive/matchers.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rspec/matchers'
 
 RSpec::Matchers.define :match_result_set do |expected|

data/lib/rspec/hive/query_builder.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'query_builder/row_transformer'
 require_relative 'query_builder/null_strategy'
 require_relative 'query_builder/value_by_type_strategy'

data/lib/rspec/hive/query_builder/null_strategy.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module RSpec
   module Hive
     class QueryBuilder

data/lib/rspec/hive/query_builder/row_transformer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'type_faker'
 
 module RSpec
@@ -23,12 +25,12 @@ module RSpec
 
         attr_reader :schema, :strategy
 
-        HIVE_NIL = '\N'.freeze
+        HIVE_NIL = '\N'
 
         def array_row(row)
           size = schema.columns.size
           missing = size - row.size
-          if missing > 0
+          if missing.positive?
             row_with_missing_columns(row)
           else
             row

data/lib/rspec/hive/query_builder/type_faker.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'faker'
 
 module RSpec

data/lib/rspec/hive/query_builder/value_by_type_strategy.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'type_faker'
 
 module RSpec

data/lib/rspec/hive/query_builder_helper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'query_builder'
 
 module RSpec

data/lib/rspec/hive/railtie.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rails'
 
 module RSpec

data/lib/rspec/hive/rake_tasks/docker.rake

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'yaml'
 require 'colorize'
 require 'tmpdir'
@@ -23,7 +25,14 @@ namespace :spec do
               'docker_shared_directory_path' =>
                 ENV['DOCKER_SHARED_DIR'] || default_config.docker_shared_directory_path,
               'hive_version' =>
-                ENV['HIVE_VERSION'] || default_config.hive_version
+                ENV['HIVE_VERSION'] || default_config.hive_version,
+              'hive_options' => {
+                'hive.exec.dynamic.partition' => 'true',
+                'hive.exec.dynamic.partition.mode' => 'nonstrict',
+                'hive.exec.max.dynamic.partitions.pernode' => '100000',
+                'hive.exec.max.dynamic.partitions' => '100000',
+                'mapred.child.java.opts' => '-Xmx2048m'
+              }
             }
         }
         system 'mkdir', '-p', default_values['hive']['host_shared_directory_path']
@@ -42,14 +51,16 @@ namespace :spec do
       desc 'Runs docker using hive config file.'\
             ' It assumes your docker-machine is running.'
       task :run do
-        fail 'Command `docker` not found.'.red unless system('which docker')
+        raise 'Command `docker` not found.'.red unless system('which docker')
 
         config_filepath = ENV['CONFIG_FILE'] || File.join('config', 'rspec-hive.yml')
-        fail "There's no config file #{config_filepath} please"\
-             "generate default or provide custom config.".red unless File.exist? config_filepath
+        unless File.exist? config_filepath
+          raise "There's no config file #{config_filepath} please"\
+               'generate default or provide custom config.'.red
+        end
 
         interpolated = ERB.new(File.read(config_filepath)).result
-        config = YAML.load(interpolated)['hive']
+        config = YAML.safe_load(interpolated)['hive']
 
         docker_image_name = ENV['DOCKER_IMAGE_NAME'] || 'nielsensocial/hive'
         cmd = "docker run -v #{config['host_shared_directory_path']}:"\
@@ -62,7 +73,7 @@ namespace :spec do
 
       desc 'Downloads docker image from dockerhub.'
       task :download_image do
-        fail 'Command `docker` not found.'.red unless system('which docker')
+        raise 'Command `docker` not found.'.red unless system('which docker')
 
         docker_image_name = ENV['DOCKER_IMAGE_NAME'] || 'nielsensocial/hive'
 
@@ -76,36 +87,37 @@ namespace :spec do
         docker_conatiners = `docker ps`.lines
         if docker_conatiners.size != 2
           raise 'There is more than 1 instance of docker container running (or no running docker containers). '\
-                'Check `docker ps` and stop containers that are not in use right now or specify CONTAINER_ID and run this command again.'.red
+                'Check `docker ps` and stop containers that are not in use right now or specify CONTAINER_ID '\
+                'and run this command again.'.red
         else
           docker_conatiners[1].split[0]
         end
       end
 
       desc 'Load Hive UDFS (user defined functions) onto docker.'
-      task :load_udfs, [:udfs_path] do |t, args|
+      task :load_udfs, [:udfs_path] do |_t, args|
         udfs_path = args[:udfs_path]
         config_filepath = ENV['CONFIG_FILE'] || File.join('config', 'rspec-hive.yml')
         interpolated = ERB.new(File.read(config_filepath)).result
-        config = YAML.load(interpolated)['hive']
+        config = YAML.safe_load(interpolated)['hive']
 
         host_hive_udfs_path = File.join(config['host_shared_directory_path'], 'hive-udfs.jar')
-        fail 'Please provide UDFS_PATH'.red unless udfs_path
+        raise 'Please provide UDFS_PATH'.red unless udfs_path
         if udfs_path.start_with?('s3://')
           puts 'Downloading from s3...'.yellow
           cmd = "aws s3 ls #{udfs_path}"
 
-          fail 'awscli is not configured.'.red unless system(cmd)
+          raise 'awscli is not configured.'.red unless system(cmd)
           cmd = "aws s3 cp #{udfs_path} #{host_hive_udfs_path}"
-          system(cmd)
         else
           puts 'Copying from local directory...'.yellow
           cmd = "cp #{udfs_path} #{host_hive_udfs_path}"
         end
+        system(cmd)
         puts 'Done'.green
 
         puts 'Copying to hadoop on docker...'.yellow
-        cmd = "docker exec -it #{container_id} /bin/bash -c 'cp #{config['docker_shared_directory_path']}/hive-udfs.jar $HADOOP_HOME'"
+        cmd = "docker exec -it #{container_id} /bin/bash -c 'cp #{host_hive_udfs_path} $HADOOP_HOME'"
         system(cmd)
         puts 'Done'.green
       end
@@ -114,7 +126,8 @@ namespace :spec do
     desc 'Runs beeline console on hive.'
     task :beeline do
       puts "Connecting to docker container: #{container_id} and running beeline. To exit: '!q'".green
-      cmd = "docker exec -it #{container_id} /bin/bash -c '$HIVE_HOME/bin/beeline -u jdbc:hive2://localhost:10000 -d org.apache.hive.jdbc.HiveDriver'"
+      bash_cmd = '$HIVE_HOME/bin/beeline -u jdbc:hive2://localhost:10000 -d org.apache.hive.jdbc.HiveDriver'
+      cmd = "docker exec -it #{container_id} /bin/bash -c #{bash_cmd}"
       system(cmd)
     end
   end

data/lib/rspec/hive/version.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 module RSpec
   module Hive
-    VERSION = '0.4.2'.freeze
+    VERSION = '0.6.0'
   end
 end

data/lib/rspec/hive/with_hive_connection.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'exponential_backoff'
 
 module RSpec

data/rspec-hive.gemspec

@@ -1,4 +1,6 @@
 # coding: utf-8
+# frozen_string_literal: true
+
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'rspec/hive/version'
@@ -7,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.name          = 'rspec-hive'
   spec.version       = RSpec::Hive::VERSION
   spec.authors       = ['Wojtek Mielczarek', 'Mikołaj Nowak']
-  spec.email         = %w(wojtek.mielczarek@u2i.com mikolaj.nowak@u2i.com)
+  spec.email         = %w[wojtek.mielczarek@u2i.com mikolaj.nowak@u2i.com]
   spec.summary       = 'RSpec addition to test hive queries'
   spec.description   = 'RSpecHive let you test your hive queries
                         connecting to hive instance installed on docker'
@@ -19,21 +21,10 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'rake', ['>= 10.0', '< 12.0']
-  spec.add_dependency 'colorize', '~> 0.7'
+  spec.add_dependency 'rake', ['>= 10.0', '< 13.0']
+  spec.add_dependency 'colorize', '~> 0.8.0'
   spec.add_dependency 'faker', '~> 1.6'
-  spec.add_dependency 'retryable', '~> 2.0.3'
+  spec.add_dependency 'retryable', '~> 2.0'
   spec.add_dependency 'rspec', '~> 3.4'
-  spec.add_dependency 'rbhive-u2i', '~> 1.0.0'
-
-  spec.add_development_dependency 'bundler', '~> 1.7'
-  spec.add_development_dependency 'rspec-its', '~> 1.2'
-  spec.add_development_dependency 'rubocop', '~> 0.39'
-  spec.add_development_dependency 'rubocop-rspec', '~> 1.4'
-  spec.add_development_dependency 'guard', '~> 2.6'
-  spec.add_development_dependency 'guard-rspec', '~> 4.3'
-  spec.add_development_dependency 'guard-rubocop', '~> 1.2'
-  spec.add_development_dependency 'codeclimate-test-reporter', '~> 0.4'
-  spec.add_development_dependency 'pry'
-  spec.add_development_dependency 'pry-byebug'
+  spec.add_dependency 'rbhive-u2i', '~> 1.0'
 end

data/spec/lib/rspec/hive/configuration_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'spec_helper'
 require 'tempfile'
 
@@ -32,34 +34,21 @@ RSpec.describe RSpec::Hive::Configuration do
     end
   end
 
-  let(:expected_host_shared_directory_path) do
-    '/Users/Shared/tmp/spec-tmp-files'
-  end
+  let(:expected_host_shared_directory_path) { '/Users/Shared/tmp/spec-tmp-files' }
   let(:expected_docker_shared_directory_path) { '/tmp/spec-tmp-files' }
-  let(:expected_hive_version) { 10 }
-  let(:expected_timeout) { 1800 }
-  let(:expected_partition_flag) { 'true' }
-  let(:expected_partition_mode) { 'nonstrict' }
-  let(:expected_partiton_pernodexi) { '100000' }
-  let(:expected_partitions) { '100000' }
-  let(:expected_java_opts) { '-Xmx2048m' }
-  let(:expected_hive_options) do
-    {'hive.exec.dynamic.partition' => expected_partition_flag,
-     'hive.exec.dynamic.partition.mode' => expected_partition_mode,
-     'hive.exec.max.dynamic.partitions.pernodexi' => expected_partiton_pernodexi,
-     'hive.exec.max.dynamic.partitions' => expected_partitions,
-     'mapred.child.java.opts' => expected_java_opts}
-  end
+  let(:expected_timeout) { 120 }
+  let(:expected_hive_options) { {} }
 
   context 'when no configuration file is provided' do
-    let(:expected_port) { 10000 }
+    let(:expected_port) { 10_000 }
     let!(:original_host_os) { RbConfig::CONFIG['host_os'] }
+    let(:expected_hive_version) { described_class::DEFAULT_VERSION }
 
     before { allow(Dir).to receive(:mktmpdir) { mock_tmpdir } }
 
     context 'when on Mac' do
       let(:mock_tmpdir) { '/Users/Shared/test/' }
-      let(:expected_host) { '192.168.99.100' }
+      let(:expected_host) { '127.0.0.1' }
       let(:expected_host_shared_directory_path) { '/Users/Shared/test/spec-tmp-files' }
 
       before do
@@ -88,64 +77,68 @@ RSpec.describe RSpec::Hive::Configuration do
 
   context 'when there is a configuration file' do
     let(:path_to_config_file) do
-      Tempfile.open(%w(config .yml)) do |f|
+      Tempfile.open(%w[config .yml]) do |f|
         f.write yaml_hash.to_yaml
         f.path
       end
     end
     let(:expected_host) { '127.0.0.2' }
-    let(:expected_port) { 10001 }
+    let(:expected_port) { 10_001 }
 
     context 'where all parameters are present' do
+      subject { described_class.new(path_to_config_file) }
+
+      let(:expected_hive_version) { 12 }
+
       let(:yaml_hash) do
         {
           'hive' =>
             {
               'host' => '127.0.0.2',
-              'port' => 10001,
+              'port' => 10_001,
               'host_shared_directory_path' => expected_host_shared_directory_path,
               'docker_shared_directory_path' => expected_docker_shared_directory_path,
-              'hive_version' => '10',
-              'timeout' => 1800
+              'hive_version' => '12',
+              'timeout' => 120
             }
         }
       end
 
       after { File.unlink(path_to_config_file) }
 
-      subject { described_class.new(path_to_config_file) }
-
       include_examples('config')
     end
 
     context 'where there are only required parameters' do
+      subject { described_class.new(path_to_config_file) }
+
+      let(:expected_hive_version) { described_class::DEFAULT_VERSION }
       let(:yaml_hash) do
         {
           'hive' =>
             {
               'host' => '127.0.0.2',
-              'port' => 10001,
+              'port' => 10_001,
               'host_shared_directory_path' => expected_host_shared_directory_path,
               'docker_shared_directory_path' => expected_docker_shared_directory_path
             }
         }
       end
-      let(:expected_hive_version) { 10 }
 
       after { File.unlink(path_to_config_file) }
 
-      subject { described_class.new(path_to_config_file) }
-
       include_examples('config')
     end
 
     context 'where there are some parameters required and optional' do
+      subject { described_class.new(path_to_config_file) }
+
       let(:yaml_hash) do
         {
           'hive' =>
             {
               'host' => '127.0.0.2',
-              'port' => 10001,
+              'port' => 10_001,
               'host_shared_directory_path' => expected_host_shared_directory_path,
               'docker_shared_directory_path' => expected_docker_shared_directory_path,
               'hive_version' => 11,
@@ -159,11 +152,14 @@ RSpec.describe RSpec::Hive::Configuration do
       let(:expected_timeout) { 60 }
       let(:expected_hive_version) { 11 }
       let(:expected_java_opts) { '-Xmx64m' }
+      let(:expected_hive_options) do
+        {
+          'mapred.child.java.opts' => expected_java_opts
+        }
+      end
 
       after { File.unlink(path_to_config_file) }
 
-      subject { described_class.new(path_to_config_file) }
-
       include_examples('config')
     end
   end