rspec-hive 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0b75898c0b7364fe12b80f91a36da04a54c5bab6
-  data.tar.gz: 86e7fcbbb5a50fc536a0954ddc27fd4cb90ef915
+  metadata.gz: ff9d8db0b9d16154a53d7da9cee367f6e0b7eb92
+  data.tar.gz: 52e4eb836d42abaa5d7493f2b77f5a851ace0593
 SHA512:
-  metadata.gz: da919d5129641e77e15c5bfeed2bef2396e3d02dd3eaa93ef61dae7af099323d6a9106a8b9b8ac02644035ab39588a88c6cbabbe40f7f53832e961d499a7244f
-  data.tar.gz: 2ea4738a1f9bb1d84426ed4873a33bfb2b8f2480a1b557fbb73e2b5fe8821fbb266defffe8fab57c3114558d6c10c3e0a8f9ee3d11fc0fd0c5138de884c4748c
+  metadata.gz: 7374dda80d27b31634bf4651369c7eb5fece08407aa824cbd915b45e7469e22570c6370efc55659a4d42b77e761f438505b7165c56929900354eafa03daf0a4e
+  data.tar.gz: 2505ba599502e1ad140472d2898d48e77bed2cfcc0e43f8b2b134882afdfbe1827bb8c09a6405213236bd9e0263ddcf1a0d46d4f0d72fc081a741c495d728397

data/lib/rspec/hive/query_builder_helper.rb

@@ -3,6 +3,8 @@ require_relative 'query_builder'
 module RSpec
   module Hive
     module QueryBuilderHelper
+      HiveConnectionNotFound = Class.new(StandardError)
+
       def into_hive(schema)
         hive_connection_guard!
         ::RSpec::Hive::QueryBuilder.new(schema, connection)
@@ -17,7 +19,7 @@ module RSpec
       end
 
       def hive_connection_guard!
-        raise 'Include WithHiveConnection' unless hive_connection_present?
+        raise HiveConnectionNotFound, 'Include WithHiveConnection' unless hive_connection_present?
       end
     end
   end

data/lib/rspec/hive/rake_tasks/docker.rake

@@ -26,10 +26,10 @@ namespace :spec do
             }
         }
         system 'mkdir', '-p', default_values['hive']['host_shared_directory_path']
-        file_path = File.join(
-          ENV['CONFIG_FILE_DIR'] || '.',
-          ENV['CONFIG_FILE_NAME'] || 'rspec-hive.yml'
-        )
+        system 'mkdir', '-p', 'config'
+
+        file_path = ENV['CONFIG_FILE'] || File.join('config', 'rspec-hive.yml')
+
         File.open(file_path, 'w+') do |f|
           f.write default_values.to_yaml
           puts "Default config written to #{f.path}".green
@@ -41,18 +41,16 @@ namespace :spec do
       desc 'Runs docker using hive config file.'\
             ' It assumes your docker-machine is running.'
       task :run do
-        puts 'Command `docker` not found.'.red unless system('which docker')
+        fail 'Command `docker` not found.'.red unless system('which docker')
 
-        config_filepath = ENV['CONFIG_FILE'] || 'rspec-hive.yml'
-        docker_image_name = ENV['DOCKER_IMAGE_NAME'] || 'nielsensocial/hive'
-        unless File.exist? config_filepath
-          puts "There's no config file #{config_filepath} please generate default or provide custom config.".red
-          raise Errno::ENOENT.new config_filepath unless File.exist? config_filepath
-        end
+        config_filepath = ENV['CONFIG_FILE'] || File.join('config', 'rspec-hive.yml')
+        fail "There's no config file #{config_filepath} please"\
+             "generate default or provide custom config.".red unless File.exist? config_filepath
 
         interpolated = ERB.new(File.read(config_filepath)).result
         config = YAML.load(interpolated)['hive']
 
+        docker_image_name = ENV['DOCKER_IMAGE_NAME'] || 'nielsensocial/hive'
         cmd = "docker run -v #{config['host_shared_directory_path']}:"\
               "#{config['docker_shared_directory_path']}"\
               " -d -p #{config['port']}:10000 #{docker_image_name}"
@@ -63,7 +61,7 @@ namespace :spec do
 
       desc 'Downloads docker image from dockerhub.'
       task :download_image do
-        puts 'Command `docker` not found.'.red unless system('which docker')
+        fail 'Command `docker` not found.'.red unless system('which docker')
 
         docker_image_name = ENV['DOCKER_IMAGE_NAME'] || 'nielsensocial/hive'
 
@@ -71,6 +69,52 @@ namespace :spec do
         puts "Running `#{cmd}`...".green
         system(cmd)
       end
+
+      def container_id
+        return ENV['CONTAINER_ID'] if ENV['CONTAINER_ID']
+        docker_conatiners = `docker ps`.lines
+        if docker_conatiners.size != 2
+          raise 'There is more than 1 instance of docker container running (or no running docker containers). '\
+                'Check `docker ps` and stop containers that are not in use right now or specify CONTAINER_ID and run this command again.'.red
+        else
+          docker_conatiners[1].split[0]
+        end
+      end
+
+      desc 'Load Hive UDFS (user defined functions) onto docker.'
+      task :load_udfs, [:udfs_path] do |t, args|
+        udfs_path = args[:udfs_path]
+        config_filepath = ENV['CONFIG_FILE'] || File.join('config', 'rspec-hive.yml')
+        interpolated = ERB.new(File.read(config_filepath)).result
+        config = YAML.load(interpolated)['hive']
+
+        host_hive_udfs_path = File.join(config['host_shared_directory_path'], 'hive-udfs.jar')
+        fail 'Please provide UDFS_PATH'.red unless udfs_path
+        if udfs_path.start_with?('s3://')
+          puts 'Downloading from s3...'.yellow
+          cmd = "aws s3 ls #{udfs_path}"
+
+          fail 'awscli is not configured.'.red unless system(cmd)
+          cmd = "aws s3 cp #{udfs_path} #{host_hive_udfs_path}"
+          system(cmd)
+        else
+          puts 'Copying from local directory...'.yellow
+          cmd = "cp #{udfs_path} #{host_hive_udfs_path}"
+        end
+        puts 'Done'.green
+
+        puts 'Copying to hadoop on docker...'.yellow
+        cmd = "docker exec -it #{container_id} /bin/bash -c 'cp #{config['docker_shared_directory_path']}/hive-udfs.jar $HADOOP_HOME'"
+        system(cmd)
+        puts 'Done'.green
+      end
+    end
+
+    desc 'Runs beeline console on hive.'
+    task :beeline do
+      puts "Connecting to docker container: #{container_id} and running beeline. To exit: '!q'".green
+      cmd = "docker exec -it #{container_id} /bin/bash -c '$HIVE_HOME/bin/beeline -u jdbc:hive2://localhost:10000 -d org.apache.hive.jdbc.HiveDriver'"
+      system(cmd)
     end
   end
 end

data/lib/rspec/hive/version.rb

@@ -1,5 +1,5 @@
 module RSpec
   module Hive
-    VERSION = '0.3.0'.freeze
+    VERSION = '0.4.0'.freeze
   end
 end

data/spec/lib/rspec/hive/query_builder_helper_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+
+RSpec.describe RSpec::Hive::QueryBuilderHelper do
+  let(:connection) { double }
+  let(:schema) { double }
+  let(:dummy_class) { double }
+
+  before { dummy_class.extend(described_class) }
+
+  describe '#into_hive' do
+    context 'when no connection has been defined' do
+      it 'raises and error' do
+        expect { dummy_class.into_hive(schema) }.
+          to raise_error(RSpec::Hive::QueryBuilderHelper::HiveConnectionNotFound).
+          with_message('Include WithHiveConnection')
+      end
+    end
+
+    context 'when RBhive connection has been given' do
+      let(:dummy_class) { double(connection: connection) }
+
+      before do
+        allow(connection).to receive(:is_a?).with(RBHive::TCLIConnection).and_return(true)
+      end
+
+      subject { dummy_class.into_hive(schema) }
+
+      it 'returns a query_builder' do
+        expect(subject).to be_a_kind_of(RSpec::Hive::QueryBuilder)
+      end
+    end
+
+    context 'when ConnectionDelegator has been given' do
+      let(:dummy_class) { double(connection: connection) }
+
+      before do
+        allow(connection).to receive(:is_a?).with(RBHive::TCLIConnection).and_return(false)
+        allow(connection).to receive(:is_a?).with(RSpec::Hive::ConnectionDelegator).and_return(true)
+      end
+
+      subject { dummy_class.into_hive(schema) }
+
+      it 'returns a query_builder' do
+        expect(subject).to be_a_kind_of(RSpec::Hive::QueryBuilder)
+      end
+    end
+  end
+end

data/spec/lib/rspec/hive/query_builder_spec.rb

@@ -4,6 +4,7 @@ RSpec.describe RSpec::Hive::QueryBuilder do
   let(:connection) { instance_double(RBHive::TCLIConnection) }
   let(:connection_delegator) { RSpec::Hive::ConnectionDelegator.new(connection, {}) }
   let(:query_builder) { described_class.new(schema, connection_delegator) }
+  let(:schema) { double }
 
   describe '#execute' do
     subject { builder.execute }
@@ -13,7 +14,6 @@ RSpec.describe RSpec::Hive::QueryBuilder do
         expect(connection_delegator).to receive(:load_into_table).with(schema, expected_rows)
       end
 
-      let(:partition) { nil }
       let(:schema) do
         RBHive::TableSchema.new('table_name', nil) do
           column :col1, :string
@@ -130,7 +130,7 @@ RSpec.describe RSpec::Hive::QueryBuilder do
       end
     end
 
-    context 'when has no partition' do
+    context 'when has a partition' do
       let(:schema) do
         RBHive::TableSchema.new('table_name', nil) do
           column :col1, :string
@@ -138,6 +138,24 @@ RSpec.describe RSpec::Hive::QueryBuilder do
           partition :dt, :int
         end
       end
+
+      before do
+        expect(connection_delegator).
+          to receive(:load_into_table).with(schema, expected_rows, partitions)
+      end
+
+      context 'when no data stubbing' do
+        context 'when single row is passed' do
+          let(:builder) { query_builder.insert(row1).partition(partitions) }
+          let(:row1) { ['col1', 343] }
+          let(:partitions) { {dt: :int} }
+          let(:expected_rows) { [row1] }
+
+          it 'loads single row' do
+            subject
+          end
+        end
+      end
     end
   end
 end

data/spec/lib/rspec/hive_spec.rb

@@ -1,7 +1,13 @@
 require 'spec_helper'
 
 RSpec.describe RSpec::Hive do
-  describe 'configure' do
+  describe '.connector' do
+    subject { described_class.connector }
+
+    it { is_expected.to be_an_instance_of(RSpec::Hive::Connector) }
+  end
+
+  describe '.configure' do
     let(:expected_host) { '127.0.0.1' }
     let(:expected_port) { '10000' }
     let(:expected_host_shared_directory_path) do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rspec-hive
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Wojtek Mielczarek
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-12 00:00:00.000000000 Z
+date: 2016-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -288,6 +288,7 @@ files:
 - spec/lib/rspec/hive/db_name_spec.rb
 - spec/lib/rspec/hive/matchers_spec.rb
 - spec/lib/rspec/hive/query_builder/row_transformer_spec.rb
+- spec/lib/rspec/hive/query_builder_helper_spec.rb
 - spec/lib/rspec/hive/query_builder_spec.rb
 - spec/lib/rspec/hive_spec.rb
 - spec/spec_helper.rb
@@ -323,6 +324,7 @@ test_files:
 - spec/lib/rspec/hive/db_name_spec.rb
 - spec/lib/rspec/hive/matchers_spec.rb
 - spec/lib/rspec/hive/query_builder/row_transformer_spec.rb
+- spec/lib/rspec/hive/query_builder_helper_spec.rb
 - spec/lib/rspec/hive/query_builder_spec.rb
 - spec/lib/rspec/hive_spec.rb
 - spec/spec_helper.rb