rspec-hive 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ca84700fd2dd11920a2cf224d634f444a9e325b9
-  data.tar.gz: ac29aa0c169716541e7df32d8ddcfed8ed8a005e
+  metadata.gz: e0ecaa7596db283e59f0ea63fbb677942e714d09
+  data.tar.gz: ae43c8734de7c85b0551454534fc1e339a74d2f5
 SHA512:
-  metadata.gz: 5ec841bc2886f78ff101a18481ff622abf64c4ae5abb387441fab204d77327c805c2181f94201e850911cdb248ca3a06c3f1f097074abfcf70438eb2730626ac
-  data.tar.gz: 4cc12280d1c740e9c8b349995def32422796ac4c96bd7061d2d13ef936bc1fe3fbeea78eb138807e2c1c4751b1de26eb171927ad5c14c2ea23c63493acbdb618
+  metadata.gz: 906bd8f828ba76a4646c535cee8679d60f3acf23714e8686c62789ee770d754f95f216930758d504bf57964a40c127e189b58dc17e03c674f12a283f181e83cf
+  data.tar.gz: 6cf54197ee350b05235eb08869226dd2b26c3e63f83764ad17356c5ef842379f8d5b5b2d4a8aafe63d212d91b02c91b78d64bcd0bb729c5bb37daf8a8da05084

data/.rspec

@@ -1,2 +1 @@
 --color
---require spec_helper

data/.rubocop.yml

@@ -1,11 +1,23 @@
 inherit_from: .rubocop_u2i.yml
 
+Rails:
+  Enabled: false
+
 AllCops:
+  TargetRubyVersion: 2.1
+  Include:
+    - '**/Rakefile'
   Exclude:
     - spec/spec_helper.rb
+    - vendor/**/*
+    - examples/query_spec.rb
 
 Metrics/LineLength:
-  Max: 80
+  Max: 120
   Enabled: true
   Exclude:
     - spec/**/*
+    - examples/spec/**/*
+
+Style/SignalException:
+  EnforcedStyle: only_raise

data/.rubocop_u2i.yml

@@ -1,7 +1,9 @@
 require: rubocop-rspec
 
+Rails:
+  Enabled: true
+
 AllCops:
-  RunRailsCops: true
   Include:
     - '**/Rakefile'
     - config.ru
@@ -11,6 +13,7 @@ AllCops:
     - config/**/*
     - script/**/*
     - db/schema.rb
+    - vendor/**/*
 
 Style/MultilineOperationIndentation:
   EnforcedStyle: indented
@@ -53,3 +56,8 @@ Metrics/MethodLength:
 RSpec/FilePath:
   Exclude:
     - spec/routing/**/*_routing_spec.rb
+
+RSpec/DescribeClass:
+  Exclude:
+    - spec/lib/rspec/hive/matchers_spec.rb
+

data/.travis.yml

@@ -0,0 +1,19 @@
+sudo: false
+
+services:
+  - docker
+language: ruby
+rvm:
+  - 2.1
+  - 2.2
+  - 2.3.0
+env: CONFIG_FILE_NAME=rspec-hive.yml PORT=10000 CONFIG_FILE=rspec-hive.yml
+
+before_script:
+  - bundle exec rake spec:hive:config:generate_default
+  - bundle exec rake spec:hive:docker:run
+
+script:
+  - bundle exec rake spec
+  - bundle exec rake rubocop
+  - bundle exec rake hive_spec

data/Guardfile

@@ -1,6 +1,11 @@
 # A sample Guardfile
 # More info at https://github.com/guard/guard#readme
 
+detect_docker = <<-BASH
+  CONTAINER_IDS=`docker ps -q --filter='ancestor=nielsensocial/hive' 2> /dev/null | xargs`
+  docker inspect --format='{{ .State.Running }}' $CONTAINER_IDS 2> /dev/null | uniq | grep true 2>&1 > /dev/null
+BASH
+
 group :red_green_refactor, halt_on_fail: true do
   guard :rspec, cmd: 'bundle exec rspec' do
     watch(%r{^spec/.+_spec\.rb$})
@@ -8,7 +13,13 @@ group :red_green_refactor, halt_on_fail: true do
     watch('spec/spec_helper.rb') { 'spec' }
   end
 
-  guard :rubocop do
+  guard :rspec, cmd: 'bundle exec rspec', spec_paths: ['examples/spec'] do
+    watch(%r{^examples/spec/.+_spec\.rb$})
+    watch(%r{^examples/lib/(.+)\.rb$}) { |m| "examples/spec/#{m[1]}_spec.rb" }
+    watch('examples/spec/spec_helper.rb') { 'examples/spec' }
+  end if system(detect_docker)
+
+  guard :rubocop, all_on_start: false do
     watch(%r{.+\.rb$})
     watch(%r{(?:.+/)?\.rubocop\.yml$}) { |m| File.dirname(m[0]) }
   end

data/README.md

@@ -1,8 +1,10 @@
+[![Build Status](https://travis-ci.org/u2i/rspec-hive.svg?branch=master)](https://travis-ci.org/u2i/rspec-hive)
+[![Dependency Status](https://gemnasium.com/u2i/rspec-hive.svg)](https://gemnasium.com/u2i/rspec-hive)
 [![Code Climate](https://codeclimate.com/repos/567b03d7bd3f3b2512002248/badges/85abbc07acb75f664185/gpa.svg)](https://codeclimate.com/repos/567b03d7bd3f3b2512002248/feed)
 [![Test Coverage](https://codeclimate.com/repos/567b03d7bd3f3b2512002248/badges/85abbc07acb75f664185/coverage.svg)](https://codeclimate.com/repos/567b03d7bd3f3b2512002248/coverage)
 [![Issue Count](https://codeclimate.com/repos/567b03d7bd3f3b2512002248/badges/85abbc07acb75f664185/issue_count.svg)](https://codeclimate.com/repos/567b03d7bd3f3b2512002248/feed)
 
-# rpsec-hive
+# rspec-hive
 
 rspec-hive is a utility gem to help you write beautiful rspec tests for hive queries. The idea is simple - you just launch a docker machine with hadoop and hive installed. To test a query you create a simple RSpec file and extend it with `RSpec::Hive::WithHiveConnection`.
 

data/Rakefile

@@ -1,5 +1,5 @@
 require 'bundler/gem_tasks'
-load 'lib/rspec/rake_tasks/docker.rake'
+load 'lib/rspec/hive/rake_tasks/docker.rake'
 
 begin
   require 'rspec/core/rake_task'
@@ -20,4 +20,13 @@ rescue LoadError
   puts 'Cannot load RuboCop Rake tasks'
 end
 
-task default: [:spec, :rubocop]
+detect_docker = <<-BASH
+  CONTAINER_IDS=`docker ps -q --filter='ancestor=nielsensocial/hive' 2> /dev/null | xargs`
+  docker inspect --format='{{ .State.Running }}' $CONTAINER_IDS 2> /dev/null | uniq | grep true 2>&1 > /dev/null
+BASH
+
+if system(detect_docker)
+  task default: [:spec, :hive_spec, :rubocop]
+else
+  task default: [:spec, :rubocop]
+end

data/examples/{query.rb → lib/query.rb}

@@ -1,3 +1,5 @@
+require 'rbhive'
+
 class Query
   def table_schema
     RBHive::TableSchema.new('people', nil, line_sep: '\n', field_sep: ';') do
@@ -10,9 +12,4 @@ class Query
   def table_name
     table_schema.name
   end
-
-  def run_hive_query(connection)
-    query = "SELECT * FROM `#{table_schema.name}` WHERE `amount` > 2.5"
-    connection.execute(query)
-  end
 end

data/examples/spec/query_spec.rb

@@ -0,0 +1,103 @@
+require_relative 'spec_helper'
+require_relative '../lib/query'
+
+RSpec.describe Query do
+  include RSpec::Hive::WithHiveConnection
+  include RSpec::Hive::QueryBuilderHelper
+
+  subject { described_class.new }
+
+  let(:schema) { subject.table_schema }
+
+  before { connection.execute(schema.create_table_statement) }
+
+  context 'if we have a partition' do
+    let(:schema) do
+      RBHive::TableSchema.new('partitioned_people', nil, line_sep: '\n', field_sep: ';') do
+        column :name, :string
+        column :address, :string
+        column :amount, :float
+        partition :dth, :int
+      end
+    end
+    let(:input_data) do
+      [
+        ['Mikolaj', 'Cos', 1.23, 1],
+        ['Wojtek', 'Cos', 3.76, 2]
+      ]
+    end
+    let(:dth) { '2016042210' }
+    let(:query) { "SELECT * FROM `#{schema.name}` WHERE amount > 3.2" }
+    let(:query_result) { connection.fetch(query) }
+    let(:expected_result_array) do
+      [[a_string_matching('Wojtek'), 'Cos', be_within(0.01).of(3.76), dth]]
+    end
+
+    before do
+      connection.execute("ALTER TABLE #{schema.name} ADD PARTITION (dth='#{dth}')")
+      into_hive(schema).insert(*input_data).partition(dth: dth).execute
+    end
+
+    it 'returns Wojtek' do
+      expect(query_result).to match_result_set(expected_result_array)
+    end
+  end
+
+  context 'without stubbing strategy' do
+    let(:input_data) do
+      [
+        ['Mikolaj', 'Cos', 1.23, 1],
+        ['Wojtek', 'Cos', 3.76, 2]
+      ]
+    end
+    let(:query_result) { connection.fetch(query) }
+
+    before { into_hive(schema).insert(*input_data).execute }
+
+    context 'when querying for amount > 3.2' do
+      let(:query) { "SELECT * FROM `#{subject.table_name}` WHERE amount > 3.2" }
+      let(:expected_result_array) do
+        [
+          [a_string_matching('Wojtek'), 'Cos', be_within(0.01).of(3.76)]
+        ]
+      end
+
+      it 'returns Wojtek' do
+        expect(query_result).to match_result_set(expected_result_array)
+      end
+    end
+  end
+
+  context 'with stubbing strategy' do
+    let(:input_data) { [{name: 'Michal'}, {name: 'Wojtek'}] }
+    let(:query_result) { connection.fetch(query) }
+
+    before { into_hive(schema).insert(*input_data).with_stubbing.execute }
+
+    context "when querying for name = 'Wojtek'" do
+      let(:query) do
+        "SELECT * FROM `#{subject.table_name}` WHERE name='Wojtek'"
+      end
+      let(:expected_result_array) do
+        [
+          ['Wojtek', a_kind_of(String), a_kind_of(Float)]
+        ]
+      end
+
+      it 'returns Wojtek' do
+        expect(query_result).to match_result_set(expected_result_array)
+      end
+    end
+
+    context 'when querying for name = Michal' do
+      let(:query) do
+        "SELECT * FROM `#{subject.table_name}` WHERE name='Michal'"
+      end
+      let(:expected_result_hash) { [{name: 'Michal'}] }
+
+      it 'returns Michal' do
+        expect(query_result).to match_result_set(expected_result_hash).partially
+      end
+    end
+  end
+end

data/examples/spec/spec_helper.rb

@@ -0,0 +1,22 @@
+if ENV['COVERAGE']
+  require 'simplecov'
+  SimpleCov.start
+  require 'simplecov'
+  SimpleCov.start
+end
+
+RSpec.configure do |config|
+  require 'rspec/hive'
+
+  RSpec::Hive.configure(File.join(__dir__, '../../rspec-hive.yml'))
+
+  config.mock_with :rspec do |mocks|
+    mocks.verify_partial_doubles = true
+  end
+
+  config.pattern = '*_spec.rb'
+
+  config.disable_monkey_patching!
+  config.order = :random
+  Kernel.srand config.seed
+end

data/lib/rspec/hive.rb

@@ -4,10 +4,15 @@ require 'rspec/hive/configuration'
 require 'rspec/hive/connection_delegator'
 require 'rspec/hive/connector'
 require 'rspec/hive/with_hive_connection'
+require 'rspec/hive/query_builder'
+require 'rspec/hive/query_builder_helper'
+require 'rspec/hive/matchers'
 
 module RSpec
   module Hive
-    attr_reader :configuration
+    def self.configuration
+      @configuration
+    end
 
     def self.configure(file_name = nil)
       @configuration = new_configuration(file_name)
@@ -28,4 +33,4 @@ module RSpec
   end
 end
 
-require 'rspec/rake_tasks/railtie' if defined?(Rails)
+require 'rspec/hive/railtie' if defined?(::Rails)

data/lib/rspec/hive/connection_delegator.rb

@@ -24,9 +24,17 @@ module RSpec
       def load_into_table(table_schema, values, partitions = nil)
         table_name = table_schema.name
         Tempfile.open(table_name, @config.host_shared_directory_path) do |file|
-          write_values_to_file(file, values, table_schema.instance_variable_get(:@field_sep))
+          write_values_to_file(
+            file,
+            values,
+            table_schema.instance_variable_get(:@field_sep)
+          )
           partition_query = partition_clause(partitions) if partitions
-          load_file_to_hive_table(table_name, docker_path(file), partition_query)
+          load_file_to_hive_table(
+            table_name,
+            docker_path(file),
+            partition_query
+          )
         end
       end
 
@@ -70,16 +78,20 @@ module RSpec
       end
 
       def load_file_to_hive_table(table_name, path, partition_clause = nil)
-        request_txt = "load data local inpath '#{path}' into table #{table_name}"
+        request_txt =
+          "load data local inpath '#{path}' into table #{table_name}"
         request_txt << " #{partition_clause}" unless partition_clause.nil?
         execute(request_txt)
       end
 
       def docker_path(file)
-        File.join(@config.docker_shared_directory_path, File.basename(file.path))
+        File.join(
+          @config.docker_shared_directory_path,
+          File.basename(file.path)
+        )
       end
 
-      def write_values_to_file(file, values, delimiter=';')
+      def write_values_to_file(file, values, delimiter = ';')
         values.each { |value| file.puts(value.join(delimiter)) }
         file.flush
       end

data/lib/rspec/hive/connector.rb

@@ -1,7 +1,6 @@
 require 'rbhive'
 require 'tempfile'
 require 'yaml'
-require 'pry'
 
 module RSpec
   module Hive

data/lib/rspec/hive/exponential_backoff.rb

@@ -0,0 +1,15 @@
+require 'retryable'
+
+module RSpec
+  module Hive
+    class ExponentialBackoff
+      class << self
+        def retryable(tries: 5, on:)
+          Retryable.retryable(tries: tries, sleep: ->(r) { 2**r }, on: on) do
+            yield
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/hive/matchers.rb

@@ -0,0 +1,29 @@
+require 'rspec/matchers'
+
+RSpec::Matchers.define :match_result_set do |expected|
+  match do |actual|
+    return false if expected.size != actual.size
+
+    expected.each.with_index.all? do |expected_row, i|
+      if expected_row.respond_to?(:each_pair)
+        if @partial_match
+          selected_actual_values = actual[i].values_at(*expected_row.keys)
+          values_match?(expected_row.values, selected_actual_values)
+        else
+          values_match?(expected_row, actual[i])
+        end
+      elsif expected_row.respond_to?(:each)
+        raise ArgumentError, "Can't use partially matcher with Arrays" if @partial_match
+        values_match?(expected_row, actual[i].values)
+      else
+        raise ArgumentError, 'Unknown type'
+      end
+    end
+  end
+
+  chain :partially do
+    @partial_match = true
+  end
+
+  diffable
+end

data/lib/rspec/hive/query_builder.rb

@@ -0,0 +1,83 @@
+require_relative 'query_builder/row_transformer'
+require_relative 'query_builder/null_strategy'
+require_relative 'query_builder/value_by_type_strategy'
+
+module RSpec
+  module Hive
+    class QueryBuilder
+      def initialize(schema, connection)
+        @schema = schema
+        @connection = connection
+        @partition_hash = {}
+        @rows = []
+        @stubbing = false
+      end
+
+      def partition(hash)
+        spawn.partition!(hash)
+      end
+
+      def partition!(partition)
+        partition_hash.merge!(partition)
+        self
+      end
+
+      def insert(*new_rows)
+        spawn.insert!(new_rows)
+      end
+
+      def insert!(new_rows)
+        rows.concat(new_rows)
+        self
+      end
+
+      def execute
+        if partition_hash.empty?
+          connection.load_into_table(schema, transformed_rows)
+        else
+          connection.load_into_table(schema, transformed_rows, partition_hash)
+        end
+      end
+
+      def with_stubbing
+        spawn.with_stubbing!
+      end
+
+      def with_stubbing!
+        self.stubbing = true
+        self
+      end
+
+      protected
+
+      attr_accessor :partition_hash, :connection, :rows, :stubbing
+
+      private
+
+      attr_reader :schema
+
+      def spawn
+        clone
+      end
+
+      def stubbing?
+        stubbing
+      end
+
+      def transformed_rows
+        transformer = RowTransformer.new(schema, missing_column_strategy)
+        rows.map do |row|
+          transformer.transform(row)
+        end
+      end
+
+      def missing_column_strategy
+        if stubbing?
+          ValueByTypeStrategy.new
+        else
+          NullStrategy.new
+        end
+      end
+    end
+  end
+end