rspec-hive 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ca84700fd2dd11920a2cf224d634f444a9e325b9
+  data.tar.gz: ac29aa0c169716541e7df32d8ddcfed8ed8a005e
+SHA512:
+  metadata.gz: 5ec841bc2886f78ff101a18481ff622abf64c4ae5abb387441fab204d77327c805c2181f94201e850911cdb248ca3a06c3f1f097074abfcf70438eb2730626ac
+  data.tar.gz: 4cc12280d1c740e9c8b349995def32422796ac4c96bd7061d2d13ef936bc1fe3fbeea78eb138807e2c1c4751b1de26eb171927ad5c14c2ea23c63493acbdb618

data/.codeclimate.yml

@@ -0,0 +1,14 @@
+---
+engines:
+  bundler-audit:
+    enabled: true
+  fixme:
+    enabled: true
+  rubocop:
+    enabled: true
+ratings:
+  paths:
+  - Gemfile.lock
+  - "**.rb"
+exclude_paths:
+- spec/**/*

data/.gitignore

@@ -0,0 +1,17 @@
+.idea
+*.iml
+
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,11 @@
+inherit_from: .rubocop_u2i.yml
+
+AllCops:
+  Exclude:
+    - spec/spec_helper.rb
+
+Metrics/LineLength:
+  Max: 80
+  Enabled: true
+  Exclude:
+    - spec/**/*

data/.rubocop_u2i.yml

@@ -0,0 +1,55 @@
+require: rubocop-rspec
+
+AllCops:
+  RunRailsCops: true
+  Include:
+    - '**/Rakefile'
+    - config.ru
+  Exclude:
+    - bin/**/*
+    - db/**/*
+    - config/**/*
+    - script/**/*
+    - db/schema.rb
+
+Style/MultilineOperationIndentation:
+  EnforcedStyle: indented
+
+Style/Documentation:
+  Enabled: false
+
+Style/WhileUntilModifier:
+  MaxLineLength: 120
+  Exclude:
+    - spec/**/*
+
+Style/SpaceInsideHashLiteralBraces:
+  EnforcedStyle: no_space
+  EnforcedStyleForEmptyBraces: no_space
+
+Style/MultilineBlockChain:
+  Enabled: true
+
+Style/DotPosition:
+  EnforcedStyle: trailing
+  Enabled: true
+
+Style/SingleLineMethods:
+  AllowIfMethodIsEmpty: false
+
+Style/NumericLiterals:
+  Enabled: false
+
+Metrics/LineLength:
+  Max: 120
+  Enabled: true
+  Exclude:
+    - spec/**/*
+
+Metrics/MethodLength:
+  CountComments: false  # count full line comments?
+  Max: 30
+
+RSpec/FilePath:
+  Exclude:
+    - spec/routing/**/*_routing_spec.rb

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.1.5

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Guardfile

@@ -0,0 +1,15 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+group :red_green_refactor, halt_on_fail: true do
+  guard :rspec, cmd: 'bundle exec rspec' do
+    watch(%r{^spec/.+_spec\.rb$})
+    watch(%r{^lib/(.+)\.rb$}) { |m| "spec/lib/#{m[1]}_spec.rb" }
+    watch('spec/spec_helper.rb') { 'spec' }
+  end
+
+  guard :rubocop do
+    watch(%r{.+\.rb$})
+    watch(%r{(?:.+/)?\.rubocop\.yml$}) { |m| File.dirname(m[0]) }
+  end
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Wojtek Mielczarek
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,123 @@
+[![Code Climate](https://codeclimate.com/repos/567b03d7bd3f3b2512002248/badges/85abbc07acb75f664185/gpa.svg)](https://codeclimate.com/repos/567b03d7bd3f3b2512002248/feed)
+[![Test Coverage](https://codeclimate.com/repos/567b03d7bd3f3b2512002248/badges/85abbc07acb75f664185/coverage.svg)](https://codeclimate.com/repos/567b03d7bd3f3b2512002248/coverage)
+[![Issue Count](https://codeclimate.com/repos/567b03d7bd3f3b2512002248/badges/85abbc07acb75f664185/issue_count.svg)](https://codeclimate.com/repos/567b03d7bd3f3b2512002248/feed)
+
+# rpsec-hive
+
+rspec-hive is a utility gem to help you write beautiful rspec tests for hive queries. The idea is simple - you just launch a docker machine with hadoop and hive installed. To test a query you create a simple RSpec file and extend it with `RSpec::Hive::WithHiveConnection`.
+
+We have prepared a few simple rake tasks that will let you create sample config file, download correct docker image and run docker container with proper parameters.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'rspec-hive'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install rspec-hive
+
+### Configuring tests
+
+#### Config file
+
+To run tests on docker you will need a configurtion file that will let you put up a docker container and maintain your connection to this container from tests. You can do this manually and provide just a path to file, but we have also prepared special rake tasks to help you out. Try running:
+
+    $ rake spec:hive:config:generate_default
+
+It will create `rspec-hive.yml` in your current directory. You can of course pass some parameters to this rake task doing something like:
+
+    $ rake spec:hive:config:generate_default HOST=127.0.0.1 PORT=5032
+
+You can specify following arguments:
+* HOST - ip of docker container
+* PORT - port to connect to docker
+* HOST_SHARED_DIR - directory on your local machine that docker will share
+* DOCKER_SHARED_DIR - directory on your docker container that will be shared with your local machine
+* HIVE_VERSION - version of hive
+* CONFIG_FILE_DIR - directory where to put generated config file
+* CONFIG_FILE_NAME - name of the config file that will be generated
+
+#### Installing Docker
+Detailed instruction may be found on https://docs.docker.com/engine/installation.
+
+Once docker is sucessfully installed on your machine you can verify if it works by using `docker` command.
+In case of error such as `Cannot connect to the Docker daemon. Is the docker daemon running on this host?` make sure you added your user to the docker group, you can do this using `sudo usermod -aG docker username` on Linux or `eval "$(docker-machine env default)"` on OSX.
+
+On Linux you can run the docker daemon by using:
+`sudo docker daemon -D -g /mnt`
+
+#### Docker image
+Once you have generated a config file you should download to your local machine proper docker image. You can create your own docker image. However if you would like to use ours just run:
+
+    $ rake spec:hive:docker:download_image
+
+It will download `nielsensocial/hive` from [dockerhub](https://hub.docker.com/r/nielsensocial/hive/).
+You can change Docker's storage base directory (where container and images go) using the -goption when starting the Docker daemon.
+If you have another image you can also use this rake task and provide special argument:
+* DOCKER_IMAGE_NAME - image name that should be pulled
+
+
+#### Running docker container
+You should now be ready to run your docker container. To do this run:
+
+    $ rake spec:hive:docker:run
+
+This command will run docker container using default config `rspec-hive.yml` and default docker image `nielsensocial/hive`. You can pass arguments like:
+* CONFIG_FILE - name of config file to use
+* DOCKER_IMAGE_NAME - docker image to use
+
+You are ready now to run your tests.
+
+#### Docker utils
+
+To check container id
+
+`$ docker ps`
+
+To attach to output of hive
+
+`$ docker attach <docker-container-id>`
+
+To run bash terminal on docker
+
+`$ docker exec -it <docker-container-id> bash`
+
+#### Hive utils
+
+When you are on hive and you have set up `JAVA_HOME` and `HADOOP_HOME` directories you might find usefull the tool named beeline. It should be present in your hive directory in `bin` folder (if you are using ours `nielsensocial/hive` when you run bash terminal on docker container this directory could be entered by calling `cd $HIVE_HOME/bin`). There you can run:
+
+    $ ./beeline
+
+And in the presented console connect by jdbc to hive:
+
+    beeline> !connect jdbc:hive2://localhost:10000 org.apache.hive.jdbc.HiveDriver
+
+## Usage
+
+In `examples/` directory we have prepared a simple query. It is available in `query_spec.rb` file. Notice how we configure `rspec-hive` by using:
+
+    require_relative 'config_helper'
+
+Where we invoke:
+
+    RSpec::Hive.configure(File.join(__dir__, '/config.yml'))
+
+## Note
+
+Please remember docker does not remove containers automatically, use `docker ps -a` to list all unused containers.
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/rspec-hive/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,23 @@
+require 'bundler/gem_tasks'
+load 'lib/rspec/rake_tasks/docker.rake'
+
+begin
+  require 'rspec/core/rake_task'
+
+  RSpec::Core::RakeTask.new(:spec)
+
+  RSpec::Core::RakeTask.new(:hive_spec) do |t|
+    t.pattern = 'examples/**/*_spec.rb'
+  end
+rescue LoadError
+  puts 'Cannot load rspec Rake tasks'
+end
+
+begin
+  require 'rubocop/rake_task'
+  RuboCop::RakeTask.new(:rubocop)
+rescue LoadError
+  puts 'Cannot load RuboCop Rake tasks'
+end
+
+task default: [:spec, :rubocop]

data/docker/Dockerfile

@@ -0,0 +1,47 @@
+FROM ubuntu:latest
+RUN apt-get update \
+  && apt-get install -y wget \
+   && rm -rf /var/lib/apt/lists/*
+
+RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true |  /usr/bin/debconf-set-selections
+
+RUN apt-get update
+RUN apt-get install -y software-properties-common
+RUN add-apt-repository ppa:webupd8team/java
+
+RUN apt-get update
+RUN apt-get install -y oracle-java8-installer
+RUN apt-get install -y oracle-java8-set-default
+
+ENV SHARED_FOLDER /tmp/spec-tmp-files/
+ENV TMP_ROOT /tmp/root/
+ENV WAREHOUSE /tmp/warehouse/
+
+ENV HADOOP_BIN $HOME/hadoop-bin/hadoop-1.2.1/
+ENV HIVE_BIN $HOME/hive-bin/hive-0.11/
+
+ENV HADOOP_HOME $HOME/hadoop/
+ENV HIVE_HOME $HOME/hive/
+
+ENV JAVA_HOME /usr/
+
+RUN mkdir -p $HADOOP_BIN \
+    && mkdir -p $HIVE_BIN \
+    && mkdir -p $HADOOP_HOME \
+    && mkdir -p $HIVE_HOME \
+    && mkdir -p $SHARED_FOLDER \
+    && mkdir -p $TMP_ROOT \
+    && mkdir -p $WAREHOUSE
+
+RUN wget https://www.apache.org/dist/hadoop/core/hadoop-1.2.1/hadoop-1.2.1-bin.tar.gz -O $HADOOP_BIN/hadoop.tar.gz
+RUN wget https://archive.apache.org/dist/hive/hive-0.11.0/hive-0.11.0-bin.tar.gz -O $HIVE_BIN/hive.tar.gz
+
+RUN tar -xzf $HADOOP_BIN/hadoop.tar.gz --strip-components=1 -C $HADOOP_HOME
+RUN tar -xzf $HIVE_BIN/hive.tar.gz --strip-components=1 -C $HIVE_HOME
+
+CMD $HIVE_HOME/bin/hive \
+        --service hiveserver2 \
+        --hiveconf hive.server2.enable.doAs=false \
+        --hiveconf hive.server2.thrift.port=10000 \
+        --hiveconf hive.root.logger=INFO,console  \
+        --hiveconf hive.metastore.warehouse.dir=$WAREHOUSE

data/examples/config_helper.rb

@@ -0,0 +1 @@
+RSpec::Hive.configure(File.join(__dir__, 'config.yml'))

data/examples/hive_tests_config.yml.example

@@ -0,0 +1,6 @@
+hive:
+  host: 192.168.99.100
+  port: 10000
+  host_shared_directory_path: /Users/Shared/tmp/spec-tmp-files
+  docker_shared_directory_path: /tmp/spec-tmp-files
+  hive_version: 10

data/examples/query.rb

@@ -0,0 +1,18 @@
+class Query
+  def table_schema
+    RBHive::TableSchema.new('people', nil, line_sep: '\n', field_sep: ';') do
+      column :name, :string
+      column :address, :string
+      column :amount, :float
+    end
+  end
+
+  def table_name
+    table_schema.name
+  end
+
+  def run_hive_query(connection)
+    query = "SELECT * FROM `#{table_schema.name}` WHERE `amount` > 2.5"
+    connection.execute(query)
+  end
+end

data/examples/query_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+require_relative 'query'
+require_relative 'config_helper'
+
+describe Query do
+  include RSpec::Hive::WithHiveConnection
+
+  subject { described_class.new }
+
+  describe 'hive query' do
+    let(:input_data) do
+      [
+        ['Mikolaj', 'Cos', 1.23],
+        ['Wojtek', 'Cos', 3.76]
+      ]
+    end
+
+    before do
+      connection.execute(subject.table_schema.create_table_statement)
+      connection.load_into_table(subject.table_schema, input_data)
+    end
+
+    it 'query returns one row' do
+      query = "SELECT * FROM `#{subject.table_name}` WHERE amount > 3.2"
+      query_result = connection.fetch(query).first.values
+      expect(query_result).to contain_exactly(
+        a_string_matching('Wojtek'),
+        a_string_matching('Cos'),
+        a_string_matching(/3\.7.*/))
+    end
+
+    it 'query returns one row 2' do
+      query = "SELECT * FROM `#{subject.table_name}` WHERE amount < 3.2"
+      query_result = connection.fetch(query).first.values
+      expect(query_result).to contain_exactly(
+        a_string_matching('Mikolaj'),
+        a_string_matching('Cos'),
+        a_string_matching(/1\.2.*/))
+    end
+  end
+end

data/lib/rspec/hive.rb

@@ -0,0 +1,31 @@
+require 'rspec/hive/version'
+require 'rspec/hive/db_name'
+require 'rspec/hive/configuration'
+require 'rspec/hive/connection_delegator'
+require 'rspec/hive/connector'
+require 'rspec/hive/with_hive_connection'
+
+module RSpec
+  module Hive
+    attr_reader :configuration
+
+    def self.configure(file_name = nil)
+      @configuration = new_configuration(file_name)
+      yield(@configuration) if block_given?
+      @configuration
+    end
+
+    def self.connector
+      @configuration ||= Configuration.new
+      Connector.new(@configuration)
+    end
+
+    def self.new_configuration(file_name)
+      Configuration.new(file_name)
+    end
+
+    private_class_method :new_configuration
+  end
+end
+
+require 'rspec/rake_tasks/railtie' if defined?(Rails)