rspec-hive 0.4.2 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3a4997a9ef4cee20af56cb4d4178f0418d0c17fd
-  data.tar.gz: 2032bb8993693f75ddeb69cdb9c6d84e114cc32a
+  metadata.gz: 9242dae1f3ad9750670856055d1b4674e69225d8
+  data.tar.gz: a04e81680d2a346cded0bf3f827695037531b010
 SHA512:
-  metadata.gz: da255d96802490d15ef57fdc6745850d0daf80499bfc9a250c16f24f745c0090c8cdc0d3f17ac593e7b406e109d8715aa55244c4421da16d028ad2bd4c26c543
-  data.tar.gz: 6528c7ce3af5771b2ca17d84bf560608245bb0d7fbc5087a430bd66d474809fa85b0808749a3b3d06dd5247f77ae5423fb3c9c9c81fff2007637a6847a78597c
+  metadata.gz: bcec72034eb383894133d8fb5dc147e57ea6a3e799982ac4cae9ad55f84dae0b31bcf16f0ed48c5e6194f31a9d72273e9147b0028a84155cb77758386389559e
+  data.tar.gz: 463964b574467f5e8e33c8eaf6c35318a89e411c8879868a61259e7e7bcbcf9e4efce2cdd0d53a5071d367450b59deed2105907b9d26d6d4297e3c54b35e0483

data/.gitignore

@@ -15,3 +15,4 @@
 *.o
 *.a
 mkmf.log
+/config/*

data/.rubocop.yml

@@ -1,10 +1,12 @@
-inherit_from: .rubocop_u2i.yml
+require: rubocop-rspec
+
+inherit_from: .rubocop_todo.yml
 
 Rails:
   Enabled: false
 
 AllCops:
-  TargetRubyVersion: 2.1
+  TargetRubyVersion: 2.3
   Include:
     - '**/Rakefile'
   Exclude:
@@ -15,9 +17,41 @@ AllCops:
 Metrics/LineLength:
   Max: 120
   Enabled: true
-  Exclude:
-    - spec/**/*
-    - examples/spec/**/*
 
 Style/SignalException:
   EnforcedStyle: only_raise
+
+Naming/FileName:
+  Exclude:
+    - Gemfile
+    - Guardfile
+    - rspec-hive.gemspec
+
+Metrics/BlockLength:
+  ExcludedMethods: ['describe', 'context']
+
+Layout/MultilineOperationIndentation:
+  EnforcedStyle: indented
+
+Style/Documentation:
+  Enabled: false
+
+Layout/SpaceInsideHashLiteralBraces:
+  EnforcedStyle: no_space
+  EnforcedStyleForEmptyBraces: no_space
+
+Layout/DotPosition:
+  EnforcedStyle: trailing
+  Enabled: true
+
+Metrics/MethodLength:
+  CountComments: false  # count full line comments?
+  Max: 30
+
+RSpec/DescribeClass:
+  Exclude:
+    - spec/lib/rspec/hive/matchers_spec.rb
+
+Style/Encoding:
+  Exclude:
+    - 'rspec-hive.gemspec'

data/.rubocop_todo.yml

@@ -0,0 +1,40 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2017-10-18 17:06:59 +0200 using RuboCop version 0.50.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 6
+# Configuration parameters: CountComments, ExcludedMethods.
+Metrics/BlockLength:
+  Max: 106
+
+# Offense count: 44
+RSpec/ExpectInHook:
+  Exclude:
+    - 'spec/lib/rspec/hive/connection_delegator_spec.rb'
+    - 'spec/lib/rspec/hive/connector_spec.rb'
+    - 'spec/lib/rspec/hive/query_builder_spec.rb'
+
+# Offense count: 46
+# Configuration parameters: SupportedStyles.
+# SupportedStyles: have_received, receive
+RSpec/MessageSpies:
+  EnforcedStyle: receive
+
+# Offense count: 36
+# Configuration parameters: Max.
+RSpec/NestedGroups:
+  Exclude:
+    - 'spec/lib/rspec/hive/matchers_spec.rb'
+    - 'spec/lib/rspec/hive/query_builder_spec.rb'
+
+# Offense count: 11
+# Configuration parameters: IgnoreSymbolicNames.
+RSpec/VerifiedDoubles:
+  Exclude:
+    - 'spec/lib/rspec/hive/connection_delegator_spec.rb'
+    - 'spec/lib/rspec/hive/connector_spec.rb'
+    - 'spec/lib/rspec/hive/query_builder_helper_spec.rb'

data/.ruby-version

@@ -1 +1 @@
-ruby-2.1.5
+ruby-2.3.3

data/.travis.yml

@@ -4,9 +4,8 @@ services:
   - docker
 language: ruby
 rvm:
-  - 2.1
-  - 2.2
-  - 2.3.0
+  - 2.3.3
+  - 2.4.1
 env: CONFIG_FILE_NAME=rspec-hive.yml PORT=10000 CONFIG_FILE=rspec-hive.yml
 
 before_script:
@@ -17,3 +16,5 @@ script:
   - bundle exec rake spec
   - bundle exec rake rubocop
   - bundle exec rake hive_spec
+
+bundler_args: --without development

data/Gemfile

@@ -1,3 +1,21 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 gemspec
+
+group :test do
+  gem 'bundler', '~> 1.15'
+  gem 'codeclimate-test-reporter', '~> 0.4'
+  gem 'rspec-its', '~> 1.2'
+  gem 'rubocop', '~> 0.50'
+  gem 'rubocop-rspec', '~> 1.18'
+end
+
+group :development do
+  gem 'guard'
+  gem 'guard-rspec'
+  gem 'guard-rubocop'
+  gem 'pry'
+  gem 'pry-byebug'
+end

data/Guardfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # A sample Guardfile
 # More info at https://github.com/guard/guard#readme
 
@@ -13,14 +15,16 @@ group :red_green_refactor, halt_on_fail: true do
     watch('spec/spec_helper.rb') { 'spec' }
   end
 
-  guard :rspec, cmd: 'bundle exec rspec', spec_paths: ['examples/spec'] do
-    watch(%r{^examples/spec/.+_spec\.rb$})
-    watch(%r{^examples/lib/(.+)\.rb$}) { |m| "examples/spec/#{m[1]}_spec.rb" }
-    watch('examples/spec/spec_helper.rb') { 'examples/spec' }
-  end if system(detect_docker)
+  if system(detect_docker)
+    guard :rspec, cmd: 'bundle exec rspec', spec_paths: ['examples/spec'] do
+      watch(%r{^examples/spec/.+_spec\.rb$})
+      watch(%r{^examples/lib/(.+)\.rb$}) { |m| "examples/spec/#{m[1]}_spec.rb" }
+      watch('examples/spec/spec_helper.rb') { 'examples/spec' }
+    end
+  end
 
   guard :rubocop, all_on_start: false do
-    watch(%r{.+\.rb$})
+    watch(/.+\.rb$/)
     watch(%r{(?:.+/)?\.rubocop\.yml$}) { |m| File.dirname(m[0]) }
   end
 end

data/README.md

@@ -47,6 +47,8 @@ You can specify following arguments:
 * CONFIG_FILE_DIR - directory where to put generated config file
 * CONFIG_FILE_NAME - name of the config file that will be generated
 
+Watch out: in some cases Rspec hive may look for config files in e.g. you're current/root directory, not in config/ folder. Make sure you've got correct config in the directory used by the gem.
+
 #### Installing Docker
 Detailed instruction may be found on https://docs.docker.com/engine/installation.
 
@@ -112,10 +114,23 @@ Where we invoke:
 
     RSpec::Hive.configure(File.join(__dir__, '/config.yml'))
 
+### Loading udfs
+
+`bundle exec rake "spec:hive:docker:load_udfs[path_to_udf_on_s3]"`
+
+By default udfs will be loaded to `docker_shared_directory_path`.
+
+
 ## Note
 
 Please remember docker does not remove containers automatically, use `docker ps -a` to list all unused containers.
 
+## Changelog
+
+0.5.0: 
+  > Updated dockerfile to build container with hive 2.1.1 on Hadoop 2.7.3.
+  > Moved hive config to config file, default hive config is no longer provided, except for default config file generated using rake task.  
+
 ## Contributing
 
 1. Fork it ( https://github.com/[my-github-username]/rspec-hive/fork )

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'bundler/gem_tasks'
 load 'lib/rspec/hive/rake_tasks/docker.rake'
 
@@ -26,7 +28,7 @@ detect_docker = <<-BASH
 BASH
 
 if system(detect_docker)
-  task default: [:spec, :hive_spec, :rubocop]
+  task default: %i[spec hive_spec rubocop]
 else
-  task default: [:spec, :rubocop]
+  task default: %i[spec rubocop]
 end

data/docker/Dockerfile

@@ -1,47 +1,38 @@
-FROM ubuntu:latest
-RUN apt-get update \
-  && apt-get install -y wget \
-   && rm -rf /var/lib/apt/lists/*
-
-RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true |  /usr/bin/debconf-set-selections
-
-RUN apt-get update
-RUN apt-get install -y software-properties-common
-RUN add-apt-repository ppa:webupd8team/java
-
-RUN apt-get update
-RUN apt-get install -y oracle-java8-installer
-RUN apt-get install -y oracle-java8-set-default
-
-ENV SHARED_FOLDER /tmp/spec-tmp-files/
-ENV TMP_ROOT /tmp/root/
-ENV WAREHOUSE /tmp/warehouse/
-
-ENV HADOOP_BIN $HOME/hadoop-bin/hadoop-1.2.1/
-ENV HIVE_BIN $HOME/hive-bin/hive-0.11/
-
-ENV HADOOP_HOME $HOME/hadoop/
-ENV HIVE_HOME $HOME/hive/
-
-ENV JAVA_HOME /usr/
-
-RUN mkdir -p $HADOOP_BIN \
-    && mkdir -p $HIVE_BIN \
-    && mkdir -p $HADOOP_HOME \
-    && mkdir -p $HIVE_HOME \
-    && mkdir -p $SHARED_FOLDER \
-    && mkdir -p $TMP_ROOT \
-    && mkdir -p $WAREHOUSE
-
-RUN wget https://www.apache.org/dist/hadoop/core/hadoop-1.2.1/hadoop-1.2.1-bin.tar.gz -O $HADOOP_BIN/hadoop.tar.gz
-RUN wget https://archive.apache.org/dist/hive/hive-0.11.0/hive-0.11.0-bin.tar.gz -O $HIVE_BIN/hive.tar.gz
-
-RUN tar -xzf $HADOOP_BIN/hadoop.tar.gz --strip-components=1 -C $HADOOP_HOME
-RUN tar -xzf $HIVE_BIN/hive.tar.gz --strip-components=1 -C $HIVE_HOME
-
-CMD $HIVE_HOME/bin/hive \
-        --service hiveserver2 \
-        --hiveconf hive.server2.enable.doAs=false \
-        --hiveconf hive.server2.thrift.port=10000 \
-        --hiveconf hive.root.logger=INFO,console  \
-        --hiveconf hive.metastore.warehouse.dir=$WAREHOUSE
+FROM ubuntu:16.04
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get -qq update && \
+  apt-get -qq install -y wget software-properties-common && \
+  add-apt-repository ppa:webupd8team/java && \
+  apt-get -qq update && \
+  (echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true |  /usr/bin/debconf-set-selections) && \
+  apt-get -qq install -y oracle-java8-installer oracle-java8-set-default
+
+ENV INSTALL_DIR=/opt
+  SHARED_FOLDER=/tmp/spec-tmp-files \
+  TMP_HOME=/tmp/home \
+  WAREHOUSE=/tmp/warehouse \
+  HADOOP_HOME=$INSTALL_DIR/hadoop \
+  HIVE_HOME=$INSTALL_DIR/hive \
+  JAVA_HOME=/usr
+
+RUN mkdir -p $HADOOP_HOME $HIVE_HOME $SHARED_FOLDER $TMP_HOME $WAREHOUSE
+
+RUN wget http://www-eu.apache.org/dist/hadoop/common/hadoop-2.7.3/hadoop-2.7.3.tar.gz --quiet -O /tmp/hadoop.tar.gz && \
+  wget http://www-eu.apache.org/dist/hive/hive-2.1.1/apache-hive-2.1.1-bin.tar.gz --quiet -O /tmp/hive.tar.gz && \
+  tar -xzf /tmp/hadoop.tar.gz --strip-components=1 -C $HADOOP_HOME && \
+  tar -xzf /tmp/hive.tar.gz --strip-components=1 -C $HIVE_HOME
+
+RUN rm -rf $HIVE_HOME/metastore_db/
+RUN $HIVE_HOME/bin/schematool -initSchema -dbType derby
+
+ADD ./hive-site.xml $HIVE_HOME/conf/hive-site.xml
+
+WORKDIR $TMP_HOME
+ENTRYPOINT $HIVE_HOME/bin/hive \
+          --service hiveserver2 \
+          --hiveconf hive.server2.enable.doAs=false \
+          --hiveconf hive.server2.thrift.port=10000 \
+          --hiveconf hive.root.logger=INFO,console  \
+          --hiveconf hive.metastore.warehouse.dir=$WAREHOUSE

data/docker/hive-site.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?><!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<configuration>
+  <property>
+    <name>hive.resultset.use.unique.column.names</name>
+    <value>false</value>
+    <description>
+      Make column names unique in the result set by qualifying column names with table alias if needed.
+      Table alias will be added to column names for queries of type "select *" or
+      if query explicitly uses table alias "select r1.x..".
+    </description>
+  </property>
+</configuration>

data/examples/lib/query.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rbhive'
 
 class Query

data/examples/rspec-hive.yml.example

@@ -3,4 +3,10 @@ hive:
   port: 10000
   host_shared_directory_path: /Users/Shared/tmp/spec-tmp-files
   docker_shared_directory_path: /tmp/spec-tmp-files
-  hive_version: 10
+  hive_version: 10
+  hive_options:
+    'hive.exec.dynamic.partition': 'true'
+    'hive.exec.dynamic.partition.mode': 'nonstrict'
+    'hive.exec.max.dynamic.partitions.pernode': '100000'
+    'hive.exec.max.dynamic.partitions': '100000'
+    'mapred.child.java.opts': '-Xmx2048m'

data/examples/spec/query_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'spec_helper'
 require_relative '../lib/query'
 

data/examples/spec/spec_helper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 if ENV['COVERAGE']
   require 'simplecov'
   SimpleCov.start

data/lib/rspec/hive.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rspec/hive/version'
 require 'rspec/hive/db_name'
 require 'rspec/hive/configuration'

data/lib/rspec/hive/configuration.rb

@@ -1,8 +1,13 @@
+# frozen_string_literal: true
+
 require 'tmpdir'
 
 module RSpec
   module Hive
     class Configuration
+      DEFAULT_VERSION = 10
+      DEFAULT_TIMEOUT = 120
+
       attr_accessor :host,
                     :port,
                     :host_shared_directory_path,
@@ -17,7 +22,7 @@ module RSpec
           load_default_variables
         else
           interpolated = ERB.new(File.read(path_to_config_file)).result
-          config = YAML.load(interpolated)['hive']
+          config = YAML.safe_load(interpolated)['hive']
           load_variables_from_config(config)
         end
         @logger = Logger.new(STDOUT)
@@ -26,13 +31,13 @@ module RSpec
       private
 
       def load_default_variables
-        @host = platform_specific_host
-        @port = 10000
+        @host = '127.0.0.1'
+        @port = 10_000
         @host_shared_directory_path = platform_specific_host_shared_dir_path
         @docker_shared_directory_path = '/tmp/spec-tmp-files'
-        @hive_version = default_version
-        @connection_timeout = default_timeout
-        @hive_options = default_hive_options
+        @hive_version = DEFAULT_VERSION
+        @connection_timeout = DEFAULT_TIMEOUT
+        @hive_options = {}
       end
 
       def load_variables_from_config(config)
@@ -40,13 +45,9 @@ module RSpec
         @port = config['port']
         @host_shared_directory_path = config['host_shared_directory_path']
         @docker_shared_directory_path = config['docker_shared_directory_path']
-        @hive_version = (config['hive_version'] || default_version).to_i
-        @connection_timeout = (config['timeout'] || default_timeout).to_i
-        @hive_options = merge_config_options(default_hive_options, config)
-      end
-
-      def merge_config_options(hash, config)
-        hash.merge(config['hive_options'].to_h)
+        @hive_version = (config['hive_version'] || DEFAULT_VERSION).to_i
+        @connection_timeout = (config['timeout'] || DEFAULT_TIMEOUT).to_i
+        @hive_options = config['hive_options'].to_h
       end
 
       def mac?
@@ -54,10 +55,6 @@ module RSpec
         host_os =~ /darwin|mac os/
       end
 
-      def platform_specific_host
-        mac? ? '192.168.99.100' : '127.0.0.1'
-      end
-
       def platform_specific_host_shared_dir_path
         if mac?
           File.join(Dir.mktmpdir(nil, '/Users/Shared'), 'spec-tmp-files')
@@ -65,22 +62,6 @@ module RSpec
           File.join(Dir.mktmpdir, 'spec-tmp-files')
         end
       end
-
-      def default_timeout
-        1800
-      end
-
-      def default_version
-        10
-      end
-
-      def default_hive_options
-        {'hive.exec.dynamic.partition' => 'true',
-         'hive.exec.dynamic.partition.mode' => 'nonstrict',
-         'hive.exec.max.dynamic.partitions.pernodexi' => '100000',
-         'hive.exec.max.dynamic.partitions' => '100000',
-         'mapred.child.java.opts' => '-Xmx2048m'}
-      end
     end
   end
 end