rspec-hive 0.1.0

data/spec/lib/rspec/hive/configuration_spec.rb

@@ -0,0 +1,167 @@
+require 'spec_helper'
+require 'tempfile'
+
+describe RSpec::Hive::Configuration do
+  RSpec.shared_examples('config') do
+    its(:host) do
+      is_expected.to eq(expected_host)
+    end
+
+    its(:port) do
+      is_expected.to eq(expected_port)
+    end
+
+    its(:host_shared_directory_path) do
+      is_expected.to match(expected_host_shared_directory_path)
+    end
+
+    its(:docker_shared_directory_path) do
+      is_expected.to eq(expected_docker_shared_directory_path)
+    end
+
+    its(:hive_version) do
+      is_expected.to eq(expected_hive_version)
+    end
+
+    its(:connection_timeout) do
+      is_expected.to eq(expected_timeout)
+    end
+
+    its(:hive_options) do
+      is_expected.to eq(expected_hive_options)
+    end
+  end
+
+  let(:expected_host_shared_directory_path) do
+    '/Users/Shared/tmp/spec-tmp-files'
+  end
+  let(:expected_docker_shared_directory_path) { '/tmp/spec-tmp-files' }
+  let(:expected_hive_version) { 10 }
+  let(:expected_timeout) { 1800 }
+  let(:expected_partition_flag) { 'true' }
+  let(:expected_partition_mode) { 'nonstrict' }
+  let(:expected_partiton_pernodexi) { '100000' }
+  let(:expected_partitions) { '100000' }
+  let(:expected_java_opts) { '-Xmx2048m' }
+  let(:expected_hive_options) do
+    {'hive.exec.dynamic.partition' => expected_partition_flag,
+     'hive.exec.dynamic.partition.mode' => expected_partition_mode,
+     'hive.exec.max.dynamic.partitions.pernodexi' => expected_partiton_pernodexi,
+     'hive.exec.max.dynamic.partitions' => expected_partitions,
+     'mapred.child.java.opts' => expected_java_opts}
+  end
+
+  context 'when no configuration file is provided' do
+    let(:expected_port) { 10000 }
+
+    before { allow(Dir).to receive(:mktmpdir) { mock_tmpdir } }
+
+    subject { described_class.new }
+
+    context 'when on Mac' do
+      let(:mock_tmpdir) { '/Users/Shared/test/' }
+      let(:expected_host) { '192.168.99.100' }
+      let(:expected_host_shared_directory_path) { '/Users/Shared/test/spec-tmp-files' }
+
+      before do
+        allow_any_instance_of(described_class).to receive(:mac?) { true }
+      end
+
+      include_examples('config')
+    end
+
+    context 'when on Linux' do
+      let(:mock_tmpdir) { '/tmp/test/' }
+      let(:expected_host) { '127.0.0.1' }
+      let(:expected_host_shared_directory_path) { '/tmp/test/spec-tmp-files' }
+
+      before do
+        allow_any_instance_of(described_class).to receive(:mac?) { false }
+      end
+
+      include_examples('config')
+    end
+  end
+
+  context 'when there is a configuration file' do
+    let(:path_to_config_file) do
+      Tempfile.open(%w(config .yml)) do |f|
+        f.write yaml_hash.to_yaml
+        f.path
+      end
+    end
+    let(:expected_host) { '127.0.0.2' }
+    let(:expected_port) { 10001 }
+
+    context 'where all parameters are present' do
+      let(:yaml_hash) do
+        {
+          'hive' =>
+            {
+              'host' => '127.0.0.2',
+              'port' => 10001,
+              'host_shared_directory_path' => expected_host_shared_directory_path,
+              'docker_shared_directory_path' => expected_docker_shared_directory_path,
+              'hive_version' => '10',
+              'timeout' => 1800
+            }
+        }
+      end
+
+      after { File.unlink(path_to_config_file) }
+
+      subject { described_class.new(path_to_config_file) }
+
+      include_examples('config')
+    end
+
+    context 'where there are only required parameters' do
+      let(:yaml_hash) do
+        {
+          'hive' =>
+            {
+              'host' => '127.0.0.2',
+              'port' => 10001,
+              'host_shared_directory_path' => expected_host_shared_directory_path,
+              'docker_shared_directory_path' => expected_docker_shared_directory_path
+            }
+        }
+      end
+      let(:expected_hive_version) { 10 }
+
+      after { File.unlink(path_to_config_file) }
+
+      subject { described_class.new(path_to_config_file) }
+
+      include_examples('config')
+    end
+
+    context 'where there are some parametres required and optional' do
+      let(:yaml_hash) do
+        {
+          'hive' =>
+            {
+              'host' => '127.0.0.2',
+              'port' => 10001,
+              'host_shared_directory_path' => expected_host_shared_directory_path,
+              'docker_shared_directory_path' => expected_docker_shared_directory_path,
+              'hive_version' => 11,
+              'timeout' => 60,
+              'hive_options' => {
+                'mapred.child.java.opts' => '-Xmx64m'
+              }
+            }
+        }
+      end
+      let(:expected_timeout) { 60 }
+      let(:expected_hive_version) { 11 }
+      let(:expected_java_opts) { '-Xmx64m' }
+
+      after { File.unlink(path_to_config_file) }
+
+      subject { described_class.new(path_to_config_file) }
+
+      include_examples('config')
+    end
+  end
+end

data/spec/lib/rspec/hive/connection_delegator_spec.rb

@@ -0,0 +1,311 @@
+require 'spec_helper'
+
+describe RSpec::Hive::ConnectionDelegator do
+  describe '#load_into_table' do
+    let(:host_shared_directory_path) { '/tmp/host' }
+    let(:docker_file_path) { '/tmp/docked/test_file' }
+    let(:config) do
+      double(
+        RSpec::Hive::Configuration,
+        host_shared_directory_path: host_shared_directory_path
+      )
+    end
+    let(:delimiter) { "\t" }
+    let(:table_name) { 'test_table' }
+    let(:table_schema) { instance_double(RBHive::TableSchema, name: table_name) }
+    let(:connection) { double('Connection') }
+    let(:file_mock) { double(Tempfile) }
+
+    let(:values) { ['a', 'b', 1] }
+
+    before do
+      table_schema.instance_variable_set(:@field_sep, delimiter)
+
+      expect(Tempfile).to receive(:open).
+        with(table_name, host_shared_directory_path).and_yield(file_mock)
+
+      expect(subject).to receive(:docker_path).
+        with(file_mock) { docker_file_path }
+
+      expect(subject).to receive(:write_values_to_file).
+        with(file_mock, values, "\t").once
+    end
+
+    context 'without partitions' do
+      before do
+        expect(subject).to receive(:load_file_to_hive_table).
+          with(table_name, docker_file_path, nil).once
+
+        expect(subject).not_to receive(:partition_clause)
+      end
+
+      subject { described_class.new(connection, config) }
+
+      it do
+        subject.load_into_table(table_schema, values)
+      end
+    end
+
+    context 'with partitions' do
+      let(:partitions) { {day: '20160101', hm: '2020'} }
+      let(:partition_query) { "PARTITION(day='20160101',hm='2020')" }
+      before do
+        expect(subject).to receive(:load_file_to_hive_table).
+          with(table_name, docker_file_path, partition_query).once
+        expect(subject).to receive(:partition_clause).
+          with(partitions) { partition_query }
+      end
+
+      subject { described_class.new(connection, config) }
+
+      it do
+        subject.load_into_table(table_schema, values, partitions)
+      end
+    end
+  end
+
+  describe '#load_partition' do
+    let(:config) { double('Config') }
+    let(:connection) { double('Connection') }
+
+    let(:table_name) { 'test_table' }
+    let(:partitions) do
+      [{dth: 'mon', country: 'us'}, {dth: 'tue', country: 'us'}]
+    end
+    let(:partition_query) do
+      "PARTITION(dth='mon',country='us') PARTITION(dth='tue',country='us')"
+    end
+
+    let(:executed_query) do
+      "ALTER TABLE test_table ADD PARTITION(dth='mon',country='us') PARTITION(dth='tue',country='us')"
+    end
+
+    before do
+      expect(subject).to receive(:partition_clause).
+        with(partitions) { partition_query }
+      expect(connection).to receive(:execute).with(executed_query)
+    end
+
+    subject { described_class.new(connection, config) }
+
+    it do
+      subject.load_partitions(table_name, partitions)
+    end
+  end
+
+  describe '#partition_clause' do
+    let(:config) { double('Config') }
+    let(:connection) { double('Connection') }
+
+    context 'with single partition' do
+      let(:partitions) { {day: '20160101', hm: '2020'} }
+      let(:partition_query) { "PARTITION(day='20160101',hm='2020')" }
+
+      subject { described_class.new(connection, config) }
+
+      it 'translates partition hash to single query' do
+        expect(subject.send(:partition_clause, partitions)).to eq(partition_query)
+      end
+    end
+
+    context 'with multiple partitions' do
+      let(:partitions) { [{day: 'mon', hm: '2020'}, {day: 'tue', hm: '2020'}, {day: 'mon', hm: '2030'}] }
+      let(:partition_query) do
+        "PARTITION(day='mon',hm='2020') PARTITION(day='tue',hm='2020') PARTITION(day='mon',hm='2030')"
+      end
+
+      subject { described_class.new(connection, config) }
+
+      it 'translates partition hash to combined query' do
+        expect(subject.send(:partition_clause, partitions)).to eq(partition_query)
+      end
+    end
+  end
+
+  describe '#write_values_to_file' do
+    let(:file) { StringIO.new }
+    let(:values) do
+      [['a', 'b', 1],
+       ['aa', 'bb', 22]]
+    end
+    let(:connection) { double('Connection') }
+    let(:config) { double('Config') }
+    let(:delimiter) { '|' }
+    let(:expected_file_content) { "a|b|1\naa|bb|22\n" }
+
+    subject { described_class.new(connection, config) }
+    it 'writes values to file in correct format' do
+      subject.send(:write_values_to_file, file, values, delimiter)
+      file.rewind
+      expect(file.read).to eq(expected_file_content)
+    end
+  end
+
+  describe '#load_file_to_hive_table' do
+    let(:connection) { double('Connection') }
+    let(:config) { double('Config') }
+    let(:table_name) { 'test_table' }
+    let(:file_path) { '/tmp/test' }
+    let(:execute_text) do
+      "load data local inpath '/tmp/test' into table test_table"
+    end
+
+    before do
+      expect(connection).to receive(:execute).with(execute_text)
+    end
+
+    subject { described_class.new(connection, config) }
+
+    it do
+      subject.send(:load_file_to_hive_table, table_name, file_path)
+    end
+  end
+
+  describe '#translate_to_docker_path' do
+    let(:file_mock) { double(File) }
+    let(:file_name) { 'testfile' }
+    let(:file_host_path) { '/tmp/host/testfile' }
+    let(:expected_file_path) { '/tmp/docker/testfile' }
+
+    let(:connection) { double('Connection') }
+    let(:docker_shared_directory_path) { '/tmp/docker' }
+    let(:config) do
+      double(
+        RSpec::Hive::Configuration,
+        docker_shared_directory_path: docker_shared_directory_path
+      )
+    end
+
+    before do
+      expect(file_mock).to receive(:path) { file_host_path }
+    end
+
+    subject { described_class.new(connection, config) }
+
+    it do
+      expect(subject.send(:docker_path, file_mock)).
+        to eq(expected_file_path)
+    end
+  end
+
+  describe '#show_tables' do
+    let(:connection) { double('Connection') }
+    let(:config) { double('Config') }
+    let(:fetch_text) { 'SHOW TABLES' }
+
+    before do
+      expect(connection).to receive(:fetch).with(fetch_text)
+    end
+
+    subject { described_class.new(connection, config) }
+
+    it do
+      subject.show_tables
+    end
+  end
+
+  describe '#create_database' do
+    let(:connection) { double('Connection') }
+    let(:config) { double('Config') }
+    let(:db_name) { 'test' }
+    let(:fetch_text) { 'CREATE DATABASE IF NOT EXISTS `test`' }
+
+    before do
+      expect(connection).to receive(:execute).with(fetch_text)
+    end
+
+    subject { described_class.new(connection, config) }
+
+    it do
+      subject.create_database(db_name)
+    end
+  end
+
+  describe '#create_table' do
+    let(:connection) { double('Connection') }
+    let(:config) { double('Config') }
+    let(:table_schema) { double('Table_schema') }
+    let(:table_statement) { 'I AM TABLE STATEMENT' }
+
+    before do
+      expect(table_schema).to receive(:dup) { table_schema }
+      expect(table_schema).to receive(:instance_variable_set).with(:@location, nil)
+      expect(table_schema).to receive(:create_table_statement) { table_statement }
+      expect(connection).to receive(:execute).with(table_statement)
+    end
+
+    subject { described_class.new(connection, config) }
+
+    it do
+      subject.create_table(table_schema)
+    end
+  end
+
+  describe '#use databaes' do
+    let(:connection) { double('Connection') }
+    let(:config) { double('Config') }
+    let(:db_name) { 'test' }
+    let(:fetch_text) { 'USE `test`' }
+
+    before do
+      expect(connection).to receive(:execute).with(fetch_text)
+    end
+
+    subject { described_class.new(connection, config) }
+
+    it do
+      subject.use_database(db_name)
+    end
+  end
+
+  describe '#drop_databse' do
+    let(:connection) { double('Connection') }
+    let(:config) { double('Config') }
+    let(:db_name) { 'test' }
+    let(:fetch_text) { 'DROP DATABASE `test`' }
+
+    before do
+      expect(connection).to receive(:execute).with(fetch_text)
+    end
+
+    subject { described_class.new(connection, config) }
+
+    it do
+      subject.drop_database(db_name)
+    end
+  end
+
+  describe '#show_databases' do
+    let(:connection) { double('Connection') }
+    let(:config) { double('Config') }
+    let(:fetch_text) { 'SHOW DATABASES' }
+
+    before do
+      expect(connection).to receive(:fetch).with(fetch_text)
+    end
+
+    subject { described_class.new(connection, config) }
+
+    it do
+      subject.show_databases
+    end
+  end
+
+  describe '#switch database' do
+    let(:connection) { double('Connection') }
+    let(:config) { double('Config') }
+
+    let(:db_name) { 'test_db' }
+
+    before do
+      expect(subject).to receive(:create_database).once
+      expect(subject).to receive(:use_database).once
+    end
+
+    subject { described_class.new(connection, config) }
+
+    it do
+      subject.switch_database(db_name)
+    end
+  end
+end