flare-up 0.1 → 0.2

data/spec/lib/flare_up/boot_spec.rb

@@ -0,0 +1,100 @@
+describe FlareUp::Boot do
+
+  describe '.boot' do
+    let(:copy_command) { instance_double('FlareUp::CopyCommand') }
+
+    context 'when there is an error' do
+      before do
+        expect(FlareUp::Boot).to receive(:create_copy_command).and_return(copy_command)
+        expect(copy_command).to receive(:execute).and_raise(copy_command_error)
+      end
+
+      context 'when there is a DataSourceError' do
+        let(:copy_command_error) { FlareUp::DataSourceError }
+        it 'should handle the error' do
+          expect(FlareUp::CLI).to receive(:bailout).with(1)
+          expect { FlareUp::Boot.boot({}) }.not_to raise_error
+        end
+      end
+
+      context 'when there is a OtherZoneBucketError' do
+        let(:copy_command_error) { FlareUp::OtherZoneBucketError }
+        it 'should handle the error' do
+          expect(FlareUp::CLI).to receive(:bailout).with(1)
+          expect { FlareUp::Boot.boot({}) }.not_to raise_error
+        end
+      end
+
+      context 'when there is a OtherZoneBucketError' do
+        let(:copy_command_error) { FlareUp::SyntaxError }
+        it 'should handle the error' do
+          expect(FlareUp::CLI).to receive(:bailout).with(1)
+          expect { FlareUp::Boot.boot({}) }.not_to raise_error
+        end
+      end
+
+    end
+
+  end
+
+  describe '.create_connection' do
+    let(:options) {
+      {
+        :redshift_endpoint => 'TEST_REDSHIFT_ENDPOINT',
+        :database => 'TEST_DATABASE',
+        :redshift_username => 'TEST_REDSHIFT_USERNAME',
+        :redshift_password => 'TEST_REDSHIFT_PASSWORD'
+      }
+    }
+
+    it 'should create a connection' do
+      connection = FlareUp::Boot.create_connection(options)
+      expect(connection.host).to eq('TEST_REDSHIFT_ENDPOINT')
+      expect(connection.dbname).to eq('TEST_DATABASE')
+      expect(connection.user).to eq('TEST_REDSHIFT_USERNAME')
+      expect(connection.password).to eq('TEST_REDSHIFT_PASSWORD')
+    end
+  end
+
+  describe '.create_copy_command' do
+
+    let(:options) {
+      {
+        :table => 'TEST_TABLE',
+        :data_source => 'TEST_DATA_SOURCE',
+        :aws_access_key => 'TEST_ACCESS_KEY',
+        :aws_secret_key => 'TEST_SECRET_KEY'
+      }
+    }
+
+    it 'should create a proper copy command' do
+      command = FlareUp::Boot.create_copy_command(options)
+      expect(command.table_name).to eq('TEST_TABLE')
+      expect(command.data_source).to eq('TEST_DATA_SOURCE')
+      expect(command.aws_access_key_id).to eq('TEST_ACCESS_KEY')
+      expect(command.aws_secret_access_key).to eq('TEST_SECRET_KEY')
+    end
+
+    context 'when columns are specified' do
+      before do
+        options.merge!(:column_list => ['c1'])
+      end
+      it 'should create a proper copy command' do
+        command = FlareUp::Boot.create_copy_command(options)
+        expect(command.columns).to eq(['c1'])
+      end
+    end
+
+    context 'when options are specified' do
+      before do
+        options.merge!(:copy_options => '_')
+      end
+      it 'should create a proper copy command' do
+        command = FlareUp::Boot.create_copy_command(options)
+        expect(command.options).to eq('_')
+      end
+    end
+
+  end
+
+end

data/spec/lib/flare_up/cli_spec.rb

@@ -0,0 +1,162 @@
+describe FlareUp::CLI do
+
+  let(:required_arguments) { %w(copy TEST_DATA_SOURCE TEST_REDSHIFT_ENDPOINT TEST_DATABASE TEST_TABLE) }
+  let(:required_options) do
+    {
+      :data_source => 'TEST_DATA_SOURCE',
+      :redshift_endpoint => 'TEST_REDSHIFT_ENDPOINT',
+      :database => 'TEST_DATABASE',
+      :table => 'TEST_TABLE',
+      :redshift_username => 'TEST_REDSHIFT_USERNAME',
+      :redshift_password => 'TEST_REDSHIFT_PASSWORD',
+      :aws_access_key => 'TEST_AWS_ACCESS_KEY',
+      :aws_secret_key => 'TEST_AWS_SECRET_KEY'
+    }
+  end
+
+  before do
+    allow(FlareUp::ENVWrap).to receive(:get).with('AWS_ACCESS_KEY_ID').and_return('TEST_AWS_ACCESS_KEY')
+    allow(FlareUp::ENVWrap).to receive(:get).with('AWS_SECRET_ACCESS_KEY').and_return('TEST_AWS_SECRET_KEY')
+    allow(FlareUp::ENVWrap).to receive(:get).with('REDSHIFT_USERNAME').and_return('TEST_REDSHIFT_USERNAME')
+    allow(FlareUp::ENVWrap).to receive(:get).with('REDSHIFT_PASSWORD').and_return('TEST_REDSHIFT_PASSWORD')
+    allow(FlareUp::Boot).to receive(:boot)
+  end
+
+  describe '#copy' do
+
+    context 'when no options are specified' do
+      it 'should boot with the proper options' do
+        expect(FlareUp::Boot).to receive(:boot).with(required_options)
+        FlareUp::CLI.start(required_arguments)
+      end
+    end
+
+    context 'when column ordering is specified' do
+      it 'should boot with the proper options' do
+        expect(FlareUp::Boot).to receive(:boot).with(required_options.merge(:column_list => %w(c1 c2 c3)))
+        FlareUp::CLI.start(required_arguments + %w(--column_list c1 c2 c3))
+      end
+    end
+
+    context 'when COPY options are specified' do
+      it 'should boot with the proper options' do
+        expect(FlareUp::Boot).to receive(:boot).with(required_options.merge(:copy_options => 'TEST_COPY_OPTIONS WITH A SPACE'))
+        FlareUp::CLI.start(required_arguments + ['--copy_options', 'TEST_COPY_OPTIONS WITH A SPACE'])
+      end
+    end
+
+    describe 'Redshift credentials' do
+
+      describe 'username' do
+        context 'when it is specified on the CLI' do
+          it 'should boot with the proper options' do
+            expect(FlareUp::Boot).to receive(:boot).with(required_options.merge(:redshift_username => 'CLI_VALUE'))
+            FlareUp::CLI.start(required_arguments + %w(--redshift_username CLI_VALUE))
+          end
+        end
+        context 'when it is not specified on the CLI' do
+          context 'when it is available via ENV' do
+            it 'should boot with the key from the environment' do
+              expect(FlareUp::Boot).to receive(:boot).with(required_options.merge(:redshift_username => 'TEST_REDSHIFT_USERNAME'))
+              FlareUp::CLI.start(required_arguments)
+            end
+          end
+          context 'when it is not available via ENV' do
+            before do
+              allow(FlareUp::ENVWrap).to receive(:get).with('REDSHIFT_USERNAME').and_return(nil)
+              expect(FlareUp::CLI).to receive(:bailout).with(1)
+            end
+            it 'should be an error' do
+              FlareUp::CLI.start(required_arguments)
+            end
+          end
+        end
+      end
+
+      describe 'password' do
+        context 'when it is specified on the CLI' do
+          it 'should boot with the proper options' do
+            expect(FlareUp::Boot).to receive(:boot).with(required_options.merge(:redshift_password => 'CLI_VALUE'))
+            FlareUp::CLI.start(required_arguments + %w(--redshift_password CLI_VALUE))
+          end
+        end
+        context 'when it is not specified on the CLI' do
+          context 'when it is available via ENV' do
+            it 'should boot with the key from the environment' do
+              expect(FlareUp::Boot).to receive(:boot).with(required_options.merge(:redshift_password => 'TEST_REDSHIFT_PASSWORD'))
+              FlareUp::CLI.start(required_arguments)
+            end
+          end
+          context 'when it is not available via ENV' do
+            before do
+              allow(FlareUp::ENVWrap).to receive(:get).with('REDSHIFT_PASSWORD').and_return(nil)
+              expect(FlareUp::CLI).to receive(:bailout).with(1)
+            end
+            it 'should be an error' do
+              FlareUp::CLI.start(required_arguments)
+            end
+          end
+        end
+      end
+
+    end
+
+    describe 'AWS credentials' do
+
+      describe 'access key' do
+        context 'when an AWS access key is specified' do
+          it 'should boot with the proper options' do
+            expect(FlareUp::Boot).to receive(:boot).with(required_options.merge(:aws_access_key => 'CLI_KEY'))
+            FlareUp::CLI.start(required_arguments + %w(--aws_access_key CLI_KEY))
+          end
+        end
+        context 'when an AWS access key is not specified' do
+          context 'when the key is available via ENV' do
+            it 'should boot with the key from the environment' do
+              expect(FlareUp::Boot).to receive(:boot).with(required_options.merge(:aws_access_key => 'TEST_AWS_ACCESS_KEY'))
+              FlareUp::CLI.start(required_arguments)
+            end
+          end
+          context 'when the key is not available via ENV' do
+            before do
+              allow(FlareUp::ENVWrap).to receive(:get).with('AWS_ACCESS_KEY_ID').and_return(nil)
+              expect(FlareUp::CLI).to receive(:bailout).with(1)
+            end
+            it 'should be an error' do
+              FlareUp::CLI.start(required_arguments)
+            end
+          end
+        end
+      end
+
+      describe 'secret key' do
+        context 'when an AWS secret key is specified' do
+          it 'should boot with the proper options' do
+            expect(FlareUp::Boot).to receive(:boot).with(required_options.merge(:aws_secret_key => 'CLI_KEY'))
+            FlareUp::CLI.start(required_arguments + %w(--aws_secret_key CLI_KEY))
+          end
+        end
+        context 'when an AWS secret key is not specified' do
+          context 'when the key is available via ENV' do
+            it 'should boot with the key from the environment' do
+              expect(FlareUp::Boot).to receive(:boot).with(required_options.merge(:aws_secret_key => 'TEST_AWS_SECRET_KEY'))
+              FlareUp::CLI.start(required_arguments)
+            end
+          end
+          context 'when the key is not available via ENV' do
+            before do
+              allow(FlareUp::ENVWrap).to receive(:get).with('AWS_SECRET_ACCESS_KEY').and_return(nil)
+              expect(FlareUp::CLI).to receive(:bailout).with(1)
+            end
+            it 'should be an error' do
+              FlareUp::CLI.start(required_arguments)
+            end
+          end
+        end
+      end
+
+    end
+
+  end
+
+end

data/spec/lib/flare_up/connection_spec.rb

@@ -0,0 +1,114 @@
+describe FlareUp::Connection do
+
+  subject do
+    FlareUp::Connection.new('TEST_HOST', 'TEST_DB_NAME', 'TEST_USER', 'TEST_PASSWORD')
+  end
+
+  its(:host) { should == 'TEST_HOST' }
+  its(:port) { should == 5439 }
+  its(:dbname) { should == 'TEST_DB_NAME' }
+  its(:user) { should == 'TEST_USER' }
+  its(:password) { should == 'TEST_PASSWORD' }
+  its(:connect_timeout) { should == 5 }
+
+  describe 'Writers' do
+    subject do
+      FlareUp::Connection.new('', '', '', '').tap do |c|
+        c.host = 'TEST_HOST'
+        c.port = 111
+        c.dbname = 'TEST_DB_NAME'
+        c.user = 'TEST_USER'
+        c.password = 'TEST_PASSWORD'
+        c.connect_timeout = 222
+      end
+    end
+
+    its(:host) { should == 'TEST_HOST' }
+    its(:port) { should == 111 }
+    its(:dbname) { should == 'TEST_DB_NAME' }
+    its(:user) { should == 'TEST_USER' }
+    its(:password) { should == 'TEST_PASSWORD' }
+    its(:connect_timeout) { should == 222 }
+  end
+
+  describe '#connect' do
+
+    context 'when the connection succeeds' do
+      let(:mock_params) { { :foo => 'bar' } }
+
+      before do
+        allow(subject).to receive(:connection_parameters).and_return(mock_params)
+      end
+
+      it 'should connect to Redshift with the appropriate parameters' do
+        expect(PG).to receive(:connect).with(mock_params)
+
+        subject.send(:connect)
+      end
+    end
+
+    context 'when the connection does not succeed' do
+
+      before do
+        expect(PG).to receive(:connect).and_raise(PG::ConnectionBad, message)
+      end
+
+      context 'when the host name is invalid' do
+        # PG::ConnectionBad: could not translate host name "redshift.amazonaws.co" to address: nodename nor servname provided, or not known
+        let(:message) { 'could not translate host name "TEST_HOSTNAME" to address: nodename nor servname provided, or not known' }
+        it 'should be an error' do
+          expect { subject.send(:connect) }.to raise_error(FlareUp::HostUnknownOrInaccessibleError)
+        end
+      end
+
+      context 'when the connection times out' do
+        # PG::ConnectionBad: timeout expired
+        let(:message) { 'timeout expired' }
+        it 'should be an error' do
+          expect { subject.send(:connect) }.to raise_error(FlareUp::TimeoutError)
+        end
+      end
+
+      context 'when the database does not exist' do
+        # PG::ConnectionBad: FATAL:  database "de" does not exist
+        let(:message) { 'FATAL:  database "TEST_DB_NAME" does not exist' }
+        it 'should be an error' do
+          expect { subject.send(:connect) }.to raise_error(FlareUp::NoDatabaseError)
+        end
+      end
+
+      context 'when authenication credentials are invalid' do
+        # PG::ConnectionBad: FATAL:  password authentication failed for user "slif1"
+        let(:message) { 'password authentication failed for user "slif1"' }
+        it 'should be an error' do
+          expect { subject.send(:connect) }.to raise_error(FlareUp::AuthenticationError)
+        end
+      end
+
+      context 'when the error is unknown' do
+        let(:message) { '_'}
+        it 'should be an error' do
+          expect { subject.send(:connect) }.to raise_error(FlareUp::UnknownError)
+        end
+      end
+
+    end
+
+  end
+
+  describe '#connection_parameters' do
+
+    it 'should return the required parameters' do
+      expect(subject.send(:connection_parameters)).to eq({
+        :host => 'TEST_HOST',
+        :port => 5439,
+        :dbname => 'TEST_DB_NAME',
+        :user => 'TEST_USER',
+        :password => 'TEST_PASSWORD',
+        :connect_timeout => 5,
+      })
+    end
+
+  end
+
+end

data/spec/lib/flare_up/copy_command_spec.rb

@@ -0,0 +1,134 @@
+describe FlareUp::CopyCommand do
+
+  subject do
+    FlareUp::CopyCommand.new('TEST_TABLE_NAME', 'TEST_DATA_SOURCE', 'TEST_ACCESS_KEY', 'TEST_SECRET_KEY')
+  end
+
+  its(:table_name) { should == 'TEST_TABLE_NAME' }
+  its(:data_source) { should == 'TEST_DATA_SOURCE' }
+  its(:aws_access_key_id) { should == 'TEST_ACCESS_KEY' }
+  its(:aws_secret_access_key) { should == 'TEST_SECRET_KEY' }
+  its(:columns) { should == [] }
+  its(:options) { should == '' }
+
+  describe '#get_command' do
+    context 'when no optional fields are provided' do
+      it 'should return a basic COPY command' do
+        expect(subject.get_command).to eq("COPY TEST_TABLE_NAME  FROM 'TEST_DATA_SOURCE' CREDENTIALS 'aws_access_key_id=TEST_ACCESS_KEY;aws_secret_access_key=TEST_SECRET_KEY' ")
+      end
+    end
+
+    context 'when column names are provided' do
+      before do
+        subject.columns = %w(column_name1 column_name2)
+      end
+      it 'should include the column names in the command' do
+        expect(subject.get_command).to start_with('COPY TEST_TABLE_NAME (column_name1, column_name2) FROM')
+      end
+    end
+
+    context 'when options are provided' do
+      before do
+        subject.options = 'OPTION1 OPTION2'
+      end
+      it 'should include the options in the command' do
+        expect(subject.get_command).to end_with(' OPTION1 OPTION2')
+      end
+    end
+  end
+
+  describe '#columns=' do
+    context 'when an array' do
+      it 'should assign the property' do
+        subject.columns = %w(column_name1 column_name2)
+        expect(subject.columns).to eq(%w(column_name1 column_name2))
+      end
+    end
+
+    context 'when not an array' do
+      it 'should not assign the property and be an error' do
+        subject.columns = %w(column_name1)
+        expect {
+          subject.columns = '_'
+        }.to raise_error(ArgumentError)
+        expect(subject.columns).to eq(%w(column_name1))
+      end
+    end
+  end
+
+  describe '#execute' do
+
+    let(:conn) { instance_double('FlareUp::Connection') }
+
+    context 'when successful' do
+      before do
+        expect(conn).to receive(:execute)
+      end
+      it 'should do something' do
+        expect(subject.execute(conn)).to eq([])
+      end
+    end
+
+    context 'when unsuccessful' do
+
+      before do
+        expect(conn).to receive(:execute).and_raise(exception, message)
+      end
+
+      context 'when there was an internal error' do
+
+        let(:exception) { PG::InternalError }
+
+        context 'when there was an error loading' do
+          let(:message) { "Check 'stl_load_errors' system table for details" }
+          before do
+            allow(FlareUp::STLLoadErrorFetcher).to receive(:fetch_errors).and_return('FOO')
+          end
+          it 'should respond with a list of errors' do
+            expect(subject.execute(conn)).to eq('FOO')
+          end
+        end
+
+        context 'when there was an error with the S3 prefix' do
+          let(:message) { "The specified S3 prefix 'test_filename.csv' does not exist" }
+          it 'should be an error' do
+            expect { subject.execute(conn) }.to raise_error(FlareUp::DataSourceError)
+          end
+        end
+
+        context 'when the bucket is not in the same zone as the Redshift instance' do
+          let(:message) { 'The bucket you are attempting to access must be addressed using the specified endpoint' }
+          it 'should be an error' do
+            expect { subject.execute(conn) }.to raise_error(FlareUp::OtherZoneBucketError)
+          end
+        end
+
+        context 'when there was another kind of internal error' do
+          let(:message) { '_' }
+          it 'should respond with a list of errors' do
+            expect { subject.execute(conn) }.to raise_error(PG::InternalError, '_')
+          end
+        end
+
+        context 'when there is a syntax error in the command' do
+          let(:message) { 'ERROR:  syntax error at or near "lmlkmlk3" (PG::SyntaxError)' }
+          it 'should be error' do
+            expect { subject.execute(conn) }.to raise_error(FlareUp::SyntaxError, 'Syntax error in the COPY command: [at or near "lmlkmlk3"].')
+          end
+        end
+
+      end
+
+      context 'when there was another type of error' do
+        let(:exception) { PG::ConnectionBad }
+        let(:message) { '_' }
+        it 'should do something' do
+          expect { subject.execute(conn) }.to raise_error(PG::ConnectionBad, '_')
+        end
+      end
+
+    end
+
+  end
+
+end

data/spec/lib/flare_up/stl_load_error_fetcher_spec.rb

@@ -0,0 +1,45 @@
+describe FlareUp::STLLoadErrorFetcher do
+
+  describe '.fetch_errors' do
+
+    let(:connection) { instance_double('FlareUp::Connection') }
+
+    before do
+      expect(connection).to receive(:execute).
+        with('SELECT * FROM stl_load_errors ORDER BY query DESC, line_number, position LIMIT 1').
+        and_return([
+        {
+          'err_reason' => 'TEST_REASON',
+          'raw_field_value' => 'TEST_RAW_FIELD_VALUE',
+          'raw_line' => 'TEST_RAW_LINE',
+          'col_length' => '1',
+          'type' => 'TEST_TYPE',
+          'colname' => 'TEST_COLNAME',
+          'filename' => 'TEST_FILENAME',
+          'position' => '2',
+          'line_number' => '3'
+        }
+      ])
+    end
+
+    it 'should return the load errors' do
+      expect(FlareUp::STLLoadErrorFetcher.fetch_errors(connection)).to eq(
+        [
+          FlareUp::STLLoadError.new(
+            'TEST_REASON',
+            'TEST_RAW_FIELD_VALUE',
+            'TEST_RAW_LINE',
+            1,
+            'TEST_TYPE',
+            'TEST_COLNAME',
+            'TEST_FILENAME',
+            2,
+            3
+          )
+        ]
+      )
+    end
+
+  end
+
+end

data/spec/lib/flare_up/stl_load_error_spec.rb

@@ -0,0 +1,61 @@
+describe FlareUp::STLLoadError do
+
+  subject do
+    FlareUp::STLLoadError.new(
+      'TEST_REASON',
+      'TEST_RAW_FIELD_VALUE',
+      'TEST_RAW_LINE',
+      1,
+      'TEST_TYPE',
+      'TEST_COLNAME',
+      'TEST_FILENAME',
+      2,
+      3
+    )
+  end
+
+  its(:err_reason) { should == 'TEST_REASON' }
+  its(:raw_field_value) { should == 'TEST_RAW_FIELD_VALUE' }
+  its(:raw_line) { should == 'TEST_RAW_LINE' }
+  its(:col_length) { should == 1 }
+  its(:type) { should == 'TEST_TYPE' }
+  its(:colname) { should == 'TEST_COLNAME' }
+  its(:filename) { should == 'TEST_FILENAME' }
+  its(:position) { should == 2 }
+  its(:line_number) { should == 3 }
+
+  describe '.from_pg_results_row' do
+    let(:values) do
+      {
+        'err_reason' => 'TEST_REASON     ',
+        'raw_field_value' => 'TEST_RAW_FIELD_VALUE    ',
+        'raw_line' => 'TEST_RAW_LINE     ',
+        'col_length' => '1      ',
+        'type' => 'TEST_TYPE     ',
+        'colname' => 'TEST_COLNAME     ',
+        'filename' => 'TEST_FILENAME    ',
+        'position' => '2   ',
+        'line_number' => '3      '
+      }
+    end
+
+    let(:load_error_from_hash) { FlareUp::STLLoadError.from_pg_results_row(values) }
+
+    it 'should be an STLLoadError' do
+      expect(load_error_from_hash).to be_a(FlareUp::STLLoadError)
+    end
+
+    it 'should store the values properly' do
+      expect(load_error_from_hash.err_reason).to eq('TEST_REASON')
+      expect(load_error_from_hash.raw_field_value).to eq('TEST_RAW_FIELD_VALUE')
+      expect(load_error_from_hash.raw_line).to eq('TEST_RAW_LINE')
+      expect(load_error_from_hash.col_length).to eq(1)
+      expect(load_error_from_hash.type).to eq('TEST_TYPE')
+      expect(load_error_from_hash.colname).to eq('TEST_COLNAME')
+      expect(load_error_from_hash.filename).to eq('TEST_FILENAME')
+      expect(load_error_from_hash.position).to eq(2)
+      expect(load_error_from_hash.line_number).to eq(3)
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+require 'flare_up'
+
+require 'rspec/its'
+
+ENV['TESTING'] = 'TESTING'