rails_redshift_replicator 0.0.1

data/spec/lib/rails_redshift_replicator/rlogger_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe RailsRedshiftReplicator::RLogger do
+  @messages = []
+  let!(:logger) { RailsRedshiftReplicator::RLogger.new(STDOUT) }
+  it 'emits info notification' do
+    expect(ActiveSupport::Notifications).to receive(:instrument).with('rails_redshift_replicator', type: :info, message: 'hello')
+    logger.info 'hello'
+  end
+  it 'emits warn notification' do
+    expect(ActiveSupport::Notifications).to receive(:instrument).with('rails_redshift_replicator', type: :warn, message: 'hello')
+    logger.warn 'hello'
+  end
+  it 'emits debug notification' do
+    expect(ActiveSupport::Notifications).to receive(:instrument).with('rails_redshift_replicator', type: :debug, message: 'hello')
+    logger.debug 'hello'
+  end
+  it 'emits error notification' do
+    expect(ActiveSupport::Notifications).to receive(:instrument).with('rails_redshift_replicator', type: :error, message: 'hello')
+    logger.error 'hello'
+  end
+end

data/spec/lib/rails_redshift_replicator/tools/analyze_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+describe RailsRedshiftReplicator::Tools::Analyze do
+  context "when uses :all for all tables" do
+    it "executes" do
+      expect(RailsRedshiftReplicator.connection).to receive(:exec).with("ANALYZE ;")
+      RailsRedshiftReplicator.analyze(:all)
+    end
+  end
+  context "with one table" do
+    it "executes" do
+      expect(RailsRedshiftReplicator.connection).to receive(:exec).with("ANALYZE users;")
+      RailsRedshiftReplicator.analyze("users")
+    end
+  end
+end

data/spec/lib/rails_redshift_replicator/tools/vacuum_spec.rb

@@ -0,0 +1,65 @@
+require 'spec_helper'
+describe RailsRedshiftReplicator::Tools::Vacuum do
+  describe ".vacuum" do
+    context "with :all" do
+      let(:sql) { "VACUUM ;" }
+      it "performs vacuum on all tables" do
+        expect(RailsRedshiftReplicator.connection).to receive(:exec).with(sql)
+        RailsRedshiftReplicator.vacuum(:all)
+      end
+    end
+    context "without tables" do
+      let(:sql) { "VACUUM ;" }
+      it "performs vacuum on all tables" do
+        expect(RailsRedshiftReplicator.connection).to receive(:exec).with(sql)
+        RailsRedshiftReplicator.vacuum
+      end
+    end
+    context "when using auto_tune" do
+      it "perfoms vacuum on all tables" do
+        expect_any_instance_of(RailsRedshiftReplicator::Tools::Vacuum).to receive(:auto_tuned_vacuum).with(:posts)
+        RailsRedshiftReplicator.vacuum(:posts, "auto_tune")
+      end
+    end
+  end
+ 
+  describe ".auto_tuned_vacuum" do
+    before do
+      allow_any_instance_of(RailsRedshiftReplicator::Tools::Vacuum)
+        .to receive(:sort_types)
+        .and_return( [{"sort_type"=>"compound", "tablename"=>"users"}, {"sort_type"=>"interleaved", "tablename"=>"posts"}])
+    end
+    context "using :all" do
+      let(:vacuum) { RailsRedshiftReplicator::Tools::Vacuum.new(:all) }
+      it "performs vacuum on all tables" do
+        expect(vacuum).to receive(:exec_vacuum_command).with("users", "compound")
+        expect(vacuum).to receive(:exec_vacuum_command).with("posts", "interleaved")
+        vacuum.auto_tuned_vacuum(:all)
+      end
+    end
+    context "with a given table" do
+      let(:vacuum) { RailsRedshiftReplicator::Tools::Vacuum.new(:users) }
+      it "performs vacuum on the given table" do
+        expect(vacuum).to receive(:exec_vacuum_command).with("users", "compound")
+        expect(vacuum).not_to receive(:exec_vacuum_command).with("posts", "interleaved")
+        vacuum.auto_tuned_vacuum("users")
+      end
+    end
+  end
+
+  describe "exec_vacuum_command" do
+    let(:vacuum) { RailsRedshiftReplicator::Tools::Vacuum.new(:all) }
+    context "with a compound key table" do
+      it "builds the command" do
+        expect(RailsRedshiftReplicator.connection).to receive(:exec).with("VACUUM FULL users;")
+        vacuum.exec_vacuum_command("users", "compound")
+      end
+    end
+    context "with an interleaved key table" do
+      it "builds the command" do
+        expect(RailsRedshiftReplicator.connection).to receive(:exec).with("VACUUM REINDEX posts;")
+        vacuum.exec_vacuum_command("posts", "interleaved")
+      end
+    end
+  end
+end

data/spec/lib/rails_redshift_replicator_spec.rb

@@ -0,0 +1,110 @@
+require 'spec_helper'
+
+describe RailsRedshiftReplicator do
+  describe "Config" do
+    %w(redshift_connection_params aws_credentials s3_bucket_params redshift_slices local_replication_path debug_mode history_cap).each do |param|
+      it "responde para configuração #{param}" do
+        expect(RailsRedshiftReplicator).to respond_to param
+      end
+    end
+  end
+
+  describe '.add_replicable', broken: true do
+    let(:replicable) { RailsRedshiftReplicator::Replicable.new(:identity_replicator, source_table: :users) }
+    around do |example|
+      old = RailsRedshiftReplicator.replicables
+      RailsRedshiftReplicator.replicables = {}
+      example.run
+      RailsRedshiftReplicator.replicables = old
+    end
+    it 'keeps a list of replicable tables' do
+      RailsRedshiftReplicator.add_replicable({ users: replicable })
+      expect(RailsRedshiftReplicator.replicables.keys).to contain_exactly(:users)
+    end
+  end
+
+  describe ".connection" do
+    it "retorna uma conexão para o redshift" do
+      expect(RailsRedshiftReplicator.connection).to be_instance_of PG::Connection
+    end
+  end
+
+  describe '#tables_to_perform' do
+    context 'when exporting all tables' do
+      before { RailsRedshiftReplicator.add_replicable(tags_users: RailsRedshiftReplicator::Replicable.new(:full_replicator, source_table: :tags_users)) }
+      it 'returns all tables' do
+        expect(RailsRedshiftReplicator.tables_to_perform(:all)).to contain_exactly('users', 'posts', 'tags', 'tags_users')
+      end
+    end
+    context 'when exporting selected tables' do
+      context 'and all given tables are replicable' do
+        it 'returns all given tables' do
+          expect(RailsRedshiftReplicator.tables_to_perform([:users, :posts])).to contain_exactly('users', 'posts')
+        end
+      end
+      context 'and some given tables are not replicable' do
+        it 'returns only the tables that are eligible for replication' do
+          expect(RailsRedshiftReplicator.tables_to_perform([:users, :posts, :nonrep])).to contain_exactly('users','posts')
+        end
+        it 'warns about unreplicable' do
+          expect(RailsRedshiftReplicator).to receive(:warn_if_unreplicable).with(['nonrep'])
+          RailsRedshiftReplicator.tables_to_perform([:users, :posts, :nonrep])
+        end
+      end
+    end
+  end
+  describe '#check_args' do
+    context 'when no tables were given' do
+      it 'raises error' do
+        expect{ RailsRedshiftReplicator.check_args([]) }.to raise_error(StandardError)
+      end
+    end
+    context 'when tables were given' do
+      it 'returns nil' do
+        expect(RailsRedshiftReplicator.check_args([:users])).to be_nil
+      end
+    end
+  end
+
+  describe '.debug_mode=' do
+    context 'enabling debug_mode' do
+      before { RailsRedshiftReplicator.debug_mode = false }
+      it 'changes logger level to debug' do
+        expect { RailsRedshiftReplicator.setup {|config| config.debug_mode = true} }.to change(RailsRedshiftReplicator.logger, :level).to(Logger::DEBUG)
+      end
+    end
+    context 'disabling debug_mode' do
+      before { RailsRedshiftReplicator.debug_mode = true }
+      it 'changes logger level to error' do
+        expect { RailsRedshiftReplicator.setup {|config| config.debug_mode = false} }.to change(RailsRedshiftReplicator.logger, :level).to(Logger::WARN)
+      end
+    end
+  end
+  describe '.history_cap=', :focus do
+    context 'when nil' do
+      it 'assigns nil' do
+        RailsRedshiftReplicator.history_cap = nil
+        expect(RailsRedshiftReplicator.history_cap).to be_nil
+      end
+    end
+    context 'when 2' do
+      it 'assigns 2' do
+        RailsRedshiftReplicator.history_cap = 2
+        expect(RailsRedshiftReplicator.history_cap).to eq 2
+      end
+    end
+    context 'when > 2' do
+      it 'assigns given cap' do
+        RailsRedshiftReplicator.history_cap = 5
+        expect(RailsRedshiftReplicator.history_cap).to eq 5
+      end
+    end
+    context 'when < 2' do
+      it 'assigns 2' do
+        RailsRedshiftReplicator.history_cap = 1
+        expect(RailsRedshiftReplicator.history_cap).to eq 2
+      end
+    end
+  end
+
+end

data/spec/models/rails_redshift_replicator/replication_spec.rb

@@ -0,0 +1,104 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe RailsRedshiftReplicator::Replication, type: :model do
+
+  it { is_expected.to validate_presence_of(:replication_type) }
+  it { is_expected.to validate_presence_of(:key) }
+  it { is_expected.to validate_presence_of(:source_table) }
+  it { is_expected.to validate_presence_of(:target_table) }
+  it { is_expected.to validate_inclusion_of(:state).in_array RailsRedshiftReplicator::Replication::STATES }
+  it { is_expected.to validate_inclusion_of(:export_format).in_array RailsRedshiftReplicator::Replication::FORMATS }
+
+  let(:redshift_replication) { build :redshift_replication }
+  describe "Format helper methods" do
+    RailsRedshiftReplicator::Replication::FORMATS.each do |format|
+      it "responds to ##{format}?" do
+        expect(redshift_replication).to respond_to "#{format}?"
+      end
+    end
+  end
+  describe '.cap_id' do
+    context 'when there are more replications than cap number' do
+      it 'returns the id of the replication based on the cap number' do
+        2.times { create :redshift_replication, source_table: 'users' }
+        rep =   create :redshift_replication, source_table: 'users'
+        3.times { create :redshift_replication, source_table: 'users' }
+        expect(RailsRedshiftReplicator::Replication.cap_id('users', 4)).to eq rep.id
+      end
+    end
+    context 'when there are no replications' do
+      it 'returns nil' do
+        expect(RailsRedshiftReplicator::Replication.cap_id('users', 4)).to be_nil  
+      end
+    end
+    context 'when there are less replications then the cap number' do
+      it 'retuns nil' do
+        3.times { create :redshift_replication, source_table: 'users' }
+        expect(RailsRedshiftReplicator::Replication.cap_id('users', 4)).to be_nil
+      end
+    end
+  end
+  describe "State helper methods" do
+    RailsRedshiftReplicator::Replication::STATES.each do |state|
+      it "responds to ##{state}?" do
+        expect(redshift_replication).to respond_to "#{state}?"
+      end
+      it "responds to ##{state}!" do
+        expect(redshift_replication).to respond_to "#{state}!"
+      end
+      it "responds to scope .#{state}" do
+        expect(RailsRedshiftReplicator::Replication).to respond_to state
+      end
+    end
+  end
+  describe "format #csv?" do
+    let(:redshift_replication) { build :redshift_replication, export_format: "csv" }
+    context "when format matches" do
+      it "returns true" do
+        expect(redshift_replication.csv?).to be true
+      end
+    end
+    context "when format does not match" do
+      it "returns false" do
+        expect(redshift_replication.gzip?).to be false
+      end
+    end
+  end
+  describe "state #uploading" do
+    let(:redshift_replication) { build :redshift_replication, state: "uploading" }
+    context "when state matches" do
+      it "returns true" do
+        expect(redshift_replication.uploading?).to be true
+      end
+    end
+    context "when state does not match" do
+      it "returns false" do
+        expect(redshift_replication.exporting?).to be false
+      end
+    end
+  end
+  describe "state #exported!" do
+    let(:redshift_replication) { create(:redshift_replication, state: 'enqueued') }
+    it "changes stage to :exporting" do
+      redshift_replication.exported!
+      expect(redshift_replication.state).to eq 'exported'
+    end
+    it "updates aditional fields" do
+      redshift_replication.exported!(export_duration: 10)
+      expect(redshift_replication.export_duration).to eq 10
+    end
+    it "returns update time" do
+      allow(Time).to receive(:now).and_return(Time.new(2016,1,1))
+      now = Time.now
+      expect(redshift_replication.exported!).to eq(now)
+    end
+  end
+  describe "scope .exporting" do
+    let!(:rep1) { create(:redshift_replication, state: 'enqueued') }
+    let!(:rep2) { create(:redshift_replication, state: 'exporting') }
+    it "returns replications in :exporting state" do
+      expect(RailsRedshiftReplicator::Replication.exporting.to_a).to eq [rep2]
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,36 @@
+ENV['RAILS_ENV'] ||= 'test'
+
+require 'dotenv'
+Dotenv.load File.expand_path("../dummy/.env", __FILE__)
+
+require File.expand_path("../dummy/config/environment.rb", __FILE__)
+
+require 'rspec/rails'
+require 'rspec/autorun'
+require 'factory_girl_rails'
+require 'shoulda-matchers'
+require 'climate_control'
+require 'pry-byebug'
+require 'timecop'
+
+Rails.backtrace_cleaner.remove_silencers!
+
+# Load support files
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
+
+RSpec.configure do |config|
+ config.mock_with :rspec
+ config.use_transactional_fixtures = true
+ config.infer_base_class_for_anonymous_controllers = false
+ config.order = "random"
+ config.filter_run_excluding :broken => true
+ config.include FactoryGirl::Syntax::Methods
+ config.include RailsRedshiftReplicatorHelpers
+end
+
+Shoulda::Matchers.configure do |config|
+  config.integrate do |with|
+    with.test_framework :rspec
+    with.library :rails
+  end
+end

data/spec/support/csv/invalid_user.csv

@@ -0,0 +1,12 @@
+como tem max error=10, tem que dar mais de 10 erros para alertar
+invalid
+invalid
+invalid
+invalid
+invalid
+invalid
+invalid
+invalid
+invalid
+invalid
+invalid

data/spec/support/csv/valid_post.csv

@@ -0,0 +1,2 @@
+1,2,content1,2015-01-01,2015-01-02
+2,3,content2,2015-01-01,2015-01-02

data/spec/support/csv/valid_tags_users.csv

@@ -0,0 +1 @@
+1,2

data/spec/support/csv/valid_user.csv

@@ -0,0 +1,2 @@
+1,digital,10,1,2015-04-12,2015-04-12
+2,pages,20,1,2015-04-12,2015-04-12

data/spec/support/rails_redshift_replicator_helpers.rb

@@ -0,0 +1,95 @@
+module RailsRedshiftReplicatorHelpers
+  def with_modified_env(options = {}, &block)
+    env = { REDSHIFT_HOST: '1',
+      REDSHIFT_DATABASE: '2',
+      REDSHIFT_PORT: '3',
+      REDSHIFT_USER: '4',
+      REDSHIFT_PASSWORD: '5',
+      REDSHIFT_SLICES: '6',
+      AWS_ACCESS_KEY_ID: '7',
+      AWS_SECRET_ACCESS_KEY: '8',
+      RDP_REPLICATION_REGION: '9',
+      LOCAL_REPLICATION_PATH: '10',
+      RDP_ANALYTICS_BUCKET: '11'}.merge(options)
+    ClimateControl.modify(env, &block)
+  end
+
+  def replication_file(name)
+    File.open("spec/support/csv/#{name}")
+  end
+
+  # Changes ActiveRecord connection to Redshift for the duration of the example
+  # and restablishes connection to the original database right afterwards.
+  def with_engine_connection
+    original = ActiveRecord::Base.remove_connection
+    ActiveRecord::Base.establish_connection(
+      adapter: "redshift",
+      host: RailsRedshiftReplicator.redshift_connection_params[:host],
+      database: RailsRedshiftReplicator.redshift_connection_params[:dbname],
+      port: RailsRedshiftReplicator.redshift_connection_params[:port],
+      username: RailsRedshiftReplicator.redshift_connection_params[:user],
+      password: RailsRedshiftReplicator.redshift_connection_params[:password])
+    yield
+  ensure
+    ActiveRecord::Base.establish_connection(original)
+  end
+
+  # Recreates the test users table on redshift
+  def recreate_users_table(custom_table_name = nil)
+    table_name = custom_table_name || :users
+    with_engine_connection do
+      ActiveRecord::Migration.create_table table_name, force: true do |t|
+        t.string :login
+        t.integer :age
+        t.boolean :confirmed
+        t.timestamps
+      end
+    end
+  end
+
+  # Recreates the posts users table on redshift
+  def recreate_posts_table(custom_table_name = nil)
+    table_name = custom_table_name || :posts
+    with_engine_connection do
+      ActiveRecord::Migration.create_table table_name, force: true do |t|
+        t.belongs_to :user
+        t.text :content
+        t.timestamps
+      end
+    end
+  end
+
+  # Recreates the test tags table on redshift
+  def recreate_tags_table(custom_table_name = nil)
+    table_name = custom_table_name || :tags
+    with_engine_connection do
+      ActiveRecord::Migration.create_table table_name, force: true do |t|
+        t.string :name
+        t.timestamps
+      end
+    end
+  end
+
+  # Recreates the test tags table on redshift
+  def recreate_tags_users_table(custom_table_name = nil)
+    table_name = custom_table_name || :tags_users
+    with_engine_connection do
+      ActiveRecord::Migration.create_table table_name, id: false, force: true do |t|
+        t.integer :user_id
+        t.integer :tag_id
+      end
+    end
+  end
+
+  # Drops a given test table on redshift
+  # @param table [String,Symbol] test table's name
+  def drop_redshift_table(table)
+    with_engine_connection do
+      begin
+        ActiveRecord::Migration.drop_table table.to_s
+      rescue => e
+        puts "Coudn't drop table #{table}: #{e.message}"
+      end
+    end
+  end
+end