rails_redshift_replicator 0.0.1

data/spec/lib/rails_redshift_replicator/deleter_spec.rb

@@ -0,0 +1,90 @@
+require 'spec_helper'
+
+describe RailsRedshiftReplicator::Deleter do
+  let(:replicable)  { RailsRedshiftReplicator.replicables[:users] }
+  let(:deleter)     { RailsRedshiftReplicator::Deleter.new(replicable) }
+  describe '#purge_deleted' do
+    before { 2.times { User.create } }
+    it 'purge deleted records for the given table' do
+      expect(deleter.deleted_ids.count).to eq 0
+      User.first.destroy
+      expect(deleter.deleted_ids.count).to eq 1
+      deleter.purge_deleted
+      expect(deleter.deleted_ids.count).to eq 0
+    end
+  end
+  describe '#delete_on_target' do
+    it 'calls redshift to delete' do
+      expect(RailsRedshiftReplicator.connection).to receive(:exec).and_return(double("result", result_status: 1))
+      deleter.delete_on_target
+    end
+  end
+  describe '#handle_delete_propagation' do
+    context 'when tracking deleted' do
+      context 'and has deleted records' do
+        before do
+          allow(RailsRedshiftReplicator.connection).to receive(:exec)
+          2.times { User.create }
+          User.delete_all
+        end
+        it 'calls #delete_on_target' do
+          expect(deleter).to receive(:delete_on_target)
+          deleter.handle_delete_propagation
+        end
+        context 'if delete on target is success' do
+          before { allow(RailsRedshiftReplicator.connection).to receive(:exec).and_return(double("result", result_status: 1)) }
+          it 'calls #purge_deleted' do
+            expect(deleter).to receive(:purge_deleted)
+            deleter.handle_delete_propagation
+          end
+        end
+        context 'if delete on target fails' do
+          before { allow(RailsRedshiftReplicator.connection).to receive(:exec).and_return(double("result", result_status: 0)) }
+          it 'does not call calls #purge_deleted and logs error' do
+            expect(deleter).not_to receive(:purge_deleted)
+            expect(RailsRedshiftReplicator.logger).to receive(:error)
+            deleter.handle_delete_propagation
+          end
+        end
+      end
+      context 'and without deleted records' do
+        it 'does not call #delete_on_target' do
+          expect(deleter).not_to receive(:delete_on_target)
+          deleter.handle_delete_propagation
+        end
+      end
+    end
+    context 'if not tracking deleted' do
+      before { replicable.instance_variable_set("@tracking", false) }
+      it 'does not call #delete_on_target' do
+        expect(deleter).not_to receive(:delete_on_target)
+        deleter.handle_delete_propagation
+      end
+    end
+  end
+  describe '#deleted_ids' do
+    it 'returns array of deleted ids' do
+      user = create :user
+      id = user.id
+      User.delete_all
+      expect(deleter.deleted_ids).to eq [id.to_s]
+    end
+  end
+  describe '#has_deleted_ids?' do
+    context 'when there are deleted records' do
+      before do
+        2.times { User.create }
+        User.delete_all
+      end
+      it 'returns true' do
+        expect(deleter.has_deleted_ids?).to be_truthy
+      end
+    end
+    context "when there aren't deleted records" do
+      it 'returns false' do
+        expect(deleter.has_deleted_ids?).to be_falsy
+      end
+    end
+  end
+
+end

data/spec/lib/rails_redshift_replicator/exporters/base_spec.rb

@@ -0,0 +1,326 @@
+require 'spec_helper'
+require 'active_support/core_ext/time/calculations'
+
+describe RailsRedshiftReplicator::Exporters::Base do
+  let(:user_replicable)  { RailsRedshiftReplicator::Replicable.new(:identity_replicator, source_table: :users)}
+  let(:post_replicable)  { RailsRedshiftReplicator::Replicable.new(:timed_replicator, source_table: :posts)}
+  let(:full_replicable)  { RailsRedshiftReplicator::Replicable.new(:full_replicator, source_table: :tags_users)}
+  let(:user_exporter)    { RailsRedshiftReplicator::Exporters::IdentityReplicator.new(user_replicable) }
+  let(:post_exporter)    { RailsRedshiftReplicator::Exporters::TimedReplicator.new(post_replicable) }
+  let(:habtm_exporter)   { RailsRedshiftReplicator::Exporters::FullReplicator.new(full_replicable) }
+  before { RailsRedshiftReplicator.debug_mode = true }
+
+
+
+  describe 'Instance Methods' do
+    describe "#table_indexes" do
+      context "with User model" do
+        it "returns user's table indexes" do
+          expect(user_exporter.table_indexes).to contain_exactly("id", "login", "age")
+        end
+      end
+    end
+    
+    describe "#upload" do
+      context "with csv format" do
+        let(:files) { %w(file1 file2) }
+        before { user_exporter.replication = create(:redshift_replication, source_table: "users", export_format: "csv") }
+        before { user_exporter.file_names = files }
+        context 'when exporter is in an error state' do
+          before { user_exporter.errors = "some error"}
+          it 'returns nil' do
+            expect(user_exporter.upload).to be_nil 
+          end
+        end
+        context 'when exporter is valid', focus: true do
+          it "calls csv uploader" do
+            expect(user_exporter.file_manager).to receive(:upload_csv).with(files)
+            user_exporter.upload
+          end
+          it "changes replication state to uploaded" do
+            expect(user_exporter.file_manager).to receive(:upload_csv).with(files)
+            user_exporter.upload
+            expect(user_exporter.replication.state).to eq "uploaded"
+          end
+        end
+      end
+    end
+
+    describe '#connection_adapter' do
+      context 'when Mysql2' do
+        before { allow(user_exporter.ar_client).to receive(:adapter_name).and_return("Mysql2") }
+        it 'returns RailsRedshiftReplicator::Adapters::Mysql2' do
+          expect(user_exporter.connection_adapter).to be_an RailsRedshiftReplicator::Adapters::Mysql2
+        end
+      end
+      context 'when PostgreSQL' do
+        before { allow(user_exporter.ar_client).to receive(:adapter_name).and_return("PostgreSQL") }
+        it 'returns RailsRedshiftReplicator::Adapters::Postgresql' do
+          expect(user_exporter.connection_adapter).to be_an RailsRedshiftReplicator::Adapters::PostgreSQL
+        end
+      end
+      context 'when SQLite' do
+        before { allow(user_exporter.ar_client).to receive(:adapter_name).and_return("SQLite") }
+        it 'returns RailsRedshiftReplicator::Adapters::Sqlite' do
+          expect(user_exporter.connection_adapter).to be_an RailsRedshiftReplicator::Adapters::SQLite
+        end
+      end
+    end
+
+    describe "#has_index?" do
+      before { allow(user_exporter).to receive(:table_indexes).and_return [:id, :updated_at] }
+      context "when replication_field has an index" do
+        before { allow(user_exporter).to receive(:replication_field).and_return(:updated_at) }
+        it "returns true" do
+          expect(user_exporter).to be_has_index
+        end
+      end
+      context "when replication_field doesn't have an index" do
+        before { allow(user_exporter).to receive(:replication_field).and_return(:login) }
+        it "returns false" do
+          expect(user_exporter).not_to be_has_index
+        end
+      end
+    end
+
+
+    describe "#replication_field" do
+      it "returns the replicable replication_field" do
+        expect(RailsRedshiftReplicator::Exporters::Base.new(user_replicable).replication_field).to eq 'id'
+      end
+    end
+
+    describe "#check_target_table" do
+      context "when the target table exists on redshift" do
+        before { recreate_users_table }
+        it "doesn't set an error" do
+          user_exporter.check_target_table
+          expect(user_exporter.errors).to be_nil
+        end
+      end
+      context "when the target table doesn't exist on redshift" do
+        before(:all) { drop_redshift_table(:users) }
+        after(:all) { recreate_users_table }
+        it "sets the exporter to an error state" do
+          user_exporter.check_target_table
+          expect(user_exporter.errors).to be_present
+        end
+      end
+    end
+
+    describe '#query_command' do
+      before { allow(user_exporter).to receive(:fields_to_sync).and_return(%w(id user_id publication_id)) }
+      before { allow(user_exporter).to receive(:last_record).and_return(10) }
+      
+      let(:sql) { "SELECT id,user_id,publication_id FROM users WHERE 1=1 AND id > '5' AND id <= '10' OR id IS NULL" }
+      context 'when adapter is Mysql2' do
+        before { allow(user_exporter.ar_client).to receive(:adapter_name).and_return('Mysql2') }
+        it "executes query with streaming option" do
+          expect(user_exporter.ar_client.instance_variable_get("@connection")).to receive(:query).with(sql, stream: true)
+          user_exporter.records(5)
+        end
+      end
+      context "when adapter is SQLite", focus: true do
+        before { allow(user_exporter.ar_client).to receive(:adapter_name).and_return('SQLite') }
+        it "executes query without other arguments" do
+          expect(user_exporter.ar_client).to receive(:exec_query).with(sql)
+          user_exporter.records(5)
+        end
+      end
+      context "when adapter is PostgreSQL" do
+        before { allow(user_exporter.ar_client).to receive(:adapter_name).and_return('PostgreSQL') }
+        xit "executes query using single row mode" do
+        end
+      end
+    end
+
+    describe '#records' do
+      
+    end
+
+    describe "#build_query_sql" do
+      before { allow(user_exporter).to receive(:fields_to_sync).and_return(%w(id user_id publication_id)) }
+      context "when there's a last record" do
+        before { allow(user_exporter).to receive(:last_record).and_return(10) }
+        context "when an initial record is given" do
+          let(:sql) { "SELECT id,user_id,publication_id FROM users WHERE 1=1 AND id > '5' AND id <= '10' OR id IS NULL" }
+          it "builds correct sql" do
+            expect(user_exporter.build_query_sql(5)).to eq sql
+          end
+        end
+        context "when an initial record isn't given" do
+          let(:sql) { "SELECT id,user_id,publication_id FROM users WHERE 1=1 AND id <= '10' OR id IS NULL" }
+          it "builds correct sql" do
+            expect(user_exporter.build_query_sql).to eq sql
+          end
+        end
+      end
+      context "when a last_record can't be found" do
+        before { allow(user_exporter).to receive(:last_record).and_return(nil) }
+        context "when an initial record is given" do
+          let(:sql) { "SELECT id,user_id,publication_id FROM users WHERE 1=1 AND id > '5'" }
+          it "builds correct sql" do
+            expect(user_exporter.build_query_sql(5)).to eq sql
+          end
+        end
+        context "when an initial record isn't given" do
+          let(:sql) { "SELECT id,user_id,publication_id FROM users WHERE 1=1" }
+          it "builds correct sql" do
+            expect(user_exporter.build_query_sql).to eq sql
+          end
+        end
+      end
+    end
+
+    describe "#from_record" do
+      context "when there's a previous replication record" do
+        let!(:replication) { create :redshift_replication, target_table: 'users', last_record: 1 }
+        let!(:replication2) { create :redshift_replication, target_table: 'users', last_record: 5 }
+        it "returns the id or timestamp of the last_record on the most recent complete replication" do
+          expect(user_exporter.from_record).to eq '5'
+        end
+      end
+      context "when there isn't a previous replication record" do
+        it "returns nil" do
+          expect(user_exporter.from_record).to be_nil
+        end
+      end
+    end
+
+    describe "#initialize_replication" do
+      before do
+        allow(user_exporter).to receive(:last_record).and_return(3)
+        allow(user_exporter).to receive(:from_record).and_return(1)
+        user_exporter.initialize_replication("replication_file", "csv", 2)
+      end
+      let(:replication) { user_exporter.replication }
+
+      it "creates a replication record" do
+        expect(replication.export_format).to    eq "csv"
+        expect(replication.slices).to           eq 2
+        expect(replication.record_count).to     be_nil
+        expect(replication.key).to              eq "rrr/users/replication_file"
+        expect(replication.last_record).to      eq '3'
+        expect(replication.first_record).to     eq '1'
+        expect(replication.state).to            eq "exporting"
+        expect(replication.replication_type).to eq "identity_replicator"
+        expect(replication.target_table).to     eq "users"
+        expect(replication.source_table).to     eq "users"
+      end
+    end
+
+    describe "#export" do
+      before { allow(user_exporter).to receive(:fields_to_sync).and_return(%w(id login age)) }
+      context "when there are incomplete replications" do
+        before { create :redshift_replication, source_table: "users", state: "uploaded" }
+        it "doesn't create replication record" do
+          user_exporter.export
+          expect(user_exporter.replication).to be_nil
+        end
+        it "returns nil" do
+          expect(user_exporter.export).to be_nil
+        end
+      end
+      context "when there are no incomplete replications" do
+        before { create :redshift_replication, source_table: "users", state: "imported" }
+        context "and there are records to export" do
+          before { create :user }
+          context 'and exporter is in error state' do
+            before { user_exporter.errors = 'some error' }
+            it 'returns false' do
+              expect(user_exporter.export).to be_nil
+            end
+          end
+          context 'and exporter is valid' do
+            it "creates export files" do
+              file_paths = user_exporter.export
+              expect(file_paths).to be_an_instance_of Array
+              expect(file_paths).to be_present
+            end
+            it "flags replication as exported" do
+              user_exporter.export
+              expect(user_exporter.replication.state).to eq "exported"
+            end
+          end
+        end
+        context "and there aren't any records to export" do
+          it "doesn't create replication record" do
+            user_exporter.export
+            expect(user_exporter.replication).to be_nil
+          end
+          it "retorns nil" do
+            expect(user_exporter.export).to be_nil
+          end
+        end
+      end
+    end
+
+    describe "#last_replication" do
+      let!(:replication1) { create :redshift_replication, source_table: "users" }
+      let!(:replication2) { create :redshift_replication, source_table: "users" }
+      let!(:replication3) { create :redshift_replication, source_table: "posts" }
+      it "retuns the last replication record for a given table" do
+        expect(user_exporter.last_replication).to eq replication2
+      end
+    end
+
+    describe "#export_and_upload" do
+      context "when the target table doesn't exist on redshift" do
+        before(:all) { drop_redshift_table(:users) }
+        after(:all)  { recreate_users_table }
+        it "doesn't call #export" do
+          expect(user_exporter.export_and_upload).not_to receive(:export)
+        end
+      end
+      context "when the target table exists on redshift" do
+        before(:all) { recreate_users_table }
+        it "calls #export" do
+          expect(user_exporter).to receive(:export)
+          user_exporter.export_and_upload
+        end
+        context "when there are records to export" do
+          before { allow(user_exporter).to receive(:export).and_return("file") }
+          it "calls #upload" do
+            expect(user_exporter).to receive(:upload)
+            user_exporter.export_and_upload
+          end
+        end
+        context "when there are no records to export" do
+          it "doesn't call #upload" do
+            expect(user_exporter.export_and_upload).not_to receive(:upload)
+          end
+        end
+
+      end
+    end
+
+    describe "#last_record" do
+      let!(:user1) { create :user }
+      let!(:post1) { create :post }
+      context "when the last replication record uses time based replication" do
+        it "returns a date" do
+          Timecop.freeze Date.tomorrow do
+            post2 = create :post
+            formatted = Time.parse(post_exporter.last_record).to_s(:db)
+            expect(formatted).to eq post2.updated_at.to_s(:db)
+          end
+        end
+      end
+      context "when the last replication record uses an identity column" do
+        let!(:post)  { create :post }
+        it "returns the id" do
+          user1 = create :user
+          user2 = create :user
+          expect(user_exporter.last_record.to_s).to eq user2.id.to_s
+        end
+      end
+    end
+
+    describe "fields_to_sync" do
+      before { recreate_users_table }
+      it "returns list of columns on the target table on redshift" do
+        expect(user_exporter.fields_to_sync).to contain_exactly("age", "confirmed", "created_at", "id", "login", "updated_at")
+      end
+    end
+  end
+end

data/spec/lib/rails_redshift_replicator/exporters/full_replicator_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+
+describe RailsRedshiftReplicator::Exporters::FullReplicator do
+  let(:file_manager)     { RailsRedshiftReplicator::FileManager.new}
+  let(:replicable) { RailsRedshiftReplicator::Replicable.new(:full_replicator, source_table: :tags_users) }
+  let(:s3_bucket) { Aws::S3::Bucket.new(name: RailsRedshiftReplicator.s3_bucket_params[:bucket], client: file_manager.s3_client) }
+  describe "#replication_field", :focus do
+    it "returns the replicable replication_field" do
+      expect(RailsRedshiftReplicator::Exporters::FullReplicator.new(replicable).replication_field).to be_nil
+    end
+  end
+  describe 'Integration Test' do
+    before(:all) { recreate_tags_users_table }
+    it "exports full replicator type replication" do
+      model = :tags_users
+      # first export
+      tag = create :tag
+      user = create :user
+      tag.users << user
+      RailsRedshiftReplicator::Exporters::FullReplicator.new(replicable).export_and_upload
+      replication1 = RailsRedshiftReplicator::Replication.from_table("tags_users").last
+      expect(replication1.state).to eq "uploaded"
+      expect(replication1.record_count).to eq 1
+      file_body = s3_bucket.object("#{replication1.key}.aa").get.body.read
+      expect(file_body).to match(/#{user.id},#{tag.id}/)
+      replication1.imported!
+      # export without new records
+      RailsRedshiftReplicator::Exporters::FullReplicator.new(replicable).export_and_upload
+      replication2 = RailsRedshiftReplicator::Replication.from_table("tags_users").last
+      expect(replication2.record_count).to eq 1
+    end
+  end
+end