RubyGems - rails_redshift_replicator - Versions diffs - 0.0.1 - Mend

rails_redshift_replicator 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (116) hide show

data/spec/dummy/public/404.html ADDED

@@ -0,0 +1,67 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>The page you were looking for doesn't exist (404)</title>
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <style>
+  body {
+    background-color: #EFEFEF;
+    color: #2E2F30;
+    text-align: center;
+    font-family: arial, sans-serif;
+    margin: 0;
+  }
+  div.dialog {
+    width: 95%;
+    max-width: 33em;
+    margin: 4em auto 0;
+  }
+  div.dialog > div {
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #BBB;
+    border-top: #B00100 solid 4px;
+    border-top-left-radius: 9px;
+    border-top-right-radius: 9px;
+    background-color: white;
+    padding: 7px 12% 0;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+  h1 {
+    font-size: 100%;
+    color: #730E15;
+    line-height: 1.5em;
+  }
+  div.dialog > p {
+    margin: 0 0 1em;
+    padding: 1em;
+    background-color: #F7F7F7;
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #999;
+    border-bottom-left-radius: 4px;
+    border-bottom-right-radius: 4px;
+    border-top-color: #DADADA;
+    color: #666;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+  </style>
+</head>
+<body>
+  <!-- This file lives in public/404.html -->
+  <div class="dialog">
+    <div>
+      <h1>The page you were looking for doesn't exist.</h1>
+      <p>You may have mistyped the address or the page may have moved.</p>
+    </div>
+    <p>If you are the application owner check the logs for more information.</p>
+  </div>
+</body>
+</html>

data/spec/dummy/public/422.html ADDED

@@ -0,0 +1,67 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>The change you wanted was rejected (422)</title>
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <style>
+  body {
+    background-color: #EFEFEF;
+    color: #2E2F30;
+    text-align: center;
+    font-family: arial, sans-serif;
+    margin: 0;
+  }
+  div.dialog {
+    width: 95%;
+    max-width: 33em;
+    margin: 4em auto 0;
+  }
+  div.dialog > div {
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #BBB;
+    border-top: #B00100 solid 4px;
+    border-top-left-radius: 9px;
+    border-top-right-radius: 9px;
+    background-color: white;
+    padding: 7px 12% 0;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+  h1 {
+    font-size: 100%;
+    color: #730E15;
+    line-height: 1.5em;
+  }
+  div.dialog > p {
+    margin: 0 0 1em;
+    padding: 1em;
+    background-color: #F7F7F7;
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #999;
+    border-bottom-left-radius: 4px;
+    border-bottom-right-radius: 4px;
+    border-top-color: #DADADA;
+    color: #666;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+  </style>
+</head>
+<body>
+  <!-- This file lives in public/422.html -->
+  <div class="dialog">
+    <div>
+      <h1>The change you wanted was rejected.</h1>
+      <p>Maybe you tried to change something you didn't have access to.</p>
+    </div>
+    <p>If you are the application owner check the logs for more information.</p>
+  </div>
+</body>
+</html>

data/spec/dummy/public/500.html ADDED

@@ -0,0 +1,66 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>We're sorry, but something went wrong (500)</title>
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <style>
+  body {
+    background-color: #EFEFEF;
+    color: #2E2F30;
+    text-align: center;
+    font-family: arial, sans-serif;
+    margin: 0;
+  }
+  div.dialog {
+    width: 95%;
+    max-width: 33em;
+    margin: 4em auto 0;
+  }
+  div.dialog > div {
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #BBB;
+    border-top: #B00100 solid 4px;
+    border-top-left-radius: 9px;
+    border-top-right-radius: 9px;
+    background-color: white;
+    padding: 7px 12% 0;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+  h1 {
+    font-size: 100%;
+    color: #730E15;
+    line-height: 1.5em;
+  }
+  div.dialog > p {
+    margin: 0 0 1em;
+    padding: 1em;
+    background-color: #F7F7F7;
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #999;
+    border-bottom-left-radius: 4px;
+    border-bottom-right-radius: 4px;
+    border-top-color: #DADADA;
+    color: #666;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+  </style>
+</head>
+<body>
+  <!-- This file lives in public/500.html -->
+  <div class="dialog">
+    <div>
+      <h1>We're sorry, but something went wrong.</h1>
+    </div>
+    <p>If you are the application owner check the logs for more information.</p>
+  </div>
+</body>
+</html>

data/spec/dummy/public/favicon.ico ADDED

File without changes

data/spec/dummy/rails_redshift_replicator_development ADDED

Binary file

data/spec/factories/rails_redshift_replicator_replications.rb ADDED

@@ -0,0 +1,31 @@
+FactoryGirl.define do
+  factory :redshift_replication, class: 'RailsRedshiftReplicator::Replication' do
+    export_format "csv"
+    replication_type "identity_replicator"
+    key "rdplibrary/replication/users/users_1442958786.csv"
+    source_table "users"
+  end
+  factory :user do
+    sequence(:login) { |n| "login_#{n}" }
+    sequence(:age)
+    factory :user_with_tags do
+      transient do
+        tags_count 5
+      end
+      after(:create) do |user, evaluator|
+        create_list(:tag, evaluator.tags_count, users: [user])
+      end
+    end
+  end
+  factory :post do
+    user
+    content "text"
+  end
+  factory :tag do
+    sequence(:name) { |n| "tag_#{n}" }
+  end
+end

data/spec/integration/rails_redshift_replicator_spec.rb ADDED

@@ -0,0 +1,148 @@
+require 'spec_helper'
+def redshift_counts(table)
+  RailsRedshiftReplicator.connection.exec("SELECT COUNT(1) FROM #{table}").first["count"].to_i
+end
+describe 'Integration Tests' do
+  before(:all) { RailsRedshiftReplicator.debug_mode = true}
+  let(:user_replicable)  { RailsRedshiftReplicator.replicables[:users] }
+  let(:post_replicable)  { RailsRedshiftReplicator.replicables[:posts] }
+  let(:habtm_replicable) { RailsRedshiftReplicator::Replicable.new(:full_replicator, source_table: :tags_users)}
+  before do
+    RailsRedshiftReplicator.add_replicable({ "tags_users" => habtm_replicable})
+  end
+  describe '.replicate' do
+    context 'replicating users' do
+      context 'without deleted records', focus: true do
+        before(:all) { recreate_users_table }
+        before do
+          5.times { create :user }
+        end
+        let(:file_manager)    { RailsRedshiftReplicator::FileManager.new }
+        let(:bucket)          { RailsRedshiftReplicator.s3_bucket_params[:bucket]}
+        let(:s3_bucket)       { Aws::S3::Bucket.new(name: bucket, client: file_manager.s3_client) }
+        let(:last_replication){ RailsRedshiftReplicator::Replication.last }
+        it 'replicates 5 users', focus: true do
+          RailsRedshiftReplicator.replicate :users
+          expect(redshift_counts('users')).to eq 5
+        end
+        context 'with auto delete from s3' do
+          before { RailsRedshiftReplicator.delete_s3_file_after_import = true }
+          it 'deletes files from s3 after replication', focus: true do
+            RailsRedshiftReplicator.replicate :users
+            expect(file_manager.s3_client.list_objects(bucket: bucket, prefix: last_replication.key).contents).to be_empty
+          end
+        end
+        context 'without auto delete from s3' do
+          before { RailsRedshiftReplicator.delete_s3_file_after_import = false }
+          it 'keeps files from s3 after replication', focus: true do
+            RailsRedshiftReplicator.replicate :users
+            expect(file_manager.s3_client.list_objects(bucket: bucket, prefix: last_replication.key).contents).not_to be_empty
+          end
+        end
+      end
+      context 'with deleted records' do
+        before(:all) { recreate_users_table }
+        before do
+          5.times { create :user }
+        end
+        it 'replicates 5 users' do
+          first_user = User.first
+          RailsRedshiftReplicator.replicate :users
+          expect(redshift_counts('users')).to eq 5
+          first_user.destroy
+          expect(User.rrr_deleter.deleted_ids.count).to eq 1
+          RailsRedshiftReplicator.replicate :users
+          expect(redshift_counts('users')).to eq 4
+        end
+      end
+      context 'with history cap' do
+        before(:all) { recreate_users_table }
+        before do
+          RailsRedshiftReplicator.history_cap = 2
+          10.times {create :redshift_replication, source_table: 'users', state: 'imported'}
+          5.times { create :user }
+        end
+        it 'caps history after import' do
+          RailsRedshiftReplicator.replicate :users
+          expect(redshift_counts('users')).to eq 5
+          expect(RailsRedshiftReplicator::Replication.count).to eq 2
+        end
+      end
+      context 'forcing full replication' do
+        before(:all) { recreate_users_table }
+        before do
+          5.times { create :user }
+        end
+        it 'replicates 5 users 2 times' do
+          RailsRedshiftReplicator.replicate :users
+          expect(redshift_counts('users')).to eq 5
+          RailsRedshiftReplicator.replicables['users'].reset_last_record
+          RailsRedshiftReplicator.replicate :users
+          expect(redshift_counts('users')).to eq 10
+        end
+      end
+    end
+    context 'replicating users and posts' do
+      before(:all) do
+        recreate_users_table
+        recreate_posts_table
+      end
+      it 'replicates 15 users and 10 posts' do
+        5.times { create :user }
+        10.times { create :post } # creates one user for each post
+        RailsRedshiftReplicator.replicate :users, :posts
+        expect(redshift_counts('users')).to eq 15
+        expect(redshift_counts('posts')).to eq 10
+      end
+      context 'replicating users and posts' do
+        before(:all) do
+          recreate_users_table
+          recreate_posts_table
+          recreate_tags_users_table
+          recreate_tags_table(:custom_tags)
+        end
+        it 'replicates 15 users, 10 posts, 25 tags_users and 25 tags' do
+          5.times { create :user_with_tags }
+          10.times { create :post } # creates one user for each post
+          RailsRedshiftReplicator.replicate :all
+          expect(redshift_counts('users')).to eq 15
+          expect(redshift_counts('posts')).to eq 10
+          expect(redshift_counts('tags_users')).to eq 25
+          expect(redshift_counts('custom_tags')).to eq 25
+        end
+      end
+    end
+  end
+  describe ".export" do
+    before {User;Post} # Load Models
+    it 'calls export for replicables' do
+      expect(user_replicable).to receive(:export)
+      expect(post_replicable).to receive(:export)
+      RailsRedshiftReplicator.export(:users, :posts)
+    end
+    it 'does not call export for non replicables' do
+      expect_any_instance_of(RailsRedshiftReplicator::Replicable).not_to receive(:export)
+      RailsRedshiftReplicator.export(:non_replicable)
+    end
+  end
+  describe ".import" do
+    before {User;Post} # Load Models
+    it 'calls import for replicables' do
+      expect(user_replicable).to receive(:import)
+      expect(post_replicable).to receive(:import)
+      RailsRedshiftReplicator.import(:users, :posts)
+    end
+    it 'does not call import for non replicables' do
+      expect_any_instance_of(RailsRedshiftReplicator::Replicable).not_to receive(:import)
+      RailsRedshiftReplicator.import(:non_replicable)
+    end
+  end
+end

data/spec/integration/setup_spec.rb ADDED

@@ -0,0 +1,149 @@
+require 'spec_helper'
+class Mylogger < Logger;end
+describe 'Setup Tests', :broken do
+  describe ".setup" do
+    context 'with default configuration' do
+      after(:all) { Dotenv.load File.expand_path("../spec/dummy/.env", __FILE__); RailsRedshiftReplicator.reload }
+      let(:env) do
+        {
+          RRR_REDSHIFT_HOST: 'default-host',
+          RRR_REDSHIFT_DATABASE: 'default-database',
+          RRR_REDSHIFT_PORT: 'default-port',
+          RRR_REDSHIFT_USER: 'default-user',
+          RRR_REDSHIFT_PASSWORD: 'default-password',
+          RRR_AWS_ACCESS_KEY_ID: 'default-key',
+          RRR_AWS_SECRET_ACCESS_KEY: 'default-secret',
+          RRR_REPLICATION_REGION: 'default-region',
+          RRR_REPLICATION_BUCKET: 'default-bucket',
+          RRR_REPLICATION_PREFIX: 'default-prefix'
+        }
+      end
+      around do |example|
+        ClimateControl.modify(env) do
+          RailsRedshiftReplicator.reload
+          example.run
+        end
+      end
+      it 'uses default redshift_connection_params' do
+        expect(RailsRedshiftReplicator.redshift_connection_params).to eq({
+          host: 'default-host',
+          dbname: 'default-database',
+          port: 'default-port',
+          user: 'default-user',
+          password: 'default-password'
+        })
+      end
+      it 'uses default aws_credentials' do
+        expect(RailsRedshiftReplicator.aws_credentials).to eq({
+          key: 'default-key',
+          secret: 'default-secret'
+        })
+      end
+      it 'uses default s3_bucket_params' do
+        expect(RailsRedshiftReplicator.s3_bucket_params).to eq({
+          region: 'default-region',
+          bucket: 'default-bucket',
+          prefix: 'default-prefix'
+        })
+      end
+      it 'uses default redshift_slices' do
+        expect(RailsRedshiftReplicator.redshift_slices).to eq 1
+      end
+      it 'uses default local_replication_path' do
+        expect(RailsRedshiftReplicator.local_replication_path).to eq '/tmp'
+      end
+      it 'uses default debug_mode' do
+        expect(RailsRedshiftReplicator.debug_mode).to be_falsy
+      end
+      it 'uses default history_cap' do
+        expect(RailsRedshiftReplicator.history_cap).to be_nil
+      end
+    end
+    context 'when changing configuration' do
+      after(:all) { Dotenv.load File.expand_path("../spec/dummy/.env", __FILE__); RailsRedshiftReplicator.reload }
+      before do
+        RailsRedshiftReplicator.setup do |config|
+          config.redshift_connection_params = {
+            host: 'redshift-host',
+            dbname: 'database-name',
+            port: 'database-port',
+            user: 'database-user',
+            password: 'database-password'
+          }
+          config.aws_credentials = {
+            key: 'aws-key',
+            secret: 'aws-secret'
+          }
+          config.s3_bucket_params = {
+            region: 's3-region',
+            bucket: 's3-bucket'
+          }
+          config.redshift_slices = '2'
+          config.local_replication_path = '/local-tmp'
+          config.debug_mode = true
+          config.history_cap = 3
+          config.logger = Mylogger.new(STDOUT)
+        end
+      end
+      it 'defines custom logger' do
+        expect(RailsRedshiftReplicator.logger).to be_a Mylogger
+      end
+      it 'defines redshift_connection_params' do
+        expect(RailsRedshiftReplicator.redshift_connection_params).to eq({
+          host: 'redshift-host',
+          dbname: 'database-name',
+          port: 'database-port',
+          user: 'database-user',
+          password: 'database-password'
+        })
+      end
+      it 'defines aws_credentials' do
+        expect(RailsRedshiftReplicator.aws_credentials).to eq({
+          key: 'aws-key',
+          secret: 'aws-secret'
+        })
+      end
+      it 'defines s3_bucket_params' do
+        expect(RailsRedshiftReplicator.s3_bucket_params).to eq({
+          region: 's3-region',
+          bucket: 's3-bucket'
+        })
+      end
+      it 'defines redshift_slices' do
+        expect(RailsRedshiftReplicator.redshift_slices).to eq '2'
+      end
+      it 'defines local_replication_path' do
+        expect(RailsRedshiftReplicator.local_replication_path).to eq '/local-tmp'
+      end
+      it 'defines debug_mode' do
+        expect(RailsRedshiftReplicator.debug_mode).to eq true
+        expect(RailsRedshiftReplicator.logger.level).to eq Logger::DEBUG
+      end
+      it 'defines history_cap' do
+        expect(RailsRedshiftReplicator.history_cap).to eq 3
+      end
+      it "changes module's configuration parameters" do
+        expect { RailsRedshiftReplicator.setup {|config| config.redshift_slices = 10} }.to change(RailsRedshiftReplicator, :redshift_slices).to(10)
+      end
+    end
+  end
+end