rails_redshift_replicator 0.0.1

data/spec/dummy/config/environments/test.rb

@@ -0,0 +1,42 @@
+Rails.application.configure do
+  # Settings specified here will take precedence over those in config/application.rb.
+
+  # The test environment is used exclusively to run your application's
+  # test suite. You never need to work with it otherwise. Remember that
+  # your test database is "scratch space" for the test suite and is wiped
+  # and recreated between test runs. Don't rely on the data there!
+  config.cache_classes = true
+
+  # Do not eager load code on boot. This avoids loading your whole application
+  # just for the purpose of running a single test. If you are using a tool that
+  # preloads Rails for running tests, you may have to set it to true.
+  config.eager_load = true
+
+  # Configure static file server for tests with Cache-Control for performance.
+  config.serve_static_files   = true
+  config.static_cache_control = 'public, max-age=3600'
+
+  # Show full error reports and disable caching.
+  config.consider_all_requests_local       = true
+  config.action_controller.perform_caching = false
+
+  # Raise exceptions instead of rendering exception templates.
+  config.action_dispatch.show_exceptions = false
+
+  # Disable request forgery protection in test environment.
+  config.action_controller.allow_forgery_protection = false
+
+  # Tell Action Mailer not to deliver emails to the real world.
+  # The :test delivery method accumulates sent emails in the
+  # ActionMailer::Base.deliveries array.
+  config.action_mailer.delivery_method = :test
+
+  # Randomize the order test cases are executed.
+  config.active_support.test_order = :random
+
+  # Print deprecation notices to the stderr.
+  config.active_support.deprecation = :stderr
+
+  # Raises error for missing translations
+  # config.action_view.raise_on_missing_translations = true
+end

data/spec/dummy/config/initializers/assets.rb

@@ -0,0 +1,11 @@
+# Be sure to restart your server when you modify this file.
+
+# Version of your assets, change this if you want to expire all your assets.
+Rails.application.config.assets.version = '1.0'
+
+# Add additional assets to the asset load path
+# Rails.application.config.assets.paths << Emoji.images_path
+
+# Precompile additional assets.
+# application.js, application.css, and all non-JS/CSS in app/assets folder are already added.
+# Rails.application.config.assets.precompile += %w( search.js )

data/spec/dummy/config/initializers/backtrace_silencers.rb

@@ -0,0 +1,7 @@
+# Be sure to restart your server when you modify this file.
+
+# You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces.
+# Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ }
+
+# You can also remove all the silencers if you're trying to debug a problem that might stem from framework code.
+# Rails.backtrace_cleaner.remove_silencers!

data/spec/dummy/config/initializers/cookies_serializer.rb

@@ -0,0 +1,3 @@
+# Be sure to restart your server when you modify this file.
+
+Rails.application.config.action_dispatch.cookies_serializer = :json

data/spec/dummy/config/initializers/filter_parameter_logging.rb

@@ -0,0 +1,4 @@
+# Be sure to restart your server when you modify this file.
+
+# Configure sensitive parameters which will be filtered from the log file.
+Rails.application.config.filter_parameters += [:password]

data/spec/dummy/config/initializers/inflections.rb

@@ -0,0 +1,16 @@
+# Be sure to restart your server when you modify this file.
+
+# Add new inflection rules using the following format. Inflections
+# are locale specific, and you may define rules for as many different
+# locales as you wish. All of these examples are active by default:
+# ActiveSupport::Inflector.inflections(:en) do |inflect|
+#   inflect.plural /^(ox)$/i, '\1en'
+#   inflect.singular /^(ox)en/i, '\1'
+#   inflect.irregular 'person', 'people'
+#   inflect.uncountable %w( fish sheep )
+# end
+
+# These inflection rules are supported but not enabled by default:
+# ActiveSupport::Inflector.inflections(:en) do |inflect|
+#   inflect.acronym 'RESTful'
+# end

data/spec/dummy/config/initializers/mime_types.rb

@@ -0,0 +1,4 @@
+# Be sure to restart your server when you modify this file.
+
+# Add new mime types for use in respond_to blocks:
+# Mime::Type.register "text/richtext", :rtf

data/spec/dummy/config/initializers/rails_redshift_replicator.rb

@@ -0,0 +1,59 @@
+RailsRedshiftReplicator.setup do |config|
+  # RRR already provides a logger pointing to STDOUT, but you can point it to your own logger.
+  # Just be sure to make it inherit from RailsRedshiftReplicator::RLogger or you will loose
+  # the notifications feature.
+  # config.logger = MyLogger.new
+
+  # Connection parameters for Redshift. Defaults to environment variables.
+  config.redshift_connection_params = {
+    host: ENV['RRR_REDSHIFT_HOST'],
+    dbname: ENV['RRR_REDSHIFT_DATABASE'],
+    port: ENV['RRR_REDSHIFT_PORT'],
+    user: ENV['RRR_REDSHIFT_USER'],
+    password: ENV['RRR_REDSHIFT_PASSWORD']
+  }
+
+  # AWS S3 Replication bucket credentials. Defaults to environment variables.
+  config.aws_credentials = {
+    key: ENV['RRR_AWS_ACCESS_KEY_ID'],
+    secret: ENV['RRR_AWS_SECRET_ACCESS_KEY']
+  }
+
+  # AWS S3 replication bucket parameters.
+  # region defaults to environment variable or US East (N. Virginia)
+  # bucket defaults to environment variable
+  config.s3_bucket_params = {
+    region: (ENV['RRR_REPLICATION_REGION'] || 'us-east-1'),
+    bucket: ENV['RRR_REPLICATION_BUCKET'],
+    prefix: ENV['RRR_REPLICATION_PREFIX']
+  }
+
+  # Number of slices available on Redshift cluster. Used to split export files. Defaults to 1.
+  # see [http://docs.aws.amazon.com/redshift/latest/dg/t_splitting-data-files.html]
+  config.redshift_slices = 1
+
+  # Folder to store temporary replication files until the S3 upload. Defaults to /tmp
+  config.local_replication_path = '/tmp'
+
+  # Command or path to executable that splits files
+  config.split_command = 'split'
+
+  # Command or path to executable that compresses files to gzip
+  config.gzip_command = 'gzip'
+
+  # Enable debug mode to output messages to STDOUT. Default to false
+  config.debug_mode = false
+
+  # Defines how many replication records are kept in history. Default to nil keeping full history.
+  config.history_cap = nil
+
+  # Preferred format for export file
+  config.preferred_format = 'csv'
+
+  # Maximum number of retries for a replication before cancelling and starting another
+  config.max_retries = 5
+
+  # If deletes should be tracked and propagated to redshift
+  # Take a look at the "A word on tracking deletions" section
+  config.enable_delete_tracking = true
+end

data/spec/dummy/config/initializers/session_store.rb

@@ -0,0 +1,3 @@
+# Be sure to restart your server when you modify this file.
+
+Rails.application.config.session_store :cookie_store, key: '_dummy_session'

data/spec/dummy/config/initializers/wrap_parameters.rb

@@ -0,0 +1,14 @@
+# Be sure to restart your server when you modify this file.
+
+# This file contains settings for ActionController::ParamsWrapper which
+# is enabled by default.
+
+# Enable parameter wrapping for JSON. You can disable this by setting :format to an empty array.
+ActiveSupport.on_load(:action_controller) do
+  wrap_parameters format: [:json] if respond_to?(:wrap_parameters)
+end
+
+# To enable root element in JSON for ActiveRecord objects.
+# ActiveSupport.on_load(:active_record) do
+#  self.include_root_in_json = true
+# end

data/spec/dummy/config/locales/en.yml

@@ -0,0 +1,23 @@
+# Files in the config/locales directory are used for internationalization
+# and are automatically loaded by Rails. If you want to use locales other
+# than English, add the necessary files in this directory.
+#
+# To use the locales, use `I18n.t`:
+#
+#     I18n.t 'hello'
+#
+# In views, this is aliased to just `t`:
+#
+#     <%= t('hello') %>
+#
+# To use a different locale, set it with `I18n.locale`:
+#
+#     I18n.locale = :es
+#
+# This would use the information in config/locales/es.yml.
+#
+# To learn more, please read the Rails Internationalization guide
+# available at http://guides.rubyonrails.org/i18n.html.
+
+en:
+  hello: "Hello world"

data/spec/dummy/config/locales/rails_redshift_replicator.en.yml

@@ -0,0 +1,19 @@
+en:
+  rails_redshift_replicator:
+    uploading_notice: "uploading %{file} to %{key}"
+    gzip_notice: "gzip %{file} to %{gzip_file} with command: %{command}"
+    exporting_results: "exporting %{counts} results"
+    importing_file: "importing %{file} to %{target_table}"
+    no_new_records: "No new records for %{table_name} to export"
+    missing_indexes: "'%{replication_field}' index is missing on table '%{table_name}'. Export performance can be improved by adding indexes to replication fields"
+    missing_table: "Coudn't find table  %{table_name} on Redshift"
+    missing_replicator_type: Replication type not implemented
+    must_specify_tables: "Specify tables to export or use :all for all eligible tables"
+    replicable_added: "Added replicable for table '%{table_name}'"
+    executing_query: "Executing query with %{adapter}: %{sql}"
+    nothing_to_import: "No replication for table '%{table_name}' was pending import"
+    table_not_replicable: "The table '%{table_name}' is not registered as replicable"
+    max_retries_reached: "The replication #%{id} for table '%{table_name}' reached the maximum number of retries"
+    resuming_replication: "Resuming %{action} replication for table '%{table_name}', which was on %{state} state"
+    propagating_deletes: "Propagating %{count} deleted records from table '%{table_name}'"
+    delete_propagation_error: "Failed to propagate %{count} deleted records from table '%{table_name}'"

data/spec/dummy/config/routes.rb

@@ -0,0 +1,4 @@
+Rails.application.routes.draw do
+
+  mount RailsRedshiftReplicator::Engine => "/rails_redshift_replicator"
+end

data/spec/dummy/config/secrets.yml

@@ -0,0 +1,22 @@
+# Be sure to restart your server when you modify this file.
+
+# Your secret key is used for verifying the integrity of signed cookies.
+# If you change this key, all old signed cookies will become invalid!
+
+# Make sure the secret is at least 30 characters and all random,
+# no regular words or you'll be exposed to dictionary attacks.
+# You can use `rake secret` to generate a secure secret key.
+
+# Make sure the secrets in this file are kept private
+# if you're sharing your code publicly.
+
+development:
+  secret_key_base: 78a339d8e8dee9f70fd66fe18c31cf9e86f3fef9304059f98d18f6c913d5f4d62583f015a4dd523bf6341138f76e310536b073e866e3b291a283f6b4ae5c4fdc
+
+test:
+  secret_key_base: c67fff4272bb077f550e70f5027315a620eafb1d1f1e7ec27790ea76142b3794658fcb07600fdd1883c44b7929968d493142091612fcb1d4ea5783d424db2104
+
+# Do not keep production secrets in the repository,
+# instead read values from the environment.
+production:
+  secret_key_base: <%= ENV["SECRET_KEY_BASE"] %>

data/spec/dummy/db/migrate/20160504120421_create_test_tables.rb

@@ -0,0 +1,40 @@
+# Migration 20160504120421
+class CreateTestTables < ActiveRecord::Migration
+  def up
+    create_table :users do |t|
+      t.string :login
+      t.string :password
+      t.integer :age
+      t.boolean :confirmed
+      t.timestamps null: false
+    end
+    add_index :users, [:login, :age]
+    
+    create_table :posts do |t|
+      t.belongs_to :user
+      t.text :content
+      t.timestamps null: false
+    end
+    add_index :posts, [:user_id, :updated_at]
+    
+    create_table :tags do |t|
+      t.string :name
+      t.timestamps null: false
+    end
+    add_index :tags, [:name, :updated_at]
+    
+    create_table :tags_users, id: false do |t|
+      t.belongs_to :user
+      t.belongs_to :tag
+    end
+    add_index :tags_users, [:user_id, :tag_id]
+
+  end
+
+  def down
+    drop_table :users
+    drop_table :posts
+    drop_table :tags
+    drop_table :tags_users
+  end
+end

data/spec/dummy/db/migrate/20160509225445_create_triggers_posts_delete_or_tags_delete_or_users_delete.rb

@@ -0,0 +1,33 @@
+# This migration was auto-generated via `rake db:generate_trigger_migration'.
+# While you can edit this file, any changes you make to the definitions here
+# will be undone by the next auto-generated trigger migration.
+
+class CreateTriggersPostsDeleteOrTagsDeleteOrUsersDelete < ActiveRecord::Migration
+  def up
+    create_trigger("posts_after_delete_row_tr", :generated => true, :compatibility => 1).
+        on("posts").
+        after(:delete) do
+      "INSERT INTO rails_redshift_replicator_deleted_ids(source_table, object_id) VALUES('posts', OLD.id);"
+    end
+
+    create_trigger("tags_after_delete_row_tr", :generated => true, :compatibility => 1).
+        on("tags").
+        after(:delete) do
+      "INSERT INTO rails_redshift_replicator_deleted_ids(source_table, object_id) VALUES('tags', OLD.id);"
+    end
+
+    create_trigger("users_after_delete_row_tr", :generated => true, :compatibility => 1).
+        on("users").
+        after(:delete) do
+      "INSERT INTO rails_redshift_replicator_deleted_ids(source_table, object_id) VALUES('users', OLD.id);"
+    end
+  end
+
+  def down
+    drop_trigger("posts_after_delete_row_tr", "posts", :generated => true)
+
+    drop_trigger("tags_after_delete_row_tr", "tags", :generated => true)
+
+    drop_trigger("users_after_delete_row_tr", "users", :generated => true)
+  end
+end

data/spec/dummy/db/migrate/20160511000937_create_rails_redshift_replicator_replications.rails_redshift_replicator.rb

@@ -0,0 +1,25 @@
+# This migration comes from rails_redshift_replicator (originally 20160503214955)
+# 20160503214955
+class CreateRailsRedshiftReplicatorReplications < ActiveRecord::Migration
+  def change
+    create_table :rails_redshift_replicator_replications do |t|
+      t.string   "replication_type"
+      t.string   "key"
+      t.string   "state",            :default => "enqueued"
+      t.string   "last_record"
+      t.integer  "retries", default: 0
+      t.text     "last_error"
+      t.string   "source_table"
+      t.string   "target_table"
+      t.integer  "slices"
+      t.string   "first_record"
+      t.integer  "record_count"
+      t.string   "export_format"
+      t.integer  "export_duration"
+      t.integer  "upload_duration"
+      t.integer  "import_duration"
+      t.datetime "created_at",                               :null => false
+      t.datetime "updated_at",                               :null => false
+    end
+  end
+end

data/spec/dummy/db/migrate/20160511000938_create_table_rails_redshift_replicator_deleted_ids.rails_redshift_replicator.rb

@@ -0,0 +1,9 @@
+# This migration comes from rails_redshift_replicator (originally 20160509193335)
+class CreateTableRailsRedshiftReplicatorDeletedIds < ActiveRecord::Migration
+  def change
+    create_table :rails_redshift_replicator_deleted_ids, id: false do |t|
+      t.string :source_table, index: true
+      t.integer :object_id
+    end
+  end
+end

data/spec/dummy/db/schema.rb

@@ -0,0 +1,99 @@
+# encoding: UTF-8
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# Note that this schema.rb definition is the authoritative source for your
+# database schema. If you need to create the application database on another
+# system, you should be using db:schema:load, not running all the migrations
+# from scratch. The latter is a flawed and unsustainable approach (the more migrations
+# you'll amass, the slower it'll run and the greater likelihood for issues).
+#
+# It's strongly recommended that you check this file into your version control system.
+
+ActiveRecord::Schema.define(version: 20160511000938) do
+
+  # These are extensions that must be enabled in order to support this database
+  enable_extension "plpgsql"
+
+  create_table "posts", force: :cascade do |t|
+    t.integer  "user_id"
+    t.text     "content"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  add_index "posts", ["user_id", "updated_at"], name: "index_posts_on_user_id_and_updated_at", using: :btree
+
+  create_table "rails_redshift_replicator_deleted_ids", id: false, force: :cascade do |t|
+    t.string  "source_table"
+    t.integer "object_id"
+  end
+
+  add_index "rails_redshift_replicator_deleted_ids", ["source_table"], name: "index_rails_redshift_replicator_deleted_ids_on_source_table", using: :btree
+
+  create_table "rails_redshift_replicator_replications", force: :cascade do |t|
+    t.string   "replication_type"
+    t.string   "key"
+    t.string   "state",            default: "enqueued"
+    t.string   "last_record"
+    t.integer  "retries",          default: 0
+    t.text     "last_error"
+    t.string   "source_table"
+    t.string   "target_table"
+    t.integer  "slices"
+    t.string   "first_record"
+    t.integer  "record_count"
+    t.string   "export_format"
+    t.integer  "export_duration"
+    t.integer  "upload_duration"
+    t.integer  "import_duration"
+    t.datetime "created_at",                            null: false
+    t.datetime "updated_at",                            null: false
+  end
+
+  create_table "tags", force: :cascade do |t|
+    t.string   "name"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  add_index "tags", ["name", "updated_at"], name: "index_tags_on_name_and_updated_at", using: :btree
+
+  create_table "tags_users", id: false, force: :cascade do |t|
+    t.integer "user_id"
+    t.integer "tag_id"
+  end
+
+  add_index "tags_users", ["user_id", "tag_id"], name: "index_tags_users_on_user_id_and_tag_id", using: :btree
+
+  create_table "users", force: :cascade do |t|
+    t.string   "login"
+    t.string   "password"
+    t.integer  "age"
+    t.boolean  "confirmed"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  add_index "users", ["login", "age"], name: "index_users_on_login_and_age", using: :btree
+
+  create_trigger("posts_after_delete_row_tr", :generated => true, :compatibility => 1).
+      on("posts").
+      after(:delete) do
+    "INSERT INTO rails_redshift_replicator_deleted_ids(source_table, object_id) VALUES('posts', OLD.id);"
+  end
+
+  create_trigger("tags_after_delete_row_tr", :generated => true, :compatibility => 1).
+      on("tags").
+      after(:delete) do
+    "INSERT INTO rails_redshift_replicator_deleted_ids(source_table, object_id) VALUES('tags', OLD.id);"
+  end
+
+  create_trigger("users_after_delete_row_tr", :generated => true, :compatibility => 1).
+      on("users").
+      after(:delete) do
+    "INSERT INTO rails_redshift_replicator_deleted_ids(source_table, object_id) VALUES('users', OLD.id);"
+  end
+
+end