rails_redshift_replicator 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: 091d07d2dc935184033dc7be33159df8b0eaca82
+  data.tar.gz: 16cc2eac454114f7b1e2ce146039e04b032a912d
+SHA512:
+  metadata.gz: e14f322cb08776821dc29c403ac176dd4546dd85c94251e9d6d0bf03eedd48a97a90493e03eac43b3ff9e3b0cdc0661d8549b3a7facbd0193a6e467874e293a6
+  data.tar.gz: 79c1f83dc575f1bb80428122770a8aab0dff9b91534a207013882944cc850c0ae2c81872f95877f3b9ed9dd5511366c1c0699d3b7d9b6bb4d7d63c22c8dd900e

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2016 Alexandre Angelim
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,3 @@
+= RailsRedshiftReplicator
+
+This project rocks and uses MIT-LICENSE.

data/Rakefile

@@ -0,0 +1,34 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'RailsRedshiftReplicator'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+APP_RAKEFILE = File.expand_path("../spec/dummy/Rakefile", __FILE__)
+load 'rails/tasks/engine.rake'
+
+
+load 'rails/tasks/statistics.rake'
+
+
+Bundler::GemHelper.install_tasks
+
+Dir[File.join(File.dirname(__FILE__), 'tasks/**/*.rake')].each {|f| load f }
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+
+desc "Run all specs in spec directory (excluding plugin specs)"
+RSpec::Core::RakeTask.new(:spec => 'app:db:test:prepare')
+
+task :default => :spec

data/app/assets/javascripts/rails_redshift_replicator/application.js

@@ -0,0 +1,13 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require_tree .

data/app/assets/stylesheets/rails_redshift_replicator/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any styles
+ * defined in the other CSS/SCSS files in this directory. It is generally better to create a new
+ * file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/app/controllers/rails_redshift_replicator/application_controller.rb

@@ -0,0 +1,5 @@
+module RailsRedshiftReplicator
+  class ApplicationController < ActionController::Base
+    protect_from_forgery with: :exception
+  end
+end

data/app/helpers/rails_redshift_replicator/application_helper.rb

@@ -0,0 +1,4 @@
+module RailsRedshiftReplicator
+  module ApplicationHelper
+  end
+end

data/app/models/rails_redshift_replicator/replication.rb

@@ -0,0 +1,98 @@
+module RailsRedshiftReplicator
+  class Replication < ActiveRecord::Base
+    STATES = %w(enqueued exporting exported uploading uploaded importing imported canceled)
+    FORMATS = %w(gzip csv)
+
+    # @return [Array] ids from source_table to delete on the next replication.
+    serialize :ids_to_delete, Array
+
+    validates :state, inclusion: { in: STATES }
+    validates :export_format, inclusion: { in: FORMATS }
+    validates_presence_of :replication_type, :key, :source_table, :target_table
+    before_validation :setup_target_table
+
+    # Clears the error column
+    def clear_errors!
+      update_attributes last_error: nil
+    end
+
+    # If replication is on an error state
+    # @return [true, false] if has error
+    def error?
+      last_error.present?
+    end
+
+    # Initializes target table if it is blank
+    def setup_target_table
+      self.target_table = source_table if target_table.blank?
+    end
+
+    # Cancels the replication
+    def cancel!
+      update_attribute :state, 'canceled'
+    end
+
+    # @return [RailsRedshiftReplicator::Replicable] replicable for this model/table
+    def replicable
+      RailsRedshiftReplicator.replicables[source_table]
+    end
+    
+    scope :from_table, ->(table) { where(source_table: Array(table).map(&:to_s)).where.not(state: 'canceled') }
+    scope :with_state, ->(state) { where(state: state) }
+    scope :older_than, ->(table, cap) { where("id < ?", cap_id(table, cap)) }
+
+    def self.cap_id(table, cap)
+      return unless cap
+      where(source_table: table).order("id desc").limit(cap).pluck(:id)[cap-1]
+    end
+
+    # Builds helper methods to identify export format.
+    # @return [true, false] if export is in a given format.
+    FORMATS.each do |format|
+      # @example
+      #   self.format = "gzip"
+      #   self.csv? #=> false
+      define_method "#{format}?" do
+        export_format == format
+      end
+    end
+
+    STATES.each do |state|
+      # Builds methods to change replication to a given state persisting changes.
+      # @example
+      #   self.uploaded! upload_duration: 10
+      #   self.state #=> "uploaded"
+      #   self.upload_duration #=> 10
+      # @return [Time] current time
+      define_method "#{state}!" do |options = {}|
+        update_attributes({ state: state }.merge(options))
+        return Time.now
+      end
+
+      # Builds methods to change replication to a given state _without_ persisting changes.
+      # @example
+      #   self.new_record? = true
+      #   self.uploaded upload_duration: 10
+      #   self.state #=> "uploaded"
+      #   self.upload_duration #=> 10
+      # @return [Time] current time
+      define_method "#{state}" do |options = {}|
+        assign_attributes({ state: state }.merge(options))
+        return Time.now
+      end
+
+      # Builds helper methods to identify the current state.
+      # @example
+      #   self.state = "uploaded"
+      #   self.uploaded? #=> true
+      define_method "#{state}?" do
+        self.state == state
+      end
+
+      # Scopes
+      # @example
+      #   scope :error, -> { where state: "error" }
+      scope state, -> { where state: state }
+    end
+  end
+end

data/app/views/layouts/rails_redshift_replicator/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>RailsRedshiftReplicator</title>
+  <%= stylesheet_link_tag    "rails_redshift_replicator/application", media: "all" %>
+  <%= javascript_include_tag "rails_redshift_replicator/application" %>
+  <%= csrf_meta_tags %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/config/locales/rails_redshift_replicator.en.yml

@@ -0,0 +1,20 @@
+en:
+  rails_redshift_replicator:
+    uploading_notice: "uploading %{file} to %{key}"
+    gzip_notice: "gzip %{file} to %{gzip_file} with command: %{command}"
+    exporting_results: "exporting %{counts} results"
+    importing_file: "importing %{file} to %{target_table}"
+    no_new_records: "No new records for %{table_name} to export"
+    missing_indexes: "'%{replication_field}' index is missing on table '%{table_name}'. Export performance can be improved by adding indexes to replication fields"
+    missing_table: "Coudn't find table  %{table_name} on Redshift"
+    missing_replicator_type: Replication type not implemented
+    must_specify_tables: "Specify tables to export or use :all for all eligible tables"
+    replicable_added: "Added replicable for table '%{table_name}'"
+    executing_query: "Executing query with %{adapter}: %{sql}"
+    nothing_to_import: "No replication for table '%{table_name}' was pending import"
+    table_not_replicable: "The table '%{table_name}' is not registered as replicable"
+    max_retries_reached: "The replication #%{id} for table '%{table_name}' reached the maximum number of retries"
+    resuming_replication: "Resuming %{action} replication for table '%{table_name}', which was on %{state} state"
+    propagating_deletes: "Propagating %{count} deleted records from table '%{table_name}'"
+    delete_propagation_error: "Failed to propagate %{count} deleted records from table '%{table_name}'"
+    deleting_file: "Deleting file on s3: %{key}"

data/config/routes.rb

@@ -0,0 +1,2 @@
+RailsRedshiftReplicator::Engine.routes.draw do
+end

data/db/migrate/20160503214955_create_rails_redshift_replicator_replications.rb

@@ -0,0 +1,24 @@
+# 20160503214955
+class CreateRailsRedshiftReplicatorReplications < ActiveRecord::Migration
+  def change
+    create_table :rails_redshift_replicator_replications do |t|
+      t.string   "replication_type"
+      t.string   "key"
+      t.string   "state",            :default => "enqueued"
+      t.string   "last_record"
+      t.integer  "retries", default: 0
+      t.text     "last_error"
+      t.string   "source_table"
+      t.string   "target_table"
+      t.integer  "slices"
+      t.string   "first_record"
+      t.integer  "record_count"
+      t.string   "export_format"
+      t.integer  "export_duration"
+      t.integer  "upload_duration"
+      t.integer  "import_duration"
+      t.datetime "created_at",                               :null => false
+      t.datetime "updated_at",                               :null => false
+    end
+  end
+end

data/db/migrate/20160509193335_create_table_rails_redshift_replicator_deleted_ids.rb

@@ -0,0 +1,8 @@
+class CreateTableRailsRedshiftReplicatorDeletedIds < ActiveRecord::Migration
+  def change
+    create_table :rails_redshift_replicator_deleted_ids, id: false do |t|
+      t.string :source_table, index: true
+      t.integer :object_id
+    end
+  end
+end

data/lib/generators/rails_redshift_replicator/install_generator.rb

@@ -0,0 +1,25 @@
+require 'rails/generators/base'
+require 'securerandom'
+
+module RailsRedshiftReplicator
+  module Generators
+
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path("../../templates", __FILE__)
+
+      desc "Creates a RRR initializer and copy locale files to your application."
+
+      def copy_initializer
+        template "rails_redshift_replicator.rb", "config/initializers/rails_redshift_replicator.rb"
+      end
+
+      def copy_locale
+        copy_file "../../../config/locales/rails_redshift_replicator.en.yml", "config/locales/rails_redshift_replicator.en.yml"
+      end
+
+      def rails_4?
+        Rails::VERSION::MAJOR == 4
+      end
+    end
+  end
+end

data/lib/generators/templates/rails_redshift_replicator.rb

@@ -0,0 +1,74 @@
+RailsRedshiftReplicator.setup do |config|
+  # RRR already provides a logger pointing to STDOUT, but you can point it to your own logger.
+  # Just be sure to make it inherit from RailsRedshiftReplicator::RLogger or you will loose
+  # the notifications feature.
+  # config.logger = MyLogger.new
+
+  # Connection parameters for Redshift. Defaults to environment variables.
+  config.redshift_connection_params = {
+    host: ENV['RRR_REDSHIFT_HOST'],
+    dbname: ENV['RRR_REDSHIFT_DATABASE'],
+    port: ENV['RRR_REDSHIFT_PORT'],
+    user: ENV['RRR_REDSHIFT_USER'],
+    password: ENV['RRR_REDSHIFT_PASSWORD']
+  }
+
+  # AWS S3 Replication bucket credentials. Defaults to environment variables.
+  config.aws_credentials = {
+    key: ENV['RRR_AWS_ACCESS_KEY_ID'],
+    secret: ENV['RRR_AWS_SECRET_ACCESS_KEY']
+  }
+
+  # AWS S3 replication bucket parameters.
+  # region defaults to environment variable or US East (N. Virginia)
+  # bucket defaults to environment variable
+  config.s3_bucket_params = {
+    region: (ENV['RRR_REPLICATION_REGION'] || 'us-east-1'),
+    bucket: ENV['RRR_REPLICATION_BUCKET'],
+    prefix: ENV['RRR_REPLICATION_PREFIX']
+  }
+
+  # see [http://docs.aws.amazon.com/redshift/latest/dg/r_COPY.html]
+  # You can add other keys aside from changing these.
+  # The keys won't be used on the copy commands. Just their values.
+  # To remove one of the defaults, set it to nil.
+  # @example:
+  #   @@copy_options = {
+  #     statupdate: nil,
+  #   }
+  config.copy_options = {
+    statupdate: 'STATUPDATE TRUE',
+    acceptinvchars: 'ACCEPTINVCHARS',
+    empty: 'EMPTYASNULL',
+    truncate: 'TRUNCATECOLUMNS'
+  }
+
+  # Number of slices available on Redshift cluster. Used to split export files. Defaults to 1.
+  # see [http://docs.aws.amazon.com/redshift/latest/dg/t_splitting-data-files.html]
+  config.redshift_slices = 1
+
+  # Folder to store temporary replication files until the S3 upload. Defaults to /tmp
+  config.local_replication_path = '/tmp'
+
+  # Command or path to executable that splits files
+  config.split_command = 'split'
+
+  # Command or path to executable that compresses files to gzip
+  config.gzip_command = 'gzip'
+
+  # Enable debug mode to output messages to STDOUT. Default to false
+  config.debug_mode = false
+
+  # Defines how many replication records are kept in history. Default to nil keeping full history.
+  config.history_cap = nil
+
+  # Preferred format for export file
+  config.preferred_format = 'csv'
+
+  # Maximum number of retries for a replication before cancelling and starting another
+  config.max_retries = 5
+
+  # If deletes should be tracked and propagated to redshift
+  # Take a look at the "A word on tracking deletions" section
+  config.enable_delete_tracking = false
+end

data/lib/rails_redshift_replicator.rb

@@ -0,0 +1,229 @@
+require 'hair_trigger'
+require 'active_support'
+require "rails_redshift_replicator/engine"
+require 'rails_redshift_replicator/model/extension'
+require 'rails_redshift_replicator/model/hair_trigger_extension'
+require 'rails_redshift_replicator/replicable'
+require 'rails_redshift_replicator/deleter'
+require 'rails_redshift_replicator/rlogger'
+require 'rails_redshift_replicator/file_manager'
+
+require 'rails_redshift_replicator/exporters/base'
+require 'rails_redshift_replicator/exporters/identity_replicator'
+require 'rails_redshift_replicator/exporters/timed_replicator'
+require 'rails_redshift_replicator/exporters/full_replicator'
+
+require 'rails_redshift_replicator/importers/base'
+require 'rails_redshift_replicator/importers/identity_replicator'
+require 'rails_redshift_replicator/importers/timed_replicator'
+require 'rails_redshift_replicator/importers/full_replicator'
+
+require 'rails_redshift_replicator/tools/analyze'
+require 'rails_redshift_replicator/tools/vacuum'
+
+
+module RailsRedshiftReplicator
+  mattr_accessor :replicables, :logger, :redshift_connection_params, :aws_credentials, :s3_bucket_params,
+                 :redshift_slices, :local_replication_path, :debug_mode, :history_cap,
+                 :split_command, :gzip_command, :preferred_format, :max_retries, :enable_delete_tracking,
+                 :delete_s3_file_after_import, :copy_options
+
+  class << self
+
+    # @note Useful for testing
+    def define_defaults
+      @@replicables = {}.with_indifferent_access
+      @@logger = RLogger.new(STDOUT).tap{ |l| l.level = Logger::WARN }
+      
+      # Connection parameters for Redshift. Defaults to environment variables.
+      @@redshift_connection_params = {
+        host: ENV['RRR_REDSHIFT_HOST'],
+        dbname: ENV['RRR_REDSHIFT_DATABASE'],
+        port: ENV['RRR_REDSHIFT_PORT'],
+        user: ENV['RRR_REDSHIFT_USER'],
+        password: ENV['RRR_REDSHIFT_PASSWORD']
+      }
+
+      # AWS S3 Replication bucket credentials. Defaults to environment variables.
+      @@aws_credentials = {
+        key: ENV['RRR_AWS_ACCESS_KEY_ID'],
+        secret: ENV['RRR_AWS_SECRET_ACCESS_KEY']
+      }
+
+      # AWS S3 replication bucket parameters.
+      # region defaults to environment variable or US East (N. Virginia)
+      # bucket defaults to environment variable
+      @@s3_bucket_params = {
+        region: (ENV['RRR_REPLICATION_REGION'] || 'us-east-1'),
+        bucket: ENV['RRR_REPLICATION_BUCKET'],
+        prefix: ENV['RRR_REPLICATION_PREFIX']
+      }
+
+      # see [http://docs.aws.amazon.com/redshift/latest/dg/r_COPY.html]
+      # You can add other keys aside from changing these.
+      # The keys won't be used on the copy commands. Just their values.
+      # To remove one of the defaults, set it to nil.
+      # @example:
+      #   @@copy_options = {
+      #     statupdate: nil,
+      #   }
+      @@copy_options = {
+        statupdate: 'STATUPDATE TRUE',
+        acceptinvchars: 'ACCEPTINVCHARS',
+        empty: 'EMPTYASNULL',
+        truncate: 'TRUNCATECOLUMNS'
+      }
+
+      # Number of slices available on Redshift cluster. Used to split export files. Defaults to 1.
+      # see [http://docs.aws.amazon.com/redshift/latest/dg/t_splitting-data-files.html]
+      @@redshift_slices = 1
+
+      # Folder to store temporary replication files until the S3 upload. Defaults to /tmp
+      @@local_replication_path = '/tmp'
+
+      # Command or path to executable that splits files
+      @@split_command = 'split'
+
+      # Command or path to executable that compresses files to gzip
+      @@gzip_command = 'gzip'
+
+      # Enable debug mode to output messages to STDOUT. Default to false
+      @@debug_mode = false
+
+      # Defines how many replication records are kept in history. Default to nil keeping full history.
+      @@history_cap = nil
+
+      # Preferred format for export file
+      @@preferred_format = 'csv'
+
+      # Maximum number of retries for a replication before cancelling and starting another
+      @@max_retries = nil
+
+      # If deletes should be tracked and propagated to redshift
+      @@enable_delete_tracking = false
+
+      # If exported files on s3 should be deleted after imported
+      @@delete_s3_file_after_import = true
+
+      return nil
+    end
+    alias reload define_defaults
+
+    def debug_mode=(value)
+      logger.level = value == true ? Logger::DEBUG : Logger::WARN
+      @@debug_mode = value
+    end
+
+    def history_cap=(value)
+      @@history_cap = value && [value,2].max
+    end
+
+    # @return [RedshiftReplicator]
+    def setup
+      yield self
+    end
+
+    def add_replicable(hash)
+      logger.debug I18n.t(:replicable_added, table_name: hash.keys.first, scope: :rails_redshift_replicator) 
+      RailsRedshiftReplicator.replicables.merge! hash
+    end
+
+    def reload_replicables
+      replicables = {}
+      replicables.each do |name, replicable|
+        add_replicable(name => RailsRedshiftReplicator::Replicable.new(replicable.replication_type, replicable.options))
+      end
+    end
+
+    # Performs full replication (export + import)
+    # @param models [Array<Symbol>, Argument list] activerecord models to export or :all
+    # @example Replicate user and post models.
+    #   RedshiftReplicator.replicate(:user, :publication)
+    # @example Replicate all models
+    #   RedshiftReplicator.replicate(:all)
+    def replicate(*tables)
+      check_args(tables)
+      replicable_definitions(tables_to_perform(tables)).each do |_, replicable|
+        replication = replicable.export
+        replicable.import
+      end
+    end
+
+    # @see .replicate
+    def export(*tables)
+      check_args(tables)
+      replicable_definitions(tables_to_perform(tables)).each { |_, replicable| replicable.export }
+    end
+
+    # @see .replicate
+    def import(*tables)
+      check_args(tables)
+      replicable_definitions(tables_to_perform(tables)).each { |_, replicable| replicable.import }
+    end
+
+    def check_args(tables)
+      if tables == []
+        error_message = I18n.t(:must_specify_tables, scope: :rails_redshift_replicator)
+        logger.error error_message
+        raise StandardError.new(error_message)
+      end
+    end
+
+    def vacuum(*args)
+      Tools::Vacuum.new(*args).perform
+    end
+
+    def analyze(*args)
+      Tools::Analyze.new(*args).perform
+    end
+
+    # Lists exporters names
+    def base_exporter_types
+      [
+        'identity_replicator',
+        'timed_replicator',
+        'full_replicator'
+      ]
+    end
+
+    # All replicable tables registered in RailsRedshiftReplicator
+    # eighter from the model or directly.
+    # @return [Array<String>] tables
+    def replicable_tables
+      RailsRedshiftReplicator.replicables.keys.map(&:to_s)
+    end
+
+    def replicable_target_tables
+      RailsRedshiftReplicator.replicables.map{ |k,v| v[:target_table] }
+    end
+
+    # @retuns [Hash] subset of key pairs of replicables
+    def replicable_definitions(tables)
+      RailsRedshiftReplicator.replicables.select { |k,_| k.to_s.in? tables.map(&:to_s) }
+    end
+
+    # Returns tables to export. :all selects all eligible
+    # @returns [Array<String>] tables to export
+    def tables_to_perform(tables)
+      tables = Array(tables).map(&:to_s)
+      if tables[0] == 'all'
+        replicable_tables
+      else
+        (replicable_tables & tables).tap do |selected|
+          warn_if_unreplicable tables-selected
+        end
+      end
+    end
+
+    def warn_if_unreplicable(tables)
+      tables.each { |table| logger.warn I18n.t(:table_not_replicable, table_name: table, scope: :rails_redshift_replicator) }
+    end
+
+    # Redshift connection
+    # @return [PG::Connection]
+    def connection
+      @redshift ||= PG.connect(redshift_connection_params)
+    end
+  end
+end
+RailsRedshiftReplicator.define_defaults