rails_redshift_replicator 0.0.1

data/lib/rails_redshift_replicator/exporters/timed_replicator.rb

@@ -0,0 +1,9 @@
+module RailsRedshiftReplicator
+  module Exporters
+    class TimedReplicator < Base
+      def self.replication_field
+        "updated_at"
+      end
+    end
+  end
+end

data/lib/rails_redshift_replicator/file_manager.rb

@@ -0,0 +1,134 @@
+require 'aws-sdk'
+module RailsRedshiftReplicator
+  class FileManager
+    attr_reader :exporter
+
+    def s3_client
+      @client ||= Aws::S3::Client.new(
+        region: RailsRedshiftReplicator.s3_bucket_params[:region],
+        access_key_id: RailsRedshiftReplicator.aws_credentials[:key],
+        secret_access_key: RailsRedshiftReplicator.aws_credentials[:secret]
+      )
+    end
+
+    # File location on s3
+    # @return [String] file location
+    def self.s3_file_key(source_table, file)
+      File.join RailsRedshiftReplicator.s3_bucket_params[:prefix], source_table, file
+    end
+
+    def initialize(exporter = nil)
+      @exporter = exporter
+    end
+
+    def bucket
+      RailsRedshiftReplicator.s3_bucket_params[:bucket]
+    end
+
+    def delete
+      response = s3_replication_files
+      if response.contents
+        response.contents.each do |file|
+          RailsRedshiftReplicator.logger.info I18n.t(:deleting_file, key: file.key, scope: :rails_redshift_replicator)
+          s3_client.delete_object(bucket: bucket, key: file.key)
+        end
+      end
+    end
+
+    def s3_replication_files
+      s3_client.list_objects(bucket: bucket, prefix: exporter.replication.key)
+    end
+
+    def temp_file_name
+      "#{exporter.source_table}_#{Time.now.to_i}.csv"
+    end
+
+    # Writes all results to one file for future splitting.
+    # @param file_name [String] name of the local export file
+    # @return [Integer] number of records to export.
+    def write_csv(file_name, records)
+      line_number = exporter.connection_adapter.write(local_file(file_name), records)
+    end
+
+    # Path to the local export file
+    # @param name [String] file name
+    # @return [String] path to file
+    def local_file(name)
+      @local_file ||= "#{RailsRedshiftReplicator.local_replication_path}/#{name}"
+    end
+
+    # Splits the CSV into a number of files determined by the number of Redshift Slices
+    # @note This method requires an executable split and is compliant with Mac and Linux versions of it.
+    # @param name [String] file name
+    # @param counts [Integer] number of files
+    def split_file(name, record_count)
+      counts = row_count_threshold(record_count)
+      file_name = local_file(name)
+      `#{RailsRedshiftReplicator.split_command} -l #{counts} #{file_name} #{file_name}.`
+    end
+
+    # Number of lines per file
+    # @param counts [Integer] number of records
+    # @return [Integer] Number of lines per export file
+    def row_count_threshold(counts)
+      (counts.to_f/exporter.replication.slices).ceil
+    end
+
+
+    # Returns the s3 key to be used
+    # @return [String] file key with extension
+    def file_key_in_format(file_name, format)
+      if format == "gzip"
+        self.class.s3_file_key exporter.source_table, gzipped(file_name)
+      else
+        self.class.s3_file_key exporter.source_table, file_name
+      end
+    end
+
+    # Rename file to use .gz extension
+    # @return [String]
+    def gzipped(file)
+      file.gsub(".csv", ".gz")
+    end
+
+    def upload_gzip(files)
+      without_base = files_without_base(files)
+      without_base.each do |file|
+        basename = File.basename(file)
+        command = "#{RailsRedshiftReplicator.gzip_command} -c #{file} > #{gzipped(file)}"
+        RailsRedshiftReplicator.logger.info I18n.t(:gzip_notice, file: file, gzip_file: gzipped(file), command: command, scope: :rails_redshift_replicator)
+        `#{command}`
+        s3_client.put_object(
+          key: self.class.s3_file_key(exporter.source_table, gzipped(basename)),
+          body: File.open(gzipped(file)),
+          bucket: bucket
+        )
+      end
+      files.each { |f| FileUtils.rm f }
+      without_base.each { |f| FileUtils.rm gzipped(f) }
+    end
+
+    def files_without_base(files)
+      files.reject{|f| f.split('.').last.in? %w(gz csv)}
+    end
+
+    # Uploads splitted CSVs
+    # @param files [Array<String>] array of files paths to upload
+    def upload_csv(files)
+      files.each do |file|
+        basename = File.basename(file)
+        next if basename == File.basename(exporter.replication.key)
+        RailsRedshiftReplicator.logger.info I18n.t(:uploading_notice,
+                                                   file: file,
+                                                   key: self.class.s3_file_key(exporter.source_table, basename),
+                                                   scope: :rails_redshift_replicator)
+        s3_client.put_object(
+          key: self.class.s3_file_key(exporter.source_table, basename),
+          body: File.open(file),
+          bucket: bucket
+        )
+      end
+      files.each { |f| FileUtils.rm f }
+    end
+  end
+end

data/lib/rails_redshift_replicator/importers/base.rb

@@ -0,0 +1,158 @@
+require 'pg'
+module RailsRedshiftReplicator
+  module Importers
+    class Base
+      attr_accessor :replication
+      def initialize(replication)
+        return if replication.blank?
+        @replication = replication
+      end
+
+      def import
+        raise NotImplementedError
+      end
+
+      # History Cap has a minimum of 2
+      def evaluate_history_cap
+        if cap = RailsRedshiftReplicator.history_cap
+          RailsRedshiftReplicator::Replication.older_than(replication.source_table, cap).delete_all
+        end
+      end
+
+      def file_manager
+        @file_manager ||= RailsRedshiftReplicator::FileManager.new(self)
+      end
+
+      # Runs Redshift COPY command to import data from S3
+      # (http://docs.aws.amazon.com/redshift/latest/dg/r_COPY.html)
+      # @param [String] table name
+      # @param options [Hash]
+      # @option options [Boolean] :mark_as_imported If record should be flagged as imported
+      # @option options [Boolean] :noload If true, data will be validated but not imported
+      def copy(table_name = replication.target_table, options = {})
+        begin
+          RailsRedshiftReplicator.logger.info I18n.t(:importing_file, file: import_file, target_table: table_name, scope: :rails_redshift_replicator)
+          result = ::RailsRedshiftReplicator.connection.exec copy_statement(table_name, options)
+          replication.imported! if result.result_status == 1 && options[:mark_as_imported]
+        rescue => e
+          drop_table(table_name) if options[:can_drop_target_on_error]
+          if e.message.index("stl_load_errors")
+            get_redshift_error
+            notify_error
+          else
+            replication.update_attribute :last_error, e.exception.inspect
+          end
+        end
+      end
+
+      # Builds the copy statement
+      # @param (see #copy)
+      # @return [String] sql statement to run
+      def copy_statement(table_name, options = {})
+        format_options = replication.csv? ? "CSV" : "GZIP DELIMITER ',' ESCAPE REMOVEQUOTES"
+        sql = <<-CS
+          COPY #{table_name} from '#{import_file}' #{"NOLOAD" if options[:noload]}
+          REGION '#{RailsRedshiftReplicator.s3_bucket_params[:region]}'
+          CREDENTIALS 'aws_access_key_id=#{RailsRedshiftReplicator.aws_credentials[:key]};aws_secret_access_key=#{RailsRedshiftReplicator.aws_credentials[:secret]}'
+          #{format_options}
+          #{copy_options}
+        CS
+        sql.squish
+      end
+
+      def copy_options
+        RailsRedshiftReplicator.copy_options.values.join(" ")
+      end
+
+      # @return [String] location of import files on s3
+      def import_file
+        "s3://#{RailsRedshiftReplicator.s3_bucket_params[:bucket]}/#{replication.key}"
+      end
+
+      # Retrieves the last copy error for a given file on redshift
+      def get_redshift_error
+        sql = <<-RE.squish
+          SELECT filename, line_number, colname, type, raw_field_value, raw_line, err_reason
+          FROM STL_LOAD_ERRORS
+          WHERE filename like '%#{import_file}%'
+          ORDER BY starttime desc
+          LIMIT 1
+        RE
+        result = ::RailsRedshiftReplicator.connection.exec(sql).entries
+        error = result.first.map{ |k, v| [k, v.strip].join('=>') }.join(";")
+        replication.update_attribute :last_error, error
+      end
+
+      # TODO
+      def notify_error
+      end
+
+      # Creates a temporary table on redshift
+      def create_temp_table
+        RailsRedshiftReplicator.connection.exec "CREATE TEMP TABLE #{temporary_table_name} (LIKE #{replication.target_table})"
+      end
+
+      # Creates a permanent table for later renaming
+      def create_side_table
+        RailsRedshiftReplicator.connection.exec "CREATE TABLE #{temporary_table_name} (LIKE #{replication.target_table})"
+      end
+
+      # Runs a merge or replace operation on a redshift table
+      # The table is replaced on a FullReplicator strategy
+      # The table is merged on a TimedReplicator strategy
+      # @param :mode [Symbol] the operation type
+      def merge_or_replace(mode:)
+        target  = replication.target_table
+        stage = temporary_table_name
+        sql = send("#{mode}_statement", target, stage)
+        ::RailsRedshiftReplicator.connection.exec sql
+      end
+
+      # Builds the merge sql statement.
+      # At first, it deletes the matching records from the target and temporary tables on the target table.
+      # After it imports everything from the temporary table into the target table.
+      # @param target [String] 
+      # @param stage [String] temporary table
+      # @return [String] Sql Statement
+      # (http://docs.aws.amazon.com/redshift/latest/dg/merge-replacing-existing-rows.html)
+      def merge_statement(target, stage)
+        <<-SQLMERGE
+          begin transaction;
+
+          delete from #{target}
+          using #{stage}
+          where #{target}.id = #{stage}.id;
+          insert into #{target}
+          select * from #{stage};
+
+          end transaction;
+        SQLMERGE
+      end
+
+      # Builds the replace sql statement.
+      # @param (see #merge_statement)
+      # @return (see #merge_statement)
+      # (http://docs.aws.amazon.com/redshift/latest/dg/performing-a-deep-copy.html)
+      def replace_statement(target, stage)
+        <<-SQLREPLACE
+          begin transaction;
+          drop table #{target};
+          alter table #{stage} rename to #{target};
+          end transaction;
+        SQLREPLACE
+      end
+
+      # Deletes the temporary table
+      # @param table_name [String]
+      def drop_table(table_name = temporary_table_name)
+        ::RailsRedshiftReplicator.connection.exec "drop table if exists #{table_name}"
+      end
+
+      # Returns a random name for a temporary table
+      # @return [String] table name
+      def temporary_table_name
+        @temp_table ||= "temp_#{replication.target_table}_#{Time.now.to_i}"
+      end
+    end
+  end
+end

data/lib/rails_redshift_replicator/importers/full_replicator.rb

@@ -0,0 +1,17 @@
+module RailsRedshiftReplicator
+  module Importers
+    class FullReplicator < Base
+      def import
+        import_start = replication.importing!
+        create_side_table
+        copy temporary_table_name, mark_as_imported: false, can_drop_target_on_error: true
+        return if replication.error?
+        merge_or_replace(mode: :replace)
+        replication.clear_errors!
+        replication.imported! import_duration: (Time.now-import_start).ceil
+        evaluate_history_cap
+        file_manager.delete if RailsRedshiftReplicator.delete_s3_file_after_import
+      end
+    end
+  end
+end

data/lib/rails_redshift_replicator/importers/identity_replicator.rb

@@ -0,0 +1,15 @@
+module RailsRedshiftReplicator
+  module Importers
+    class IdentityReplicator < Base
+      def import
+        import_start = replication.importing!
+        copy replication.target_table, mark_as_imported: true
+        return if replication.error?
+        replication.clear_errors!
+        replication.update_attributes import_duration: (Time.now-import_start).ceil
+        evaluate_history_cap
+        file_manager.delete if RailsRedshiftReplicator.delete_s3_file_after_import
+      end
+    end
+  end
+end

data/lib/rails_redshift_replicator/importers/timed_replicator.rb

@@ -0,0 +1,18 @@
+module RailsRedshiftReplicator
+  module Importers
+    class TimedReplicator < Base
+      def import
+        import_start = replication.importing!
+        create_temp_table
+        copy temporary_table_name, mark_as_imported: false, can_drop_target_on_error: true
+        return if replication.error?
+        merge_or_replace(mode: :merge)
+        drop_table temporary_table_name
+        replication.clear_errors!
+        replication.imported! import_duration: (Time.now-import_start).ceil
+        evaluate_history_cap
+        file_manager.delete if RailsRedshiftReplicator.delete_s3_file_after_import
+      end
+    end
+  end
+end

data/lib/rails_redshift_replicator/model/extension.rb

@@ -0,0 +1,45 @@
+module RailsRedshiftReplicator
+  module Model
+    module Extension
+      def self.included(base)
+        base.send :extend, ClassMethods
+      end
+      
+      module ClassMethods
+        def has_redshift_replication(replication_type, options = {})
+          cattr_accessor :rails_redshift_replicator_replicable
+          replication_type = replication_type.to_s
+          raise I18n.t(:replication_type_not_supported,
+                       replication_type: replication_type,
+                       types: RailsRedshiftReplicator.base_exporter_types.join(","),
+                       scope: :exception_messages) unless replication_type.in? RailsRedshiftReplicator.base_exporter_types
+          extend Actions
+          options[:source_table] ||= self.table_name
+          self.rails_redshift_replicator_replicable = RailsRedshiftReplicator::Replicable.new(replication_type, options)
+          RailsRedshiftReplicator.add_replicable({ options[:source_table] => rails_redshift_replicator_replicable })
+        end
+      end
+    end
+    module Actions
+      def rrr_export
+        rails_redshift_replicator_replicable.export
+      end
+      def rrr_import
+        rails_redshift_replicator_replicable.import
+      end
+      def rrr_replicate
+        rails_redshift_replicator_replicable.replicate
+      end
+      def rrr_vacuum
+        rails_redshift_replicator_replicable.vacuum
+      end
+      def rrr_analyze
+        rails_redshift_replicator_replicable.analyze
+      end
+      def rrr_deleter
+        rails_redshift_replicator_replicable.deleter
+      end
+    end
+  end
+end
+ActiveRecord::Base.send :include, RailsRedshiftReplicator::Model::Extension

data/lib/rails_redshift_replicator/model/hair_trigger_extension.rb

@@ -0,0 +1,8 @@
+HairTrigger.class_eval do
+  # includes triggers defined on replicables
+  def self.current_triggers
+    canonical_triggers = models.map(&:triggers).flatten.compact || []
+    canonical_triggers += RailsRedshiftReplicator.replicables.values.map(&:triggers).flatten.compact
+    canonical_triggers.each(&:prepare!) # interpolates any vars so we match the migrations
+  end
+end

data/lib/rails_redshift_replicator/replicable.rb

@@ -0,0 +1,143 @@
+module RailsRedshiftReplicator
+  class Replicable
+    include HairTrigger::Base
+    attr_reader :source_table, :target_table, :replication_field, :replication_type, :tracking_deleted, :options
+    # alias for hairtrigger
+    alias table_name source_table
+
+    # @param replication_type [String, Symbol]
+    # @param options [Hash] Replication options
+    # @option options [String, Symbol] :source_table name of the source table to replicate
+    # @option options [String, Symbol] :target_table name of the target table on redshift
+    # @option options [String, Symbol] :replication_field name of the replication field
+    def initialize(replication_type, options = {})
+      @replication_type = replication_type
+      @options = options
+      @source_table = options[:source_table].to_s
+      @target_table = (options[:target_table] || source_table).to_s
+      replication_field = options[:replication_field] || exporter_class.replication_field
+      @replication_field = replication_field && replication_field.to_s
+      @tracking_deleted = delete_tracking_enabled?
+      if tracking_deleted
+        trigger.after(:delete) do
+          "INSERT INTO rails_redshift_replicator_deleted_ids(source_table, object_id) VALUES('#{source_table}', OLD.id);"
+        end
+      end
+    end
+
+    def reset_last_record
+      last_imported_replication = RailsRedshiftReplicator::Replication.from_table(source_table).with_state(:imported).last
+      last_imported_replication && last_imported_replication.update_attribute(:last_record, nil)
+    end
+
+    def delete_tracking_enabled?
+      RailsRedshiftReplicator.enable_delete_tracking &&
+      (options[:enable_delete_tracking].blank? || options[:enable_delete_tracking]) &&
+      table_supports_tracking?
+    end
+
+    def table_supports_tracking?
+      ActiveRecord::Base.connection.columns(source_table).map(&:name).include? "id"
+    end
+
+    def replicate
+      export
+      import
+    end
+
+    def import
+      @last_replication = nil
+      if last_replication.present?
+        if last_replication.uploaded?
+          perform_import
+        elsif last_replication.imported?
+          RailsRedshiftReplicator.logger.info I18n.t(:nothing_to_import, table_name: source_table, scope: :rails_redshift_replicator)
+        elsif max_retries_reached?
+          last_replication.cancel!
+        else
+          resume_replication
+        end
+      else
+        RailsRedshiftReplicator.logger.info I18n.t(:nothing_to_import, table_name: source_table, scope: :rails_redshift_replicator)
+      end
+    end
+
+    def export
+      @last_replication = nil
+      if last_replication.blank? || (last_replication && last_replication.imported?)
+        perform_export
+      else
+        if max_retries_reached?
+          last_replication.cancel!
+          perform_export
+        else
+          resume_replication
+        end
+      end
+    end
+
+    def perform_export(replication = nil)
+      exporter_class.new(self, replication).export_and_upload
+    end
+
+    def perform_import
+      importer_class.new(last_replication).import
+    end
+
+    def max_retries_reached?
+      if RailsRedshiftReplicator.max_retries && (RailsRedshiftReplicator.max_retries == last_replication.retries)
+        RailsRedshiftReplicator.logger.warn I18n.t(:max_retries_reached, id: last_replication, table_name: source_table, scope: :rails_redshift_replicator)
+        return true
+      else
+        false
+      end
+    end
+
+    def last_replication
+      @last_replication ||= RailsRedshiftReplicator::Replication.from_table(source_table).last
+    end
+
+    def resume_replication
+      last_replication.increment! :retries, 1
+      if last_replication.state.in? %w(enqueued exporting exported uploading)
+        log_resuming('export')
+        perform_export(last_replication)
+      else
+        log_resuming('import')
+        perform_import
+      end
+    end
+
+    def log_resuming(action)
+      RailsRedshiftReplicator.logger.info I18n.t(:resuming_replication, table_name: source_table, action: action, state: last_replication.state, scope: :rails_redshift_replicator)
+    end
+
+    def vacuum
+      RailsRedshiftReplicator.vacuum(target_table)
+    end
+
+    def analyze
+      RailsRedshiftReplicator.analyze(target_table)
+    end
+
+    def deleter
+      RailsRedshiftReplicator::Deleter.new(self)
+    end
+
+    def exporter_class
+      @exporter_class ||= begin
+        "RailsRedshiftReplicator::Exporters::#{replication_type.to_s.classify}".constantize
+      rescue
+        raise StandardError.new I18n.t(:missing_replicator_type, scope: :rails_redshift_replicator)
+      end
+    end
+
+    def importer_class
+      @importer_class ||= begin
+        "RailsRedshiftReplicator::Importers::#{replication_type.to_s.classify}".constantize
+      rescue
+        raise StandardError.new I18n.t(:missing_replicator_type, scope: :rails_redshift_replicator)
+      end
+    end
+  end
+end