andyjeffries-rubyrep 1.2.1

data/lib/rubyrep/command_runner.rb

@@ -0,0 +1,144 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'optparse'
+
+module RR
+
+  # This class implements the functionality to dispatch rubyrep commands.
+  class CommandRunner
+    
+    # Returns a hash of all commands registered with #register.
+    def self.commands
+      @commands ||= {}
+    end
+
+    # Registers one or multiple commands.
+    # +commands+ is a hash with
+    # * key: name of the command
+    # * value: a command hash defining the command
+    #
+    # A command hash consists of
+    # * :+description+: short description of the command
+    # * :+command+: an object / class implementing the hash.
+    #               Must have a method
+    #
+    #               # runs a command
+    #               # * +args+: array of command line parameters
+    #               #           note: will not contain the command name itself.
+    #               def run(args)
+    def self.register(commands)
+      self.commands.merge!(commands)
+    end
+    
+    # Prints the version to stderr
+    def self.show_version
+      $stdout.puts "rubyrep version #{RR::VERSION::STRING}"
+    end
+
+
+    # Dispatches commands as per given command line parameters.
+    # * +args+: array of command line parameters
+    def self.run(args)
+      status = 0
+      options = {}
+
+      parser = OptionParser.new do |opts|
+        opts.banner = <<EOS
+Usage: #{$0} [general options] command [parameters, ...]
+
+Asynchronous master-master replication of relational databases.
+EOS
+        opts.separator ""
+        opts.separator "Available options:"
+
+        opts.on("--verbose", "Show errors with full stack trace") do
+          options[:verbose] = true
+        end
+
+        opts.on("-v", "--version", "Show version information.") do
+          show_version
+          options = nil
+        end
+
+        opts.on_tail("--help", "Show this message") do
+          $stderr.puts opts
+          
+          $stderr.puts "\nAvailable commands:"
+          commands.sort.each do |command_name, command_hash|
+            $stderr.puts "  #{command_name.ljust(15)} #{command_hash[:description]}"
+          end
+
+          options = nil
+        end
+      end
+
+      begin
+
+        # extract general options
+        general_args = []
+        until args.empty?
+          if args[0] =~ /^-/
+            general_args << args.shift
+          else
+            break
+          end
+        end
+
+        # parse general options
+        parser.parse!(general_args)
+
+        # process commands
+        if options # this will be +nil+ if the --help or --version are specified
+          if args.empty?
+            $stderr.puts "No command specified.\n\n"
+            run(['--help'])
+            status = 1
+          else
+            command = args[0]
+            if command == 'help' and args.size == 1
+              run(['--help'])
+              status = 0
+            elsif commands.include? command
+              status = commands[command][:command].run(args.slice(1, 1_000_000))
+            else
+              $stderr.puts "Error: Unknown command specified.\n\n"
+              run(['--help'])
+              status = 1
+            end
+          end
+        end
+      rescue Exception => e
+        $stderr.puts "Exception caught: #{e}"
+        $stderr.puts e.backtrace if options && options[:verbose]
+        status = 1
+      end
+
+      return status
+    end
+  end
+
+  # Command runner to show help for other commands
+  class HelpRunner
+    CommandRunner.register 'help' => {
+      :command => self,
+      :description => "Shows detailed help for the specified command"
+    }
+
+    # Runs the help command
+    # * +args+: array of command line parameters
+    def self.run(args)
+      if args[0] == 'help' or args[0] == '--help'
+        $stderr.puts(<<EOS)
+Usage: #{$0} help [command]
+
+Shows the help for the specified command.
+EOS
+        0
+      else
+        CommandRunner.run([args[0], '--help'])
+      end
+    end
+  end
+end
+
+

data/lib/rubyrep/committers/buffered_committer.rb

@@ -0,0 +1,151 @@
+module RR
+  module Committers
+
+    # This committer periodically commits transactions. It can be used for
+    # pre-replication syncs as it
+    # * updates the activity marker table.
+    # * switches existing triggers to filter out rubyrep activity
+    class BufferedCommitter < DefaultCommitter
+
+      # Register the committer
+      Committers.register :buffered_commit => self
+
+      # Unless overwritten via configuration, transactions are commited after the
+      # given number of record changes
+      DEFAULT_COMMIT_FREQUENCY = 1000
+
+      # Switches the trigger mode of the specified +table+ in the specified
+      # +database+ to ignore rubyrep activity.
+      # * +database+: identifying the database (either :+left+ or :+right+)
+      # * +table+: name of the table
+      def exclude_rr_activity(database, table)
+        trigger_mode_switcher.exclude_rr_activity database, table
+      end
+
+      # Returns the TriggerModeSwitcher (creates it if necessary)
+      def trigger_mode_switcher
+        @trigger_mode_switcher ||= TriggerModeSwitcher.new session
+      end
+
+      # Returns the name of the activity marker table
+      def activity_marker_table
+        @activity_marker_table ||= "#{session.configuration.options[:rep_prefix]}_running_flags"
+      end
+
+      # Returns +true+ if the activity marker table should be maintained.
+      def maintain_activity_status?
+        unless @activity_status_checked
+          @activity_status_checked = true
+          @maintain_activity_status = session.left.tables.include?(activity_marker_table)
+        end
+        @maintain_activity_status
+      end
+
+      # Returns the number of changes, after which the open transactions should
+      # be committed and new transactions be started.
+      def commit_frequency
+        unless @commit_frequency
+          @commit_frequency = session.configuration.options[:commit_frequency]
+          @commit_frequency ||= DEFAULT_COMMIT_FREQUENCY
+        end
+        @commit_frequency
+      end
+
+      # Commits the open transactions in both databases. Before committing,
+      # clears the rubyrep activity marker.
+      def commit_db_transactions
+        [:left, :right].each do |database|
+          if maintain_activity_status?
+            session.send(database).execute("delete from #{activity_marker_table}")
+          end
+          session.send(database).commit_db_transaction
+        end
+      end
+
+      # Begins new transactions in both databases. After starting the transaction,
+      # marks the activity of rubyrep.
+      def begin_db_transactions
+        [:left, :right].each do |database|
+          session.send(database).begin_db_transaction
+          if maintain_activity_status?
+            session.send(database).execute("insert into #{activity_marker_table} values(1)")
+          end
+        end
+      end
+
+      # Rolls back the open transactions in both databases.
+      def rollback_db_transactions
+        session.left.rollback_db_transaction
+        session.right.rollback_db_transaction
+      end
+
+      # Commits the open tranactions and starts new one if the #commit_frequency
+      # number of record changes have been executed.
+      def commit
+        @change_counter ||= 0
+        @change_counter += 1
+        if @change_counter == commit_frequency
+          @change_counter = 0
+          commit_db_transactions
+          begin_db_transactions
+        end
+      end
+
+      # Returns +true+ if a new transaction was started since the last
+      # insert / update / delete.
+      def new_transaction?
+        @change_counter == 0
+      end
+
+      # A new committer is created for each table sync.
+      # * session: a Session object representing the current database session
+      def initialize(session)
+        super
+        begin_db_transactions
+      end
+
+      # Inserts the specified record in the specified database.
+      # * +database+: identifying the database (either :+left+ or :+right+)
+      # * +table+: name of the table
+      # * +values+: a hash of column_name => value pairs.
+      def insert_record(database, table, values)
+        exclude_rr_activity database, table
+        super
+        commit
+      end
+
+      # Updates the specified record in the specified database.
+      # * +database+: identifying the database (either :+left+ or :+right+)
+      # * +table+: name of the table
+      # * +values+: a hash of column_name => value pairs.
+      # * +old_key+:
+      #   A column_name => value hash identifying original primary key.
+      #   If +nil+, then the primary key must be contained in +values+.
+      def update_record(database, table, values, old_key = nil)
+        exclude_rr_activity database, table
+        super
+        commit
+      end
+
+      # Deletes the specified record in the specified database.
+      # * +database+: identifying the database (either :+left+ or :+right+)
+      # * +table+: name of the table
+      # * +values+: a hash of column_name => value pairs (must only contain primary key columns).
+      def delete_record(database, table, values)
+        exclude_rr_activity database, table
+        super
+        commit
+      end
+
+      # Is called after the last insert / update / delete query.
+      # * +success+: should be true if there were no problems, false otherwise.
+      def finalize(success = true)
+        if success
+          commit_db_transactions
+        else
+          rollback_db_transactions
+        end
+      end
+    end
+  end
+end

data/lib/rubyrep/committers/committers.rb

@@ -0,0 +1,152 @@
+module RR
+  # Committers are classes that implement transaction policies.
+  # This module provides functionality to register committers and access the
+  # list of registered committers.
+  # Every Committer needs to register itself with Committers#register.
+  # Each Committer must implement at the following methods:
+  #
+  #   # Creates a new committer
+  #   #   * session: a Session object representing the current database session
+  #   def initialize(session)
+  #
+  #   # Inserts the specified record in the specified +database+ (either :left or :right).
+  #   # +table+ is the name of the target table.
+  #   # +values+ is a hash of column_name => value pairs.
+  #   def insert_record(database, values)
+  #
+  #   # Updates the specified record in the specified +database+ (either :left or :right).
+  #   # +table+ is the name of the target table.
+  #   # +values+ is a hash of column_name => value pairs.
+  #   # +old_key+ is a column_name => value hash with the original primary key.
+  #   # If +old_key+ is +nil+, then the primary key must be contained in +values+.
+  #   def update_record(database, values, old_key)
+  #
+  #   # Deletes the specified record in the specified +database+ (either :left or :right).
+  #   # +table+ is the name of the target table.
+  #   # +values+ is a hash of column_name => value pairs. (Only the primary key
+  #   # values will be used and must be included in the hash.)
+  #   def delete_record(database, values)
+  #
+  #   # Is called after the last insert / update / delete query.
+  #   def finalize
+  #
+  module Committers
+    # Returns a Hash of currently registered committers.
+    # (Empty Hash if no connection committers were defined.)
+    def self.committers
+      @committers ||= {}
+      @committers
+    end
+  
+    # Registers one or multiple committers.
+    # committer_hash is a Hash with 
+    #   key::   The adapter symbol as used to reference the committer
+    #   value:: The class implementing the committer
+    def self.register(committer_hash)
+      @committers ||= {}
+      @committers.merge! committer_hash
+    end
+    
+    # This committer does not do anything. This means that the default DBMS 
+    # behaviour is used (for most DBMS: every DML statement (insert, update, 
+    # delete) runs in it's own transaction.
+    class DefaultCommitter
+      
+      # Register the committer
+      Committers.register :default => self
+
+      # The current Session object
+      attr_accessor :session 
+      
+      # A hash holding the proxy connections
+      # E. g. {:left => <left connection>, :right => <right connection>}
+      attr_accessor :connections
+    
+      # A new committer is created for each table sync.
+      #   * session: a Session object representing the current database session
+      def initialize(session)
+        self.session = session
+        self.connections = {:left => session.left, :right => session.right}
+      end
+
+      # Returns +true+ if a new transaction was started since the last
+      # insert / update / delete.
+      def new_transaction?
+        false
+      end
+      
+      # Inserts the specified record in the specified +database+ (either :left or :right).
+      # +table+ is the name of the target table.
+      # +values+ is a hash of column_name => value pairs.
+      def insert_record(database, table, values)
+        connections[database].insert_record(table, values)
+      end
+      
+      # Updates the specified record in the specified +database+ (either :left or :right).
+      # +table+ is the name of the target table.
+      # +values+ is a hash of column_name => value pairs.
+      # # +old_key+ is a column_name => value hash with the original primary key.
+      # If +old_key+ is +nil+, then the primary key must be contained in +values+.
+      def update_record(database, table, values, old_key = nil)
+        connections[database].update_record(table, values, old_key)
+      end
+      
+      # Deletes the specified record in the specified +database+ (either :left or :right).
+      # +table+ is the name of the target table.
+      # +values+ is a hash of column_name => value pairs. (Only the primary key
+      # values will be used and must be included in the hash.)
+      def delete_record(database, table, values)
+        connections[database].delete_record(table, values)
+      end
+      
+      # Is called after the last insert / update / delete query.
+      # +success+ should be true if there were no problems, false otherwise.
+      def finalize(success = true)
+      end
+    end
+    
+    # Starts a transaction but does never commit it.
+    # Useful during testing.
+    class NeverCommitter < DefaultCommitter
+      Committers.register :never_commit => self
+      @@current_session = nil
+      
+      # Returns the last active data session
+      def self.current_session
+        @@current_session
+      end
+      
+      # Saves the provided database session as class variable.
+      # Purpose: the last database session stays available after the 
+      # NeverCommitter is destroyed so that also later the transaction rollback
+      # can still be executed.
+      def self.current_session=(session)
+        @@current_session = session
+      end
+      
+      # Rolls back transactions of current session (if there is one).
+      # This would be called e. g. in rspec's after(:each) to ensure that 
+      # the next test case finds the original test data.
+      def self.rollback_current_session
+        if self.current_session
+          self.current_session.left.rollback_db_transaction
+          self.current_session.right.rollback_db_transaction
+          self.current_session = nil
+        end
+      end
+
+      # Refer to DefaultCommitter#initialize for details.
+      # Starts new transactions on left and right database connectin of session.
+      # Additionally rolls back transactions started in previous 
+      # +NeverCommitter+ instances.
+      def initialize(session)
+        super
+        self.class.rollback_current_session
+        self.class.current_session = session
+        session.left.begin_db_transaction
+        session.right.begin_db_transaction
+      end
+    end
+    
+  end
+end

data/lib/rubyrep/configuration.rb

@@ -0,0 +1,275 @@
+module RR
+
+  # The Configuration class holds the default configuration options for Rubyrep.
+  # Configuration values are changed with the Initializer::run method.
+  class Configuration
+    # Connection settings for the "left" database.
+    # See Configuration#right for details.
+    attr_accessor :left
+
+    # Connection settings for the "right" database.
+    # Takes a similar hash as ActiveRecord::Base.establish_connection.
+    # Additional settings in case a proxy is used:
+    # * :+proxy_host+: name or IP address of where the proxy is running
+    # * :+proxy_port+: port on which the proxy is listening
+    # Other additional settings:
+    # * :+logger+:
+    #   Specify an SQL statement logger for this database connection.
+    #   Can be either
+    #   * a logger instance itself (Logger or Log4r::Logger) or
+    #   * the parameter to create a Logger with Logger.new
+    #   Examples:
+    #     config.left[:logger] = STDOUT
+    #     config.right[:logger] = Logger.new('rubyrep_debug.log')
+    attr_accessor :right
+    
+    # Returns true unless running on windows...
+    def self.true_if_running_in_a_terminal_and_not_under_windows
+      # Not using RUBY_PLATFORM as it should also work under JRuby
+      $stdout.tty? and not ENV['OS'] =~ /windows/i
+    end
+
+    # Default #options for a new Configuration object.
+    DEFAULT_OPTIONS = {
+      :proxy_block_size => 1000,
+      :row_buffer_size => 1000,
+      :replicator => :two_way,
+      :committer => :buffered_commit,
+      :commit_frequency => 1000,
+      :table_ordering => true,
+      :scan_progress_printer => :progress_bar,
+      :use_ansi => true_if_running_in_a_terminal_and_not_under_windows,
+      :initial_sync => true,
+      :adjust_sequences => true,
+      :sequence_adjustment_buffer => 0,
+      :sequence_increment => 2,
+      :left_sequence_offset => 0,
+      :right_sequence_offset => 1,
+      :replication_interval => 1,
+      :auto_key_limit => 0,
+      :database_connection_timeout => 5,
+
+      :rep_prefix => 'rr',
+      :key_sep => '|',
+    }
+    
+    # General options.
+    # Possible settings:
+    # * :+proxy_block_size+: The proxy cursor will calculate the checksum for block_size number of records each.
+    # * :+row_buffer_size+:
+    #   The number of rows that is read into memory at once.
+    #   Only needed for database drivers that don't stream results one-by-one to the client.
+    # * :+committer+:
+    #   A committer key as registered by Committers#register.
+    #   Determines the transaction management to be used during the sync.
+    # * :+commit_frequency+:
+    #   Used by BufferedCommitter. Number of changes after which the open
+    #   transactions should be committed and new transactions be started.
+    # * :+table_ordering+:
+    #   If true, sort tables before syncing as per foreign key dependencies.
+    #   (Dependent tables are synced last to reduce risk of foreign key
+    #   constraint violations.)
+    # * :+scan_progress_printer+:
+    #   The progress printer key as registered by ScanProgressPrinters#register.
+    #   Determines how the scan progress is visualized.
+    # * :+use_ansi+: Only use ANSI codes for text output if +true+.
+    # * :+auto_key_limit+:
+    #   If a table has no primary keys and no primary keys have been specified
+    #   manually using the :+primary_key_names+ option, then this option can be
+    #   activated to simply use all columns of the table as a big combined key.
+    #   This option specifies up to how many columns a table may have in order
+    #   to use them as one big, combined primary key.
+    #   Typical use case: the database has a lot of tables to map many-to-many
+    #   relationshipts and no combined primary key is set up for them.
+    # Sync specific settings
+    # * :+before_table_sync+:
+    #   A hook that is executed before a table sync.
+    #   Can be either
+    #   * a String: executed as SQL command on both databases.
+    #   * a Proc:
+    #     Called once before the table sync.
+    #     The Proc is called with one parameter: the current SyncHelper instance.
+    #     Through the sync helper there is access to the name of the synced table,
+    #     the current session, etc
+    #     Example:
+    #     lambda {|helper| $stderr.puts "Hook called for #{helper.left_table}."}
+    # * :+after_table_sync+:
+    #   Same as :+before_table_sync+ (but called after the sync is completed).
+    # * :+syncer+:
+    #   A syncer key as registered by TableSync#register_syncer.
+    #   Determines which sync algorithm is used.
+    # * further options as defined by each syncer
+    # Replication specific settings:
+    # * :+rep_prefix+: the prefix that is put in front of all created database objects
+    # * :+key_sep+: which string separates columns in the key column of the change log table
+    # * :+replicator+:
+    #   Determines which replicator algorithm to use.
+    #   For each replicator must also exist a corresponding +:syncer+. (It is
+    #   used for the initial sync of a table.)
+    #   If no +:syncer+ option is specified, than a syncer as named by this
+    #   option is used.
+    # * :+initial_sync+:
+    #   If +true+, syncs a table when initializing replication.
+    #   Disable with care!
+    #   (I. e. ensure that the table(s) have indeed same data in both databases
+    #   before starting replication.)
+    # * :+adjust_sequences+:
+    #   If +true+, adjust sequences to avoid number conflicts between left and
+    #   right database during replication.
+    # * :+sequence_adjustement_buffer+:
+    #   When updating a sequence, this is the additional gap to avoid sequence
+    #   conflicts to appear due to concurrent record insertions.
+    # * :+sequence_increment+: new sequence value = last sequence value + this
+    # * :+left_sequence_offset+, +right_sequence_offset+:
+    #   Default sequence offset for the table in the according data base.
+    #   E. g. with a +sequence_increment+ of 2, an offset of 0 will produce even,
+    #   an offset of 1 will produce odd numbers.
+    # * :+replication_interval+: time in seconds between replication runs
+    # * :+database_connection_timeout+:
+    #   Time in seconds after which database connections time out.
+    # * :+:after_infrastructure_setup+:
+    #   A Proc that is called after the replication infrastructure tables are
+    #   set up. Useful to e. g. tweak the access settings for the table.
+    #   The block is called with the current Session object.
+    #   The block is called every time replication is started, even if the
+    #   the infrastructure tables already existed.
+    #
+    # Example of an :+after_infrastructure_setup+ handler:
+    #   lambda do |session|
+    #     [:left, :right].each do |database|
+    #       session.send(database).execute "GRANT SELECT, UPDATE, INSERT ON rr_pending_changes TO scott"
+    #     end
+    #   end
+    attr_reader :options
+    
+    # Merges the specified +options+ hash into the existing options
+    def options=(options)
+      @options ||= {}
+      @options = @options.merge! options
+    end
+
+    # Array of table specifications for tables that should be processed
+    # Refer to #add_table_options for what constitutes a valid table specification.
+    def included_table_specs
+      @included_table_specs ||= []
+    end
+
+    # Array of table specifications for tables that should *not* be processed
+    # Refer to #add_table_options for what constitutes a valid table specification.
+    def excluded_table_specs
+      @excluded_table_specs ||= []
+    end
+    
+    # Ensures that rubyrep infrastructure tables are excluded
+    def exclude_rubyrep_tables
+      exclude_tables Regexp.new("^#{options[:rep_prefix]}_.*")
+    end
+    
+    # A list of tables having table specific options that should be considered
+    # during processing (scanned, synced, ...)
+    # +tables_with_options+ is a 2 element array with
+    # * first element: A +table_spec+ (either a table name or a regexp matching multiple tables)
+    # * second element: The +options+ hash (detailed format described in #add_tables
+    # Should only be accessed via #add_table_options and #options_for_table
+    def tables_with_options
+      @tables_with_options ||= []
+    end
+
+    # Adds the specified tables to the list of tables that should be processed.
+    # If options are provided, store them for future processing.
+    # Refer to #add_table_options for detailed description of parameters.
+    def include_tables(table_spec, options = nil)
+      included_table_specs << table_spec unless included_table_specs.include?(table_spec)
+      add_table_options(table_spec, options) if options
+    end
+    alias_method :include_table, :include_tables
+
+    # Excludes the specified table from the list of tables that should be
+    # processed.
+    # Refer to #add_table_options for detailed description of what constitutes a
+    # valid table specification.
+    def exclude_tables(table_spec)
+      excluded_table_specs << table_spec unless excluded_table_specs.include?(table_spec)
+    end
+    alias_method :exclude_table, :exclude_tables
+    
+    # Adds the specified options for the provided +table_spec+.
+    # A +table_spec+ can be either
+    # * a table name or
+    # * a table pair (e. g. "my_left_table, my_right_table")
+    # * a regexp matching multiple tables.
+    # +options+ is hash with possible generic values as described under #options.
+    # Additional, exclusively table specific options:
+    # * :+primary_key_names+: array of primary key names
+    def add_table_options(table_spec, options)
+      i = nil
+      tables_with_options.each_with_index { |table_options, k|
+        i = k if table_options[0] == table_spec
+      }
+      if i
+        table_options = tables_with_options[i][1]
+      else
+        table_options = {}
+        tables_with_options << [table_spec, table_options]
+      end
+      table_options.merge! options
+    end
+    alias_method :add_table_option, :add_table_options
+
+    # Yields all table specs that have been set up with the given option
+    # * +key+: the option key
+    # Yields:
+    # * +table_spec+: the table specification of the matching option (or nil if non-table specific setting)
+    # * +option_value+: the option value for the specified +key+
+    def each_matching_option(key)
+      yield nil, options[key] if options.include?(key)
+      tables_with_options.each do |table_options|
+        yield table_options[0], table_options[1][key] if table_options[1].include? key
+      end
+    end
+    
+    # Returns an option hash for the given table.
+    # Accumulates options for all matching table specs (most recently added options
+    # overwrite according options added before).
+    #
+    # Also includes the general options as returned by #options.
+    # (Table specific options overwrite the general options).
+    # 
+    # Possible option values are described under #add_tables.
+    def options_for_table(table)
+      resulting_options = options.clone
+      tables_with_options.each do |table_options|
+        match = false
+        if table_options[0].kind_of? Regexp
+          match = (table_options[0] =~ table)
+        else
+          match = (table_options[0].sub(/(^.*),.*/,'\1').strip == table)
+        end
+        resulting_options.merge! table_options[1] if match
+      end
+
+      # Merge the default syncer& replicator options in
+      [
+        Syncers.configured_syncer(resulting_options),
+        Replicators.configured_replicator(resulting_options)
+      ].each do |processor_class|
+        if processor_class.respond_to? :default_options
+          default_processor_options = processor_class.default_options.clone
+        else
+          default_processor_options = {}
+        end
+        resulting_options = default_processor_options.merge!(resulting_options)
+      end
+
+      resulting_options
+    end
+
+    # initialize configuration settings
+    def initialize
+      self.left = {}
+      self.right = {}
+      self.options = DEFAULT_OPTIONS.clone
+    end
+    
+  end
+end