rubyrep 1.0.1 → 1.0.2

data/History.txt

@@ -1,3 +1,13 @@
+== 1.0.2 2009-06-13
+
+* 3 minor enhancements:
+  * Simplified low-level functions to read records from the database
+    (easier integration of additional databases)
+  * Better replication efficiency with large replication backlogs
+  * Reduced costs of an empty replication run
+  * Ignore tables that only exist in one database
+    (tables created for testing / backup don't block scan / sync / replication)
+
 == 1.0.1 2009-03-25
 
 * 3 minor enhancements:

data/README.txt

@@ -9,7 +9,7 @@ Development of an open-source solution for asynchronous, master-master replicati
 
 == MORE INFORMATION:
 
-Refer to the project website at http://rubyrep.github.com
+Refer to the project website at http://www.rubyrep.org
 
 == LICENSE:
 

data/lib/rubyrep/connection_extenders/connection_extenders.rb

@@ -71,9 +71,8 @@ module RR
       else
         raise "No ConnectionExtender available for :#{config[:adapter]}"
       end
-      connection_module = ConnectionExtenders.extenders[extender]
-      connection.extend connection_module
-
+      connection.extend ConnectionExtenders.extenders[extender]
+      
       # Hack to get Postgres schema support under JRuby to par with the standard
       # ruby version
       if RUBY_PLATFORM =~ /java/ and config[:adapter].to_sym == :postgresql

data/lib/rubyrep/connection_extenders/jdbc_extender.rb

@@ -7,106 +7,6 @@ module RR
     module JdbcSQLExtender
       RR::ConnectionExtenders.register :jdbc => self
       
-      # A cursor to iterate over the records returned by select_cursor.
-      # Only one row is kept in memory at a time.
-      module JdbcResultSet
-        # Returns true if there are more rows to read.
-        def next?
-          if @next_status == nil
-            @next_status = self.next
-          end
-          @next_status
-        end
-        
-        # Returns the row as a column => value hash and moves the cursor to the next row.
-        def next_row
-          raise("no more rows available") unless next?
-          @next_status = nil
-
-          unless @columns
-            meta_data = self.getMetaData
-            stores_upper = self.getStatement.getConnection.getMetaData.storesUpperCaseIdentifiers
-            column_count = meta_data.getColumnCount
-            @columns = Array.new(column_count)
-            @columns.each_index do |i|
-              column_name = meta_data.getColumnName(i+1)
-              if stores_upper and not column_name =~ /[a-z]/
-                column_name.downcase!
-              end
-              @columns[i] = {
-                :index => i+1,
-                :name => column_name,
-                :type => meta_data.getColumnType(i+1)
-                #:scale => meta_data.getScale(i+1)
-              }
-            end
-          end
-
-          row = {}
-          @columns.each_index do |i|
-            row[@columns[i][:name]] = jdbc_to_ruby(@columns[i])
-          end
-
-          row
-        end
-        
-        # Releases the databases resources hold by this cursor
-        def clear
-          @columns = nil
-          self.close
-        end
-
-        Types = java.sql.Types unless const_defined?(:Types)
-        
-        # Converts the specified column of the current row to the proper ruby string
-        # column is a hash with the following elements:
-        #   * :index: field number (starting with 1) of the result set field
-        #   * :type: the java.sql.Type constant specifying the type of the result set field
-        def jdbc_to_ruby(column)
-          case column[:type]
-          when Types::BINARY, Types::BLOB, Types::LONGVARBINARY, Types::VARBINARY
-            is = self.getBinaryStream(column[:index])
-            if is == nil or self.wasNull
-              return nil
-            end
-            byte_list = org.jruby.util.ByteList.new(2048)
-            buffer = Java::byte[2048].new
-            while (n = is.read(buffer)) != -1
-              byte_list.append(buffer, 0, n)
-            end
-            is.close
-            return byte_list.toString
-          when Types::LONGVARCHAR, Types::CLOB
-            rss = self.getCharacterStream(column[:index])
-            if rss == nil or self.wasNull
-              return nil
-            end
-            str = java.lang.StringBuffer.new(2048)
-            cuf = Java::char[2048].new
-            while (n = rss.read(cuf)) != -1
-              str.append(cuf, 0, n)
-            end
-            rss.close
-            return str.toString
-          when Types::TIMESTAMP
-            time = self.getTimestamp(column[:index]);
-            if  time == nil or self.wasNull
-              return nil
-            end
-            time_string = time.toString()
-            time_string = time_string.gsub(/ 00:00:00.0$/, '')
-            return time_string
-          else
-            value = self.getString(column[:index])
-            if value == nil or self.wasNull
-              return nil
-            end
-            return value
-          end
-        end
-        private :jdbc_to_ruby
-      end
-
       # Monkey patch for activerecord-jdbc-adapter-0.7.2 as it doesn't set the 
       # +@active+ flag to false, thus ActiveRecord#active? incorrectly confirms
       # the connection to still be active.
@@ -115,20 +15,6 @@ module RR
         @active = false
       end
 
-      # Executes the given sql query with the otional name written in the 
-      # ActiveRecord log file.
-      # * +row_buffer_size+: not used.
-      # Returns the results as a Cursor object supporting
-      #   * next? - returns true if there are more rows to read
-      #   * next_row - returns the row as a column => value hash and moves the cursor to the next row
-      #   * clear - clearing the cursor (making allocated memory available for GC)
-      def select_cursor(sql, row_buffer_size = 1000)
-        statement = @connection.connection.createStatement
-        statement.setFetchSize row_buffer_size
-        result_set = statement.executeQuery(sql)
-        result_set.send :extend, JdbcResultSet
-      end
-      
       # Returns an ordered list of primary key column names of the given table
       def primary_key_names(table)
         if not tables.include? table
@@ -169,46 +55,12 @@ module RR
       end
     end
 
-    require 'connection_extenders/postgresql_extender'
-
-    # Adds the correct query executioner functionality to the base class
-    class JdbcPostgreSQLFetcher < PostgreSQLFetcher
-      # Executes the given statements and returns the result set.
-      def execute(sql)
-        statement = connection.instance_variable_get(:@connection).connection.createStatement
-        execute_method = sql =~ /close/i ? :execute : :executeQuery
-        result_set = statement.send(execute_method, sql)
-        result_set.send :extend, RR::ConnectionExtenders::JdbcSQLExtender::JdbcResultSet
-      end
-    end
-
     # PostgreSQL specific functionality not provided by the standard JDBC
     # connection extender:
-    # * Integration of memory efficient select_cursor.
     # * Hack to get schema support for Postgres under JRuby on par with the
     #   standard ruby version.
     module JdbcPostgreSQLExtender
 
-      # Executes the given sql query with the otional name written in the
-      # ActiveRecord log file.
-      #
-      # :+row_buffer_size+ controls how many records are ready into memory at a
-      # time. Implemented using the PostgeSQL "DECLARE CURSOR" and "FETCH" constructs.
-      # This is necessary as the postgresql driver always reads the
-      # complete resultset into memory.
-      #
-      # Returns the results as a Cursor object supporting
-      #   * next? - returns true if there are more rows to read
-      #   * next_row - returns the row as a column => value hash and moves the cursor to the next row
-      #   * clear - clearing the cursor (making allocated memory available for GC)
-      def select_cursor(sql, row_buffer_size = 1000)
-        cursor_name = "RR_#{Time.now.to_i}#{rand(1_000_000)}"
-
-        statement = @connection.connection.createStatement
-        statement.execute("DECLARE #{cursor_name} NO SCROLL CURSOR WITH HOLD FOR " + sql)
-        JdbcPostgreSQLFetcher.new(self, cursor_name, row_buffer_size)
-      end
-
       # Returns the list of a table's column names, data types, and default values.
       #
       # The underlying query is roughly:
@@ -242,11 +94,16 @@ module RR
         end
       end
 
+      require 'jdbc_adapter/jdbc_postgre'
+      class JdbcPostgreSQLColumn < ActiveRecord::ConnectionAdapters::Column
+        include ::JdbcSpec::PostgreSQL::Column
+      end
+
       # Returns the list of all column definitions for a table.
       def columns(table_name, name = nil)
         # Limit, precision, and scale are all handled by the superclass.
         column_definitions(table_name).collect do |name, type, default, notnull|
-          ActiveRecord::ConnectionAdapters::PostgreSQLColumn.new(name, default, type, notnull == 'f')
+          JdbcPostgreSQLColumn.new(name, default, type, notnull == 'f')
         end
       end
 
@@ -281,4 +138,5 @@ module RR
       end
     end
   end
-end
+end
+

data/lib/rubyrep/connection_extenders/mysql_extender.rb

@@ -1,120 +1,11 @@
-# A cursor to iterate over the records returned by select_cursor.
-# Only one row is kept in memory at a time.
-
-module MysqlResultExtender
-  # Returns true if there are more rows to read.
-  def next?
-    @current_row_num ||= 0
-    @num_rows ||= self.num_rows()
-    @current_row_num < @num_rows
-  end
-  
-  # Returns the row as a column => value hash and moves the cursor to the next row.
-  def next_row
-    raise("no more rows available") unless next?
-    row = fetch_hash()
-    @current_row_num += 1
-    row
-  end
-  
-  # Releases the database resources hold by this cursor
-  def clear
-    free
-  end
-end
-
 module RR
 
-  # Overwrites #select_cursor to allow fetching of MySQL results in chunks
-  class ProxyConnection
-
-    # Allow selecting of MySQL results in chunks.
-    # For full documentation of method interface refer to ProxyConnection#select_cursor.
-    def select_cursor_with_mysql_chunks(options)
-      if config[:adapter] != 'mysql' or !options.include?(:row_buffer_size) or options.include?(:query)
-        select_cursor_without_mysql_chunks options
-      else
-        ConnectionExtenders::MysqlFetcher.new(self, options)
-      end
-    end
-    alias_method_chain :select_cursor, :mysql_chunks unless method_defined?(:select_cursor_without_mysql_chunks)
-
-  end
-
   module ConnectionExtenders
 
-    # Fetches MySQL results in chunks
-    class MysqlFetcher
-
-      # The current database ProxyConnection
-      attr_accessor :connection
-
-      # hash of select options
-      attr_accessor :options
-
-      # column_name => value hash of the last returned row
-      attr_accessor :last_row
-
-      # Creates a new fetcher.
-      # * +connection+: the current database connection
-      # * +cursor_name+: name of the cursor from which to fetch
-      # * +row_buffer_size+: number of records to read at once
-      def initialize(connection, options)
-        self.connection = connection
-        self.options = options.clone
-      end
-
-      # Returns +true+ if there are more rows to read.
-      def next?
-        unless @current_result
-          if last_row
-            options.merge! :from => last_row, :exclude_starting_row => true
-          end
-          options[:query] = 
-            connection.table_select_query(options[:table], options) +
-            " limit #{options[:row_buffer_size]}"
-          @current_result = connection.select_cursor_without_mysql_chunks(options)
-        end
-        @current_result.next?
-      end
-
-      # Returns the row as a column => value hash and moves the cursor to the next row.
-      def next_row
-        raise("no more rows available") unless next?
-        self.last_row = @current_result.next_row
-        unless @current_result.next?
-          @current_result.clear
-          @current_result = nil
-        end
-        self.last_row
-      end
-
-      # Closes the cursor and frees up all ressources
-      def clear
-        if @current_result
-          @current_result.clear
-          @current_result = nil
-        end
-      end
-    end
-
     # Provides various MySQL specific functionality required by Rubyrep.
     module MysqlExtender
       RR::ConnectionExtenders.register :mysql => self
 
-      # Executes the given sql query with the optional name written in the 
-      # ActiveRecord log file.
-      # :+row_buffer_size+ is not currently used.
-      # Returns the results as a Cursor object supporting
-      #   * next? - returns true if there are more rows to read
-      #   * next_row - returns the row as a column => value hash and moves the cursor to the next row
-      #   * clear - clearing the cursor (making allocated memory available for GC)
-      def select_cursor(sql, row_buffer_size = 1000)
-        result = execute sql
-        result.send :extend, MysqlResultExtender
-        result
-      end
-      
       # Returns an ordered list of primary key column names of the given table
       def primary_key_names(table)
         row = self.select_one(<<-end_sql)

data/lib/rubyrep/connection_extenders/postgresql_extender.rb

@@ -1,34 +1,5 @@
 require 'time'
 
-# A cursor to iterate over the records returned by select_cursor.
-# Only one row is kept in memory at a time.
-class PGresult
-  # Returns true if there are more rows to read.
-  def next?
-    @current_row_num ||= 0
-    @num_rows ||= self.ntuples()
-    @current_row_num < @num_rows
-  end
-  
-  # Returns the row as a column => value hash and moves the cursor to the next row.
-  def next_row
-    raise("no more rows available") unless next?
-    row = {}
-    @fields ||= self.fields
-    @fields.each_with_index do |field, field_index| 
-      if self.getisnull(@current_row_num, field_index)
-        value = nil
-      else
-        value = self.getvalue @current_row_num, field_index
-      end
-      
-      row[@fields[field_index]] = value
-    end
-    @current_row_num += 1
-    row
-  end
-end
-
 # Hack:
 # For some reasons these methods were removed in Rails 2.2.2, thus breaking
 # the binary and multi-lingual data loading.
@@ -110,83 +81,10 @@ end
 module RR
   module ConnectionExtenders
 
-    # Fetches results from a PostgreSQL cursor object.
-    class PostgreSQLFetcher
-
-      # The current database connection
-      attr_accessor :connection
-
-      # Name of the cursor from which to fetch
-      attr_accessor :cursor_name
-
-      # Number of rows to be read at once
-      attr_accessor :row_buffer_size
-
-      # Creates a new fetcher.
-      # * +connection+: the current database connection
-      # * +cursor_name+: name of the cursor from which to fetch
-      # * +row_buffer_size+: number of records to read at once
-      def initialize(connection, cursor_name, row_buffer_size)
-        self.connection = connection
-        self.cursor_name = cursor_name
-        self.row_buffer_size = row_buffer_size
-      end
-
-      # Executes the specified SQL staements, returning the result
-      def execute(sql)
-        connection.execute sql
-      end
-
-      # Returns true if there are more rows to read.
-      def next?
-        @current_result ||= execute("FETCH FORWARD #{row_buffer_size} FROM #{cursor_name}")
-        @current_result.next?
-      end
-
-      # Returns the row as a column => value hash and moves the cursor to the next row.
-      def next_row
-        raise("no more rows available") unless next?
-        row = @current_result.next_row
-        unless @current_result.next?
-          @current_result.clear
-          @current_result = nil
-        end
-        row
-      end
-
-      # Closes the cursor and frees up all ressources
-      def clear
-        if @current_result
-          @current_result.clear
-          @current_result = nil
-        end
-        result = execute("CLOSE #{cursor_name}")
-        result.clear if result
-      end
-    end
-
     # Provides various PostgreSQL specific functionality required by Rubyrep.
     module PostgreSQLExtender
       RR::ConnectionExtenders.register :postgresql => self
 
-      # Executes the given sql query with the otional name written in the 
-      # ActiveRecord log file.
-      #
-      # :+row_buffer_size+ controls how many records are ready into memory at a
-      # time. Implemented using the PostgeSQL "DECLARE CURSOR" and "FETCH" constructs.
-      # This is necessary as the postgresql driver always reads the
-      # complete resultset into memory.
-      #
-      # Returns the results as a Cursor object supporting
-      #   * next? - returns true if there are more rows to read
-      #   * next_row - returns the row as a column => value hash and moves the cursor to the next row
-      #   * clear - clearing the cursor (making allocated memory available for GC)
-      def select_cursor(sql, row_buffer_size = 1000)
-        cursor_name = "RR_#{Time.now.to_i}#{rand(1_000_000)}"
-        execute("DECLARE #{cursor_name} NO SCROLL CURSOR WITH HOLD FOR " + sql)
-        PostgreSQLFetcher.new(self, cursor_name, row_buffer_size)
-      end
-      
       # Returns an ordered list of primary key column names of the given table
       def primary_key_names(table)
         row = self.select_one(<<-end_sql)

data/lib/rubyrep/logged_change.rb

@@ -46,7 +46,7 @@ module RR
     # 2nd level tree:
     # * key: the change_key value of the according change log records.
     # * value:
-    #   The according change log record (column_name => value hash).
+    #   An array of according change log records (column_name => value hash).
     #   Additional entry of each change log hash:
     #   * key: 'array_index'
     #   * value: index to the change log record in +change_array+
@@ -84,6 +84,19 @@ module RR
       
       self.last_updated = Time.now
 
+      # First, let's use a LIMIT clause (via :row_buffer_size option) to verify
+      # if there are any pending changes.
+      # (If there are many pending changes, this is (at least with PostgreSQL)
+      # much faster.)
+      cursor = connection.select_cursor(
+        :table => change_log_table,
+        :from => {'id' => current_id},
+        :exclude_starting_row => true,
+        :row_buffer_size => 1
+      )
+      return unless cursor.next?
+
+      # Something is here. Let's actually load it.
       cursor = connection.select_cursor(
         :table => change_log_table,
         :from => {'id' => current_id},

data/lib/rubyrep/proxy_connection.rb

@@ -9,6 +9,87 @@ require 'active_record/connection_adapters/abstract_adapter'
 
 module RR
 
+  # Enables the fetching of (potential large) result sets in chunks.
+  class ResultFetcher
+
+    # The current database ProxyConnection
+    attr_accessor :connection
+
+    # hash of select options as described under ProxyConnection#select_cursor
+    attr_accessor :options
+
+    # column_name => value hash of the last returned row
+    attr_accessor :last_row
+
+    # The current row set: an array of column_name => value hashes
+    attr_accessor :rows
+
+    # Index to the current row in rows
+    attr_accessor :current_row_index
+
+    # Creates a new fetcher.
+    # * +connection+: the current ProxyConnection
+    # * +options+: hash of select options as described under ProxyConnection#select_cursor
+    def initialize(connection, options)
+      self.connection = connection
+      self.options = options.clone
+    end
+
+    # Returns +true+ if there are more rows to read.
+    def next?
+      unless self.rows
+        # Try to load some records
+        
+        if options[:query] and last_row != nil
+          # A query was directly specified and all it's rows were returned
+          # ==> Finished.
+          return false
+        end
+
+        if options[:query]
+          # If a query has been directly specified, just directly execute it
+          query = options[:query]
+        else
+          # Otherwise build the query
+          if last_row
+            # There was a previous batch.
+            # Next batch will start after the last returned row
+            options.merge! :from => last_row, :exclude_starting_row => true
+          end
+
+          query = connection.table_select_query(options[:table], options)
+
+          if options[:row_buffer_size]
+            # Set the batch size
+            query += " limit #{options[:row_buffer_size]}"
+          end
+        end
+
+        self.rows = connection.select_all query
+        self.current_row_index = 0
+      end
+      self.current_row_index < self.rows.size
+    end
+
+    # Returns the row as a column => value hash and moves the cursor to the next row.
+    def next_row
+      raise("no more rows available") unless next?
+      self.last_row = self.rows[self.current_row_index]
+      self.current_row_index += 1
+
+      if self.current_row_index == self.rows.size
+        self.rows = nil
+      end
+
+      self.last_row
+    end
+
+    # Frees up all ressources
+    def clear
+      self.rows = nil
+    end
+  end
+
   # This class represents a remote activerecord database connection.
   # Normally created by DatabaseProxy
   class ProxyConnection
@@ -102,9 +183,7 @@ module RR
     # * :+row_buffer_size+:
     #   Integer controlling how many rows a read into memory at one time.
     def select_cursor(options)
-      row_buffer_size = options[:row_buffer_size] || DEFAULT_ROW_BUFFER_SIZE
-      query = options[:query] || table_select_query(options[:table], options)
-      cursor = connection.select_cursor query, row_buffer_size
+      cursor = ResultFetcher.new(self, options)
       if options[:type_cast]
         cursor = TypeCastingCursor.new(self, options[:table], cursor)
       end

data/lib/rubyrep/replication_run.rb

@@ -19,24 +19,31 @@ module RR
 
     # Executes the replication run.
     def run
-      success = false
-      replicator # ensure that replicator is created and has chance to validate settings
+      return unless [:left, :right].any? do |database|
+        session.send(database).select_one(
+          "select id from #{session.configuration.options[:rep_prefix]}_pending_changes"
+        ) != nil
+      end
+      begin
+        success = false
+        replicator # ensure that replicator is created and has chance to validate settings
 
-      loop do
-        begin
-          session.reload_changes # ensure the cache of change log records is up-to-date
-          diff = ReplicationDifference.new session
-          diff.load
-          break unless diff.loaded?
-          replicator.replicate_difference diff if diff.type != :no_diff
-        rescue Exception => e
-          helper.log_replication_outcome diff, e.message, 
-            e.class.to_s + "\n" + e.backtrace.join("\n")
+        loop do
+          begin
+            session.reload_changes # ensure the cache of change log records is up-to-date
+            diff = ReplicationDifference.new session
+            diff.load
+            break unless diff.loaded?
+            replicator.replicate_difference diff if diff.type != :no_diff
+          rescue Exception => e
+            helper.log_replication_outcome diff, e.message,
+              e.class.to_s + "\n" + e.backtrace.join("\n")
+          end
         end
+        success = true # considered to be successful if we get till here
+      ensure
+        helper.finalize success
       end
-      success = true # considered to be successful if we get till here
-    ensure
-      helper.finalize success
     end
 
     # Creates a new ReplicationRun instance.

data/lib/rubyrep/table_spec_resolver.rb

@@ -55,7 +55,7 @@ module RR
     # 
     # Takes care that a table is only returned once.
     def resolve(included_table_specs, excluded_table_specs = [], verify = true)
-      table_pairs = expand_table_specs(included_table_specs)
+      table_pairs = expand_table_specs(included_table_specs, verify)
       table_pairs = table_pairs_without_duplicates(table_pairs)
       table_pairs = table_pairs_without_excluded(table_pairs, excluded_table_specs)
 
@@ -70,11 +70,15 @@ module RR
     end
 
     # Helper for #resolve
-    # Takes the specified table_specifications and expands it into an array of
-    # according table pairs.
-    # Returns the result
-    # Refer to #resolve for a full description of parameters and result.
-    def expand_table_specs(table_specs)
+    # Expands table specifications into table pairs.
+    # Parameters:
+    # * +table_specs+:
+    #   An array of table specifications as described under #resolve.
+    # * +verify+:
+    #   If +true+, table specs in regexp format only resolve if the table exists
+    #   in left and right database.
+    # Return value: refer to #resolve for a detailed description
+    def expand_table_specs(table_specs, verify)
       table_pairs = []
       table_specs.each do |table_spec|
 
@@ -86,7 +90,9 @@ module RR
           table_spec = table_spec.sub(/^\/(.*)\/$/,'\1') # remove leading and trailing slash
           matching_tables = tables(:left).grep(Regexp.new(table_spec, Regexp::IGNORECASE, 'U'))
           matching_tables.each do |table|
-            table_pairs << {:left => table, :right => table}
+            if !verify or tables(:right).include? table
+              table_pairs << {:left => table, :right => table}
+            end
           end
         when /.+,.+/ # matches e. g. 'users,users_backup'
           pair = table_spec.match(/(.*),(.*)/)[1..2].map { |str| str.strip }
@@ -107,7 +113,7 @@ module RR
     # * :+right+: name of the corresponding right table
     # +excluded_table_specs+ is the array of table specifications to be excluded.
     def table_pairs_without_excluded(table_pairs, excluded_table_specs)
-      excluded_tables = expand_table_specs(excluded_table_specs).map do |table_pair|
+      excluded_tables = expand_table_specs(excluded_table_specs, false).map do |table_pair|
         table_pair[:left]
       end
       table_pairs.select {|table_pair| not excluded_tables.include? table_pair[:left]}

data/lib/rubyrep/version.rb

@@ -2,7 +2,7 @@ module RR #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 1
     MINOR = 0
-    TINY  = 1
+    TINY  = 2
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/spec/connection_extender_interface_spec.rb

@@ -1,5 +1,6 @@
 require File.dirname(__FILE__) + '/spec_helper.rb'
 require 'yaml'
+require 'digest/md5'
 
 include RR
 
@@ -89,20 +90,27 @@ describe "ConnectionExtender", :shared => true do
     )
     result.next_row.should == {'first_id' => '3', 'second_id' => '1', 'name' => nil}
   end
-  
+
   it "should read and write binary data correctly" do
     session = Session.new
 
-    org_data = Marshal.dump(['bla',:dummy,1,2,3])
+    org_data = File.new(File.dirname(__FILE__) + '/dolphins.jpg').read
     result_data = nil
     begin
       session.left.begin_db_transaction
-      sql = "insert into extender_type_check(id, binary_test) values(2, '#{org_data}')"
-      session.left.execute sql
+      session.left.insert_record('extender_type_check', {'id' => 6, 'binary_test' => org_data})
+      
+      row = session.left.select_one(
+        'select md5(binary_test) as md5 from extender_type_check where id = 6'
+      )
+      row['md5'].should == Digest::MD5.hexdigest(org_data)
 
-      org_cursor = session.left.select_cursor(:query => "select id, binary_test from extender_type_check where id = 2")
-      cursor = TypeCastingCursor.new session.left, 'extender_type_check', org_cursor
+      cursor = session.left.select_cursor(
+        :query => "select id, binary_test from extender_type_check where id = 6"
+      )
+      cursor = TypeCastingCursor.new session.left, 'extender_type_check', cursor
       result_data = cursor.next_row['binary_test']
+      Digest::MD5.hexdigest(result_data).should == Digest::MD5.hexdigest(org_data)
     ensure
       session.left.rollback_db_transaction
     end

data/spec/connection_extenders_registration_spec.rb

@@ -60,8 +60,8 @@ describe ConnectionExtenders, "Registration" do
   
   it "db_connect should include the connection extender into connection" do
     connection = ConnectionExtenders.db_connect Initializer.configuration.left
-    
-    connection.should respond_to(:select_cursor)
+
+    connection.should respond_to(:primary_key_names)
   end
   
   it "db_connect should raise an Exception if no fitting connection extender is available" do

data/spec/proxy_connection_spec.rb

@@ -120,28 +120,20 @@ describe ProxyConnection do
     @connection.primary_key_names('dummy_table').should == ['dummy_key']
   end
 
-  it "select_cursor should forwarded the handed over query" do
-    @connection.connection.should_receive(:select_cursor).with('bla', 123)
-    @connection.select_cursor(:query => 'bla', :row_buffer_size => 123)
-  end
-
-  it "select_cursor should use the default row buffer size if no explicit value is specified" do
-    @connection.connection.should_receive(:select_cursor) \
-      .with('bla', ProxyConnection::DEFAULT_ROW_BUFFER_SIZE)
-    @connection.select_cursor(:query => 'bla')
-  end
+  # Note:
+  # Additional select_cursor tests are executed via
+  # 'db_specific_connection_extenders_spec.rb'
+  # (To verify the behaviour for all supported databases)
 
-  it "select_cursor should construct the query and execute it" do
-    @connection.connection.should_receive(:select_cursor) \
-      .with(sql_to_regexp('select "id", "name" from "scanner_records" order by "id"'), ProxyConnection::DEFAULT_ROW_BUFFER_SIZE)
-    @connection.select_cursor(:table => 'scanner_records')
+  it "select_cursor should return the result fetcher" do
+    fetcher = @connection.select_cursor(:table => 'scanner_records')
+    fetcher.connection.should == @connection
+    fetcher.options.should == {:table => 'scanner_records'}
   end
 
   it "select_cursor should return a type casting cursor if :type_cast option is specified" do
-    @connection.select_cursor(:table => 'scanner_records').
-      should_not be_an_instance_of(TypeCastingCursor)
-    @connection.select_cursor(:table => 'scanner_records', :type_cast => true).
-      should be_an_instance_of(TypeCastingCursor)
+    fetcher = @connection.select_cursor(:table => 'scanner_records', :type_cast => true)
+    fetcher.should be_an_instance_of(TypeCastingCursor)
   end
 
   it "table_select_query should handle queries without any conditions" do

data/spec/replication_run_spec.rb

@@ -64,6 +64,13 @@ describe ReplicationRun do
     end
   end
 
+  it "run should not create the replicator if there are no pending changes" do
+    session = Session.new
+    run = ReplicationRun.new session
+    run.should_not_receive(:replicator)
+    run.run
+  end
+
   it "run should only replicate real differences" do
     session = Session.new
     session.left.begin_db_transaction
@@ -121,8 +128,22 @@ describe ReplicationRun do
   it "run should not catch exceptions raised during replicator initialization" do
     config = deep_copy(standard_config)
     config.options[:logged_replication_events] = [:invalid_option]
-    run = ReplicationRun.new Session.new(config)
-    lambda {run.run}.should raise_error(ArgumentError)
+    session = Session.new config
+    session.left.begin_db_transaction
+    begin
+
+      session.left.insert_record 'rr_pending_changes', {
+        'change_table' => 'extender_no_record',
+        'change_key' => 'id|1',
+        'change_type' => 'D',
+        'change_time' => Time.now
+      }
+
+      run = ReplicationRun.new session
+      lambda {run.run}.should raise_error(ArgumentError)
+    ensure
+      session.left.rollback_db_transaction
+    end
   end
 
   it "run should process trigger created change log records" do

data/spec/table_spec_resolver_spec.rb

@@ -34,6 +34,15 @@ describe TableSpecResolver do
   it "resolve should complain about non-existing tables" do
     lambda {@resolver.resolve(['dummy, scanner_records'])}.
       should raise_error(/non-existing.*dummy/)
+    lambda {@resolver.resolve(['left_table, left_table'])}.
+      should raise_error(/non-existing.*left_table/)
+    lambda {@resolver.resolve(['left_table'])}.
+      should raise_error(/non-existing.*left_table/)
+  end
+
+  it "resolve should not complain about regexp specified tables not existing in right database" do
+    @resolver.resolve([/^scanner_records$/, /left_table/]).
+      should == [{:left => 'scanner_records', :right => 'scanner_records'}]
   end
 
   it "resolve should not check for non-existing tables if that is disabled" do
@@ -85,7 +94,7 @@ describe TableSpecResolver do
     @resolver.non_existing_tables(table_pairs).should == {}
   end
 
-  it "non_existing_tables should return an empty hash if all tables exist" do
+  it "non_existing_tables should return a hash of non-existing tables" do
     table_pairs = [{:left => 'scanner_records', :right => 'bla'}]
     @resolver.non_existing_tables(table_pairs).should == {:right => ['bla']}
 

data/tasks/database.rake

@@ -102,11 +102,13 @@ def drop_postgres_schema(config)
 end
 
 # Creates the sample schema in the database specified by the given 
-# configuration hash.
-def create_sample_schema(config)
-  create_postgres_schema config
+# configuration.
+# * :+database+: either :+left+ or +:right+
+# * :+config+: the Configuration object
+def create_sample_schema(database, config)
+  create_postgres_schema config.send(database)
 
-  ActiveRecord::Base.establish_connection config
+  ActiveRecord::Base.establish_connection config.send(database)
   
   ActiveRecord::Schema.define do
     create_table :scanner_text_key, :id => false do |t|
@@ -252,11 +254,11 @@ def create_sample_schema(config)
 
     create_table :left_table do |t|
       t.column :name, :string
-    end
+    end if database == :left
 
     create_table :right_table do |t|
       t.column :name, :string
-    end
+    end if database == :right
   end
 end
 
@@ -410,8 +412,8 @@ namespace :db do
     
     desc "Create the sample schemas"
     task :create_schema do
-      create_sample_schema RR::Initializer.configuration.left rescue nil
-      create_sample_schema RR::Initializer.configuration.right rescue nil
+      create_sample_schema :left, RR::Initializer.configuration rescue nil
+      create_sample_schema :right, RR::Initializer.configuration rescue nil
     end
     
     desc "Writes the sample data"

data/tasks/java.rake

@@ -23,7 +23,8 @@ EOS
     pkg_name = "rubyrep-#{RR::VERSION::STRING}"
 
     system "rm -rf /tmp/#{pkg_name}"
-    system "hg archive /tmp/#{pkg_name}"
+    system "mkdir /tmp/#{pkg_name}"
+    system "git archive master |tar -x -C /tmp/#{pkg_name}"
     system "mkdir -p /tmp/#{pkg_name}/jruby"
     system "cp -r #{JRUBY_HOME}/* /tmp/#{pkg_name}/jruby/"
     system "cd /tmp/#{pkg_name}/jruby; rm -rf samples share/ri lib/ruby/gems/1.8/doc"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: rubyrep
 version: !ruby/object:Gem::Version 
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors: 
 - Arndt Lehmann
@@ -30,7 +30,7 @@ cert_chain:
   NwT26VZnE2nr8g==
   -----END CERTIFICATE-----
 
-date: 2009-03-25 00:00:00 +09:00
+date: 2009-06-13 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency