rubyrep 1.0.0

data/lib/rubyrep/connection_extenders/connection_extenders.rb

@@ -0,0 +1,133 @@
+class ActiveRecord::ConnectionAdapters::Column
+  # Bug in ActiveRecord parsing of PostgreSQL timestamps with microseconds:
+  # Certain values are incorrectly rounded, thus ending up with timestamps
+  # that are off by one microsecond.
+  # This monkey patch fixes the problem.
+  def self.fast_string_to_time(string)
+    if string =~ Format::ISO_DATETIME
+      microsec = ($7.to_f * 1_000_000).round # used to be #to_i instead
+      new_time $1.to_i, $2.to_i, $3.to_i, $4.to_i, $5.to_i, $6.to_i, microsec
+    end
+  end
+end
+
+module RR
+  
+  # Connection extenders provide additional database specific functionality
+  # not coming in the ActiveRecord library.
+  # This module itself only provides functionality to register and retrieve
+  # such connection extenders.
+  module ConnectionExtenders
+    # Returns a Hash of currently registered connection extenders.
+    # (Empty Hash if no connection extenders were defined.)
+    def self.extenders
+      @extenders ||= {}
+      @extenders
+    end
+  
+    # Registers one or multiple connection extender.
+    # extender is a Hash with 
+    #   key::   The adapter symbol as used by ActiveRecord::Connection Adapters, e. g. :postgresql
+    #   value:: Name of the module implementing the connection extender
+    def self.register(extender)
+      @extenders ||= {}
+      @extenders.merge! extender
+    end
+
+    # Dummy ActiveRecord descendant only used to create database connections.
+    class DummyActiveRecord < ActiveRecord::Base
+    end
+    
+    # Creates an ActiveRecord database connection according to the provided +config+ connection hash.
+    # Possible values of this parameter are described in ActiveRecord::Base#establish_connection.
+    # The database connection is extended with the correct ConnectionExtenders module.
+    # 
+    # ActiveRecord only allows one database connection per class.
+    # (It disconnects the existing database connection if a new connection is established.)
+    # To go around this, we delete ActiveRecord's memory of the existing database connection
+    # as soon as it is created.
+    def self.db_connect_without_cache(config)
+      if RUBY_PLATFORM =~ /java/
+        adapter = config[:adapter]
+        
+        # As recommended in the activerecord-jdbc-adapter use the jdbc versions
+        # of the Adapters. E. g. instead of "postgresql", "jdbcpostgresql".
+        adapter = 'jdbc' + adapter unless adapter =~ /^jdbc/
+
+        DummyActiveRecord.establish_connection(config.merge(:adapter => adapter))
+      else
+        DummyActiveRecord.establish_connection(config)
+      end
+      connection = DummyActiveRecord.connection
+
+      # Delete the database connection from ActiveRecords's 'memory'
+      ActiveRecord::Base.connection_handler.connection_pools.delete DummyActiveRecord.name
+      
+      extender = ""
+      if RUBY_PLATFORM =~ /java/
+        extender = :jdbc
+      elsif ConnectionExtenders.extenders.include? config[:adapter].to_sym
+        extender = config[:adapter].to_sym
+      else
+        raise "No ConnectionExtender available for :#{config[:adapter]}"
+      end
+      connection_module = ConnectionExtenders.extenders[extender]
+      connection.extend connection_module
+
+      # Hack to get Postgres schema support under JRuby to par with the standard
+      # ruby version
+      if RUBY_PLATFORM =~ /java/ and config[:adapter].to_sym == :postgresql
+        connection.extend RR::ConnectionExtenders::JdbcPostgreSQLExtender
+        connection.initialize_search_path
+      end
+
+      replication_module = ReplicationExtenders.extenders[config[:adapter].to_sym]
+      connection.extend replication_module if replication_module
+      
+      connection
+    end
+    
+    @@use_cache = true
+    
+    # Returns the current cache status (+true+ if caching is used; +false+ otherwise).
+    def self.use_cache?; @@use_cache; end
+    
+    # Returns the connection cache hash.
+    def self.connection_cache; @@connection_cache; end
+    
+    # Sets a new connection cache
+    def self.connection_cache=(cache)
+      @@connection_cache = cache
+    end
+    
+    # Creates database connections by calling #db_connect_without_cache with the 
+    # provided +config+ configuration hash.
+    # A new database connection is created only if no according cached connection
+    # is available.
+    def self.db_connect(config)
+      config_dump = Marshal.dump config.reject {|key, | [:proxy_host, :proxy_port].include? key}
+      config_checksum = Digest::SHA1.hexdigest(config_dump)
+      @@connection_cache ||= {}
+      cached_db_connection = connection_cache[config_checksum]
+      if use_cache? and cached_db_connection and cached_db_connection.active?
+        cached_db_connection
+      else
+        db_connection = db_connect_without_cache config
+        connection_cache[config_checksum] = db_connection if @@use_cache
+        db_connection
+      end
+    end
+    
+    # If status == true: enable the cache. If status == false: don' use cache
+    # Returns the old connection caching status
+    def self.use_db_connection_cache(status)
+      old_status, @@use_cache = @@use_cache, status
+      old_status
+    end
+    
+    # Free up all cached connections
+    def self.clear_db_connection_cache
+      @@connection_cache = {}
+    end
+  end
+end

data/lib/rubyrep/connection_extenders/jdbc_extender.rb

@@ -0,0 +1,284 @@
+require 'java'
+
+module RR
+  module ConnectionExtenders
+
+    # Provides various JDBC specific functionality required by Rubyrep.
+    module JdbcSQLExtender
+      RR::ConnectionExtenders.register :jdbc => self
+      
+      # A cursor to iterate over the records returned by select_cursor.
+      # Only one row is kept in memory at a time.
+      module JdbcResultSet
+        # Returns true if there are more rows to read.
+        def next?
+          if @next_status == nil
+            @next_status = self.next
+          end
+          @next_status
+        end
+        
+        # Returns the row as a column => value hash and moves the cursor to the next row.
+        def next_row
+          raise("no more rows available") unless next?
+          @next_status = nil
+
+          unless @columns
+            meta_data = self.getMetaData
+            stores_upper = self.getStatement.getConnection.getMetaData.storesUpperCaseIdentifiers
+            column_count = meta_data.getColumnCount
+            @columns = Array.new(column_count)
+            @columns.each_index do |i|
+              column_name = meta_data.getColumnName(i+1)
+              if stores_upper and not column_name =~ /[a-z]/
+                column_name.downcase!
+              end
+              @columns[i] = {
+                :index => i+1,
+                :name => column_name,
+                :type => meta_data.getColumnType(i+1)
+                #:scale => meta_data.getScale(i+1)
+              }
+            end
+          end
+
+          row = {}
+          @columns.each_index do |i|
+            row[@columns[i][:name]] = jdbc_to_ruby(@columns[i])
+          end
+
+          row
+        end
+        
+        # Releases the databases resources hold by this cursor
+        def clear
+          @columns = nil
+          self.close
+        end
+
+        Types = java.sql.Types unless const_defined?(:Types)
+        
+        # Converts the specified column of the current row to the proper ruby string
+        # column is a hash with the following elements:
+        #   * :index: field number (starting with 1) of the result set field
+        #   * :type: the java.sql.Type constant specifying the type of the result set field
+        def jdbc_to_ruby(column)
+          case column[:type]
+          when Types::BINARY, Types::BLOB, Types::LONGVARBINARY, Types::VARBINARY
+            is = self.getBinaryStream(column[:index])
+            if is == nil or self.wasNull
+              return nil
+            end
+            byte_list = org.jruby.util.ByteList.new(2048)
+            buffer = Java::byte[2048].new
+            while (n = is.read(buffer)) != -1
+              byte_list.append(buffer, 0, n)
+            end
+            is.close
+            return byte_list.toString
+          when Types::LONGVARCHAR, Types::CLOB
+            rss = self.getCharacterStream(column[:index])
+            if rss == nil or self.wasNull
+              return nil
+            end
+            str = java.lang.StringBuffer.new(2048)
+            cuf = Java::char[2048].new
+            while (n = rss.read(cuf)) != -1
+              str.append(cuf, 0, n)
+            end
+            rss.close
+            return str.toString
+          when Types::TIMESTAMP
+            time = self.getTimestamp(column[:index]);
+            if  time == nil or self.wasNull
+              return nil
+            end
+            time_string = time.toString()
+            time_string = time_string.gsub(/ 00:00:00.0$/, '')
+            return time_string
+          else
+            value = self.getString(column[:index])
+            if value == nil or self.wasNull
+              return nil
+            end
+            return value
+          end
+        end
+        private :jdbc_to_ruby
+      end
+
+      # Monkey patch for activerecord-jdbc-adapter-0.7.2 as it doesn't set the 
+      # +@active+ flag to false, thus ActiveRecord#active? incorrectly confirms
+      # the connection to still be active.
+      def disconnect!
+        super
+        @active = false
+      end
+
+      # Executes the given sql query with the otional name written in the 
+      # ActiveRecord log file.
+      # * +row_buffer_size+: not used.
+      # Returns the results as a Cursor object supporting
+      #   * next? - returns true if there are more rows to read
+      #   * next_row - returns the row as a column => value hash and moves the cursor to the next row
+      #   * clear - clearing the cursor (making allocated memory available for GC)
+      def select_cursor(sql, row_buffer_size = 1000)
+        statement = @connection.connection.createStatement
+        statement.setFetchSize row_buffer_size
+        result_set = statement.executeQuery(sql)
+        result_set.send :extend, JdbcResultSet
+      end
+      
+      # Returns an ordered list of primary key column names of the given table
+      def primary_key_names(table)
+        if not tables.include? table
+          raise "table '#{table}' does not exist"
+        end
+        columns = []
+        result_set = @connection.connection.getMetaData.getPrimaryKeys(nil, nil, table);
+        while result_set.next
+          column_name = result_set.getString("COLUMN_NAME")
+          key_seq = result_set.getShort("KEY_SEQ")
+          columns << {:column_name => column_name, :key_seq => key_seq}
+        end
+        columns.sort! {|a, b| a[:key_seq] <=> b[:key_seq]}
+        key_names = columns.map {|column| column[:column_name]}
+        key_names
+      end
+
+      # Returns for each given table, which other tables it references via
+      # foreign key constraints.
+      # * tables: an array of table names
+      # * returns: a hash with
+      #   * key: name of the referencing table
+      #   * value: an array of names of referenced tables
+      def referenced_tables(tables)
+        result = {}
+        tables.each do |table|
+          references_of_this_table = []
+          result_set = @connection.connection.getMetaData.getImportedKeys(nil, nil, table)
+          while result_set.next
+            referenced_table = result_set.getString("PKTABLE_NAME")
+            unless references_of_this_table.include? referenced_table
+              references_of_this_table << referenced_table
+            end
+          end
+          result[table] = references_of_this_table
+        end
+        result
+      end
+    end
+
+    require 'connection_extenders/postgresql_extender'
+
+    # Adds the correct query executioner functionality to the base class
+    class JdbcPostgreSQLFetcher < PostgreSQLFetcher
+      # Executes the given statements and returns the result set.
+      def execute(sql)
+        statement = connection.instance_variable_get(:@connection).connection.createStatement
+        execute_method = sql =~ /close/i ? :execute : :executeQuery
+        result_set = statement.send(execute_method, sql)
+        result_set.send :extend, RR::ConnectionExtenders::JdbcSQLExtender::JdbcResultSet
+      end
+    end
+
+    # PostgreSQL specific functionality not provided by the standard JDBC
+    # connection extender:
+    # * Integration of memory efficient select_cursor.
+    # * Hack to get schema support for Postgres under JRuby on par with the
+    #   standard ruby version.
+    module JdbcPostgreSQLExtender
+
+      # Executes the given sql query with the otional name written in the
+      # ActiveRecord log file.
+      #
+      # :+row_buffer_size+ controls how many records are ready into memory at a
+      # time. Implemented using the PostgeSQL "DECLARE CURSOR" and "FETCH" constructs.
+      # This is necessary as the postgresql driver always reads the
+      # complete resultset into memory.
+      #
+      # Returns the results as a Cursor object supporting
+      #   * next? - returns true if there are more rows to read
+      #   * next_row - returns the row as a column => value hash and moves the cursor to the next row
+      #   * clear - clearing the cursor (making allocated memory available for GC)
+      def select_cursor(sql, row_buffer_size = 1000)
+        cursor_name = "RR_#{Time.now.to_i}#{rand(1_000_000)}"
+
+        statement = @connection.connection.createStatement
+        statement.execute("DECLARE #{cursor_name} NO SCROLL CURSOR WITH HOLD FOR " + sql)
+        JdbcPostgreSQLFetcher.new(self, cursor_name, row_buffer_size)
+      end
+
+      # Returns the list of a table's column names, data types, and default values.
+      #
+      # The underlying query is roughly:
+      #  SELECT column.name, column.type, default.value
+      #    FROM column LEFT JOIN default
+      #      ON column.table_id = default.table_id
+      #     AND column.num = default.column_num
+      #   WHERE column.table_id = get_table_id('table_name')
+      #     AND column.num > 0
+      #     AND NOT column.is_dropped
+      #   ORDER BY column.num
+      #
+      # If the table name is not prefixed with a schema, the database will
+      # take the first match from the schema search path.
+      #
+      # Query implementation notes:
+      #  - format_type includes the column size constraint, e.g. varchar(50)
+      #  - ::regclass is a function that gives the id for a table name
+      def column_definitions(table_name) #:nodoc:
+        rows = select_all(<<-end_sql)
+            SELECT a.attname as name, format_type(a.atttypid, a.atttypmod) as type, d.adsrc as default, a.attnotnull as notnull
+              FROM pg_attribute a LEFT JOIN pg_attrdef d
+                ON a.attrelid = d.adrelid AND a.attnum = d.adnum
+             WHERE a.attrelid = '#{table_name}'::regclass
+               AND a.attnum > 0 AND NOT a.attisdropped
+             ORDER BY a.attnum
+        end_sql
+          
+        rows.map do |row|
+          [row['name'], row['type'], row['default'], row['notnull']]
+        end
+      end
+
+      # Returns the list of all column definitions for a table.
+      def columns(table_name, name = nil)
+        # Limit, precision, and scale are all handled by the superclass.
+        column_definitions(table_name).collect do |name, type, default, notnull|
+          ActiveRecord::ConnectionAdapters::PostgreSQLColumn.new(name, default, type, notnull == 'f')
+        end
+      end
+
+      # Sets the schema search path as per configuration parameters
+      def initialize_search_path
+        execute "SET search_path TO #{config[:schema_search_path]}" if config[:schema_search_path]
+      end
+
+      # Returns the active schema search path.
+      def schema_search_path
+        @schema_search_path ||= select_one('SHOW search_path')['search_path']
+      end
+
+      # Returns the list of all tables in the schema search path or a specified schema.
+      def tables(name = nil)
+        schemas = schema_search_path.split(/,/).map { |p| quote(p) }.join(',')
+        select_all(<<-SQL, name).map { |row| row['tablename'] }
+          SELECT tablename
+            FROM pg_tables
+           WHERE schemaname IN (#{schemas})
+        SQL
+      end
+
+      # Converts the given Time object into the correctly formatted string
+      # representation.
+      # 
+      # Monkeypatched as activerecord-jdbcpostgresql-adapter (at least in version
+      # 0.8.2) does otherwise "loose" the microseconds when writing Time values
+      # to the database.
+      def quoted_date(value)
+        "#{value.strftime("%Y-%m-%d %H:%M:%S")}#{value.respond_to?(:usec) ? ".#{value.usec.to_s.rjust(6, '0')}" : ""}"
+      end
+    end
+  end
+end

data/lib/rubyrep/connection_extenders/mysql_extender.rb

@@ -0,0 +1,168 @@
+# A cursor to iterate over the records returned by select_cursor.
+# Only one row is kept in memory at a time.
+
+module MysqlResultExtender
+  # Returns true if there are more rows to read.
+  def next?
+    @current_row_num ||= 0
+    @num_rows ||= self.num_rows()
+    @current_row_num < @num_rows
+  end
+  
+  # Returns the row as a column => value hash and moves the cursor to the next row.
+  def next_row
+    raise("no more rows available") unless next?
+    row = fetch_hash()
+    @current_row_num += 1
+    row
+  end
+  
+  # Releases the database resources hold by this cursor
+  def clear
+    free
+  end
+end
+
+module RR
+
+  # Overwrites #select_cursor to allow fetching of MySQL results in chunks
+  class ProxyConnection
+
+    # Allow selecting of MySQL results in chunks.
+    # For full documentation of method interface refer to ProxyConnection#select_cursor.
+    def select_cursor_with_mysql_chunks(options)
+      if config[:adapter] != 'mysql' or !options.include?(:row_buffer_size) or options.include?(:query)
+        select_cursor_without_mysql_chunks options
+      else
+        ConnectionExtenders::MysqlFetcher.new(self, options)
+      end
+    end
+    alias_method_chain :select_cursor, :mysql_chunks unless method_defined?(:select_cursor_without_mysql_chunks)
+
+  end
+
+  module ConnectionExtenders
+
+    # Fetches MySQL results in chunks
+    class MysqlFetcher
+
+      # The current database ProxyConnection
+      attr_accessor :connection
+
+      # hash of select options
+      attr_accessor :options
+
+      # column_name => value hash of the last returned row
+      attr_accessor :last_row
+
+      # Creates a new fetcher.
+      # * +connection+: the current database connection
+      # * +cursor_name+: name of the cursor from which to fetch
+      # * +row_buffer_size+: number of records to read at once
+      def initialize(connection, options)
+        self.connection = connection
+        self.options = options.clone
+      end
+
+      # Returns +true+ if there are more rows to read.
+      def next?
+        unless @current_result
+          if last_row
+            options.merge! :from => last_row, :exclude_starting_row => true
+          end
+          options[:query] = 
+            connection.table_select_query(options[:table], options) +
+            " limit #{options[:row_buffer_size]}"
+          @current_result = connection.select_cursor_without_mysql_chunks(options)
+        end
+        @current_result.next?
+      end
+
+      # Returns the row as a column => value hash and moves the cursor to the next row.
+      def next_row
+        raise("no more rows available") unless next?
+        self.last_row = @current_result.next_row
+        unless @current_result.next?
+          @current_result.clear
+          @current_result = nil
+        end
+        self.last_row
+      end
+
+      # Closes the cursor and frees up all ressources
+      def clear
+        if @current_result
+          @current_result.clear
+          @current_result = nil
+        end
+      end
+    end
+
+    # Provides various MySQL specific functionality required by Rubyrep.
+    module MysqlExtender
+      RR::ConnectionExtenders.register :mysql => self
+
+      # Executes the given sql query with the optional name written in the 
+      # ActiveRecord log file.
+      # :+row_buffer_size+ is not currently used.
+      # Returns the results as a Cursor object supporting
+      #   * next? - returns true if there are more rows to read
+      #   * next_row - returns the row as a column => value hash and moves the cursor to the next row
+      #   * clear - clearing the cursor (making allocated memory available for GC)
+      def select_cursor(sql, row_buffer_size = 1000)
+        result = execute sql
+        result.send :extend, MysqlResultExtender
+        result
+      end
+      
+      # Returns an ordered list of primary key column names of the given table
+      def primary_key_names(table)
+        row = self.select_one(<<-end_sql)
+          select table_name from information_schema.tables 
+          where table_schema = database() and table_name = '#{table}'
+        end_sql
+        if row.nil?
+          raise "table '#{table}' does not exist"
+        end
+        
+        rows = self.select_all(<<-end_sql)
+          select column_name from information_schema.key_column_usage
+          where table_schema = database() and table_name = '#{table}' 
+          and constraint_name = 'PRIMARY'
+          order by ordinal_position
+        end_sql
+
+        columns = rows.map {|_row| _row['column_name']}
+        columns
+      end
+
+      # Returns for each given table, which other tables it references via
+      # foreign key constraints.
+      # * tables: an array of table names
+      # Returns: a hash with
+      # * key: name of the referencing table
+      # * value: an array of names of referenced tables
+      def referenced_tables(tables)
+        rows = self.select_all(<<-end_sql)
+          select distinct table_name as referencing_table, referenced_table_name as referenced_table
+          from information_schema.key_column_usage
+          where table_schema = database()
+          and table_name in ('#{tables.join("', '")}')
+        end_sql
+        result = {}
+        rows.each do |row|
+          unless result.include? row['referencing_table']
+            result[row['referencing_table']] = []
+          end
+          if row['referenced_table'] != nil
+            result[row['referencing_table']] << row['referenced_table']
+          end
+        end
+        tables.each do |table|
+          result[table] = [] unless result.include? table
+        end
+        result
+      end
+    end
+  end
+end