xbar 0.0.1 → 0.4.0

data/lib/xbar/proxy.rb

@@ -0,0 +1,249 @@
+require "set"
+
+# For debugging.
+require File.expand_path(File.join(File.dirname(__FILE__),  'colors'))
+
+class UsageStatistics < ActiveRecord::Base
+end
+
+class XBar::Proxy
+  
+  include XBar::Mapper
+  include Colors
+
+  # No setter method.    
+  attr_reader :last_current_shard
+  
+  # Setters for these are written by hand below.
+  attr_reader :current_model, :current_shard
+  
+  attr_reader :shard_list
+ 
+  def initialize
+    puts "Initializing new proxy."
+    register
+    reset_shards
+    clean_proxy
+  end
+  
+  def reset_proxy
+    @current_shard = :master
+    puts ">>> reset_proxy: #{current_shard}" if XBar.debug
+    clear_block_scope
+    reset_shards
+  end
+  
+  def clean_proxy
+    if XBar.debug
+      puts "Proxy##{BLUE_TEXT}clean_proxy=#{RESET_COLORS}: " +
+        "current shard = #{@current_shard}"
+    end
+    @current_shard = :master
+    clear_block_scope
+  end
+   
+  def current_shard=(shard_name)
+    # shard name might actually be a list of shard names in the 
+    # case of migration.  Make it an array in all cases to check it.
+    Array(shard_name).each do |s|
+      if !@shard_list.member? s
+        puts "<<< #{@shard_list.keys} >>>"
+        puts "<<< #{XBar::Mapper.shards.keys} >>>"
+        raise "Nonexistent Shard Name: #{s}"
+      end
+    end
+    if XBar.debug
+      puts "Proxy##{BLUE_TEXT}current_shard=#{RESET_COLORS}: " +
+        "previous_shard = #{@current_shard}, new_shard = #{shard_name}"
+    end
+    @current_shard = shard_name
+  end
+
+  def select_shard
+    if current_shard.kind_of? Array
+      shard = current_shard.first
+      if current_shard.size != 1
+        puts "WARNING: selecting only first shard from array"
+      end
+    else
+      shard = current_shard
+    end
+    unless @shard_list[shard] 
+       puts "Shard not found: current_shard = #{shard}, @shard_list = #{@shard_list.keys}"     
+    end
+    @shard_list[shard]
+  end
+
+  def current_model=(model)
+    # The way that this function is used internally, kind_of?(ActiveRecord::Base)
+    # is always false -- we're always passing the class.cur
+    @current_model = model.kind_of?(ActiveRecord::Base) ? model.class : model
+  end
+  
+  def enter_block_scope
+    @old_in_block = @in_block
+    @depth += 1
+    @in_block = true
+  end
+  
+  def leave_block_scope
+    @in_block = @old_in_block
+    @old_in_block = false
+    @depth -= 1
+  end
+  
+  def in_block_scope?
+    @in_block
+  end
+  
+  def clear_block_scope
+    @in_block = @old_in_block = false
+    @depth = 0
+  end
+  
+  def should_clean_table_name?
+    adapters.size > 1
+  end
+  
+  def verify_connection
+    XBar::Mapper.options[:verify_connection]
+  end
+
+  def run_queries_on_shard(shard_name, use_scope = true)
+    older_shard = current_shard 
+    enter_block_scope if use_scope
+    self.current_shard = shard_name
+    if XBar.debug
+      puts "Proxy##{BLUE_TEXT}run_queries_on_shard#{RESET_COLORS}: " +
+        "current shard = #{current_shard}, use_scope = #{use_scope}, " +
+        "previous shard = #{older_shard}"
+    end
+    result = yield
+  ensure
+    if XBar.debug
+      puts "Proxy##{BLUE_TEXT}run_queries_on_shard#{RESET_COLORS}: " +
+        "restoring previous shard = #{older_shard}"
+    end
+    leave_block_scope if use_scope
+    self.current_shard = older_shard
+  end
+
+  def send_queries_to_multiple_shards(shard_names, &block)
+    shard_names = Array(shard_names)
+    shard_names.each do |name|
+      run_queries_on_shard(name, &block)
+    end
+  end
+
+  def check_schema_migrations(shard_name)
+    @shard_list[shard_name].check_schema_migrations
+  end
+
+  def transaction(options = {}, &block)
+    select_shard.transaction(options, &block)
+  end
+
+  def schema_cache
+    select_shard.schema_cache
+  end
+
+  def quote_table_name(table_name)
+    select_shard.quote_table_name(table_name)
+  end
+
+  def method_missing(method, *args, &block)
+    if XBar.debug
+      puts("\nProxy##{BLUE_TEXT}method_missing#{RESET_COLORS}: " + 
+        "method = #{RED_TEXT}#{method}#{RESET_COLORS}, " +
+        "current_shard=#{current_shard}, " +
+        "in_block_scope=#{in_block_scope?}")
+    end
+
+    if method.to_s =~ /insert|select|execute/ && !in_block_scope? # should clean connection
+      shard = @last_current_shard = current_shard
+      clean_proxy
+      @shard_list[shard].run_queries(method, *args, &block)
+    else
+      select_shard.run_queries(method, *args, &block)
+    end
+  end
+  
+  def respond_to?(method, include_private = false)
+    super || current_shard.respond_to?(method, include_private)
+  end
+
+  def connection_pool
+    cp = shards[current_shard].first 
+    cp.automatic_reconnect = true if XBar.rails31?
+    cp
+  end
+
+  def enter_statistics(shard_name, config, method)
+   
+    return unless XBar.collect_stats?
+   
+    saved_current_model = current_model
+    if UsageStatistics.connection.kind_of?(XBar::Proxy)
+      connection_pool = UsageStatistics.connection.shards[:master][0]
+      connection_pool.automatic_reconnect = true
+      connection = connection_pool.connection
+     
+      return unless connection.table_exists? :usage_statistics
+
+      # We have an AbstractAdapter.  We must insert statistics using this
+      # adapter, not the Proxy, because otherwise the insertion triggers
+      # 'enter_statistics' again and we would be in an infinite loop.
+
+=begin
+      params = {
+        shard_name: shard_name,
+        method: method.to_s,
+        adapter: config[:adapter],
+        username: config[:username],
+        thread_id: Thread.current.object_id.to_s,
+        port: (config[:port] || "default"),
+        host: config[:host],
+        database_name: config[:database]}
+      insert(connection, params) -- alternatively
+=end
+     
+      sql = "INSERT INTO usage_statistics " +
+        "(shard_name, method, adapter, username, thread_id, port, host, database_name) " +
+        "VALUES (\'#{shard_name}\', \'#{method.to_s}\', \'#{config[:adapter]}\', " +
+        "\'#{config[:username]}\', \'#{Thread.current.object_id.to_s}\', " +
+        "#{config[:port] || "default"}, \'#{config[:host]}\', \'#{config[:database]}\')"
+      connection.execute(sql)
+
+      self.current_model = saved_current_model
+    end
+  end
+
+  private
+  
+  def reset_shards
+    @shard_list = HashWithIndifferentAccess.new
+
+    # The proxy can have a number of shards, each shard can have a number
+    # of master/slave replicas.  In the new refactoring, each shard is truly
+    # a shard, not a member of a replica set.  The shards are available because
+    # the Mapper module is included.
+    shards.each do |shard_name, replicas|
+      master = replicas.first # car
+      slaves = replicas[1..-1] # cdr, could be empty array
+      @shard_list[shard_name] = XBar::Shard.new(self, shard_name, master, slaves)
+    end
+
+  end
+
+  def insert_some_sql(conn, data)
+    binds   = data.map { |name, value|
+      [conn.columns('usage_statistics').find { |x| x.name == name.to_s }, value]
+    }
+    columns = binds.map(&:first).map(&:name)
+    puts "binds = #{binds.to_yaml}"
+    sql = "INSERT INTO usage_statistics (#{columns.join(", ")})
+      VALUES (#{(['?'] * columns.length).join(', ')})"
+    conn.exec_insert(sql, 'SQL', binds)
+  end
+  
+end

data/lib/xbar/rails2/association.rb

@@ -0,0 +1,133 @@
+module XBar
+  module Rails2
+    module Association
+      def association_accessor_methods(reflection, association_proxy_class)
+        define_method(reflection.name) do |*params|
+          force_reload = params.first unless params.empty?
+          reload_connection()
+          association = association_instance_get(reflection.name)
+
+          if association.nil? || force_reload
+            association = association_proxy_class.new(self, reflection)
+            retval = force_reload ? reflection.klass.uncached { association.reload } : association.reload
+            if retval.nil? and association_proxy_class == ActiveRecord::Associations::BelongsToAssociation
+              association_instance_set(reflection.name, nil)
+              return nil
+            end
+            association_instance_set(reflection.name, association)
+          end
+
+          association.target.nil? ? nil : association
+        end
+
+        define_method("loaded_#{reflection.name}?") do
+          reload_connection()
+          association = association_instance_get(reflection.name)
+          association && association.loaded?
+        end
+
+        define_method("#{reflection.name}=") do |new_value|
+          association = association_instance_get(reflection.name)
+          reload_connection()
+          if association.nil? || association.target != new_value
+            association = association_proxy_class.new(self, reflection)
+          end
+
+          if association_proxy_class == ActiveRecord::Associations::HasOneThroughAssociation
+            association.create_through_record(new_value)
+            if new_record?
+              association_instance_set(reflection.name, new_value.nil? ? nil : association)
+            else
+              self.send(reflection.name, new_value)
+            end
+          else
+            association.replace(new_value)
+            association_instance_set(reflection.name, new_value.nil? ? nil : association)
+          end
+        end
+
+        define_method("set_#{reflection.name}_target") do |target|
+          reload_connection()
+          return if target.nil? and association_proxy_class == ActiveRecord::Associations::BelongsToAssociation
+          association = association_proxy_class.new(self, reflection)
+          association.target = target
+          association_instance_set(reflection.name, association)
+        end
+      end
+
+      def collection_reader_method(reflection, association_proxy_class)
+        define_method(reflection.name) do |*params|
+          force_reload = params.first unless params.empty?
+          reload_connection()
+          association = association_instance_get(reflection.name)
+
+          unless association
+            association = association_proxy_class.new(self, reflection)
+            association_instance_set(reflection.name, association)
+          end
+
+          reflection.klass.uncached { association.reload } if force_reload
+
+          association
+        end
+
+        define_method("#{reflection.name.to_s.singularize}_ids") do
+          reload_connection()
+          if send(reflection.name).loaded? || reflection.options[:finder_sql]
+            send(reflection.name).map(&:id)
+          else
+            send(reflection.name).all(:select => "#{reflection.quoted_table_name}.#{reflection.klass.primary_key}").map(&:id)
+          end
+        end
+      end
+
+      def collection_accessor_methods(reflection, association_proxy_class, writer = true)
+        collection_reader_method(reflection, association_proxy_class)
+
+        if writer
+          define_method("#{reflection.name}=") do |new_value|
+            reload_connection()
+            # Loads proxy class instance (defined in collection_reader_method) if not already loaded
+            association = send(reflection.name)
+            association.replace(new_value)
+            association
+          end
+
+          define_method("#{reflection.name.to_s.singularize}_ids=") do |new_value|
+            reload_connection()
+            ids = (new_value || []).reject { |nid| nid.blank? }.map(&:to_i)
+            send("#{reflection.name}=", reflection.klass.find(ids).index_by(&:id).values_at(*ids))
+          end
+        end
+      end
+
+      def association_constructor_method(constructor, reflection, association_proxy_class)
+        define_method("#{constructor}_#{reflection.name}") do |*params|
+          reload_connection()
+          attributees      = params.first unless params.empty?
+          replace_existing = params[1].nil? ? true : params[1]
+          association      = association_instance_get(reflection.name)
+
+          unless association
+            association = association_proxy_class.new(self, reflection)
+            association_instance_set(reflection.name, association)
+          end
+
+          if association_proxy_class == ActiveRecord::Associations::HasOneAssociation
+            ret_val = association.send(constructor, attributees, replace_existing)
+          else
+            ret_val = association.send(constructor, attributees)
+          end
+
+          if should_set_current_shard?
+            ret_val.current_shard = self.current_shard
+          end
+
+          return ret_val
+        end
+      end
+    end
+  end
+end
+
+ActiveRecord::Base.extend(XBar::Rails2::Association)

data/lib/xbar/rails2/persistence.rb

@@ -0,0 +1,39 @@
+module XBar
+  module Rails2
+    module Persistence
+      def self.included(base)
+        base.instance_eval do
+          alias_method_chain :destroy, :xbar
+          alias_method_chain :delete, :xbar
+          alias_method_chain :reload, :xbar
+        end
+      end
+
+      def delete_with_xbar()
+        if should_set_current_shard?
+          XBar.using(self.current_shard) { delete_without_xbar() }
+        else
+          delete_without_xbar()
+        end
+      end
+
+      def destroy_with_xbar()
+        if should_set_current_shard?
+          XBar.using(self.current_shard) { destroy_without_xbar() }
+        else
+          destroy_without_xbar()
+        end
+      end
+
+      def reload_with_xbar(options = nil)
+        if should_set_current_shard?
+          XBar.using(self.current_shard) { reload_without_xbar(options) }
+        else
+          reload_without_xbar(options)
+        end
+      end
+    end
+  end
+end
+
+ActiveRecord::Base.send(:include, XBar::Rails2::Persistence)

data/lib/xbar/rails3/arel.rb

@@ -0,0 +1,13 @@
+class Arel::Visitors::ToSql
+  def quote value, column = nil
+    ActiveRecord::Base.connection.quote value, column
+  end
+
+  def quote_table_name name
+    ActiveRecord::Base.connection.quote_table_name(name)
+  end
+
+  def quote_column_name name
+    Arel::Nodes::SqlLiteral === name ? name : ActiveRecord::Base.connection.quote_column_name(name)
+  end
+end

data/lib/xbar/rails3/association.rb

@@ -0,0 +1,112 @@
+module XBar
+  module Rails3
+    module Association
+      def collection_reader_method(reflection, association_proxy_class)
+        define_method(reflection.name) do |*params|
+          force_reload = params.first unless params.empty?
+          reload_connection()
+
+          association = association_instance_get(reflection.name)
+
+          unless association
+            association  = association_proxy_class.new(self, reflection)
+            association_instance_set(reflection.name, association)
+          end
+
+          reflection.klass.uncached { association.reload } if force_reload
+
+          association
+        end
+
+        define_method("#{reflection.name.to_s.singularize}_ids") do
+          reload_connection()
+          if send(reflection.name).loaded? || reflection.options[:finder_sql]
+            send(reflection.name).map(&:id)
+          else
+            if reflection.through_reflection && reflection.source_reflection.belongs_to?
+              through = reflection.through_reflection
+              primary_key = reflection.source_reflection.primary_key_name
+              send(through.name).select("DISTINCT #{through.quoted_table_name}.#{primary_key}").map!(&:"#{primary_key}")
+            else
+              send(reflection.name).select("#{reflection.quoted_table_name}.#{reflection.klass.primary_key}").except(:includes).map!(&:id)
+            end
+          end
+        end
+      end
+
+      def association_constructor_method(constructor, reflection, association_proxy_class)
+        define_method("#{constructor}_#{reflection.name}") do |*params|
+          reload_connection()
+          attributees      = params.first unless params.empty?
+          replace_existing = params[1].nil? ? true : params[1]
+          association      = association_instance_get(reflection.name)
+
+          unless association
+            association = association_proxy_class.new(self, reflection)
+            association_instance_set(reflection.name, association)
+          end
+
+          if association_proxy_class == ActiveRecord::Associations::HasOneAssociation
+            return_val = association.send(constructor, attributees, replace_existing)
+          else
+            return_val = association.send(constructor, attributees)
+          end
+
+          if should_set_current_shard?
+            return_val.current_shard = self.current_shard
+          end
+
+          return_val
+        end
+      end
+
+      def association_accessor_methods(reflection, association_proxy_class)
+        define_method(reflection.name) do |*params|
+          reload_connection()
+          force_reload = params.first unless params.empty?
+          association = association_instance_get(reflection.name)
+
+          if association.nil? || force_reload
+            association = association_proxy_class.new(self, reflection)
+            retval = force_reload ? reflection.klass.uncached { association.reload } : association.reload
+            if retval.nil? and association_proxy_class == ActiveRecord::Associations::BelongsToAssociation
+              association_instance_set(reflection.name, nil)
+              return nil
+            end
+            association_instance_set(reflection.name, association)
+          end
+
+          association.target.nil? ? nil : association
+        end
+
+        define_method("loaded_#{reflection.name}?") do
+          reload_connection()
+          association = association_instance_get(reflection.name)
+          association && association.loaded?
+        end
+
+        define_method("#{reflection.name}=") do |new_value|
+          reload_connection()
+          association = association_instance_get(reflection.name)
+
+          if association.nil? || association.target != new_value
+            association = association_proxy_class.new(self, reflection)
+          end
+
+          association.replace(new_value)
+          association_instance_set(reflection.name, new_value.nil? ? nil : association)
+        end
+
+        define_method("set_#{reflection.name}_target") do |target|
+          return if target.nil? and association_proxy_class == ActiveRecord::Associations::BelongsToAssociation
+          reload_connection()
+          association = association_proxy_class.new(self, reflection)
+          association.target = target
+          association_instance_set(reflection.name, association)
+        end
+      end
+    end
+  end
+end
+
+ActiveRecord::Base.extend(XBar::Rails3::Association)

data/lib/xbar/rails3/persistence.rb

@@ -0,0 +1,37 @@
+module XBar
+  module Rails3
+    module Persistence
+      def update_attribute(name, value)
+        reload_connection()
+        super
+      end
+
+      def update_attributes(attributes)
+        reload_connection()
+        super
+      end
+
+      def update_attributes!(attributes)
+        reload_connection()
+        super
+      end
+
+      def reload(options = nil)
+        reload_connection()
+        super(options)
+      end
+
+      def delete
+        reload_connection()
+        super
+      end
+
+      def destroy
+        reload_connection()
+        super
+      end
+    end
+  end
+end
+
+ActiveRecord::Base.send(:include, XBar::Rails3::Persistence)

data/lib/xbar/rails3.1/singular_association.rb

@@ -0,0 +1,34 @@
+module XBar::SingularAssociation
+  def self.included(base)
+    base.instance_eval do
+      alias_method_chain :reader, :xbar
+      alias_method_chain :writer, :xbar
+      alias_method_chain :create, :xbar
+      alias_method_chain :create!, :xbar
+      alias_method_chain :build, :xbar
+    end
+  end
+
+  def reader_with_xbar(*args)
+    owner.reload_connection_safe { reader_without_xbar(*args) }
+  end
+
+  def writer_with_xbar(*args)
+    owner.reload_connection_safe { writer_without_xbar(*args) }
+  end
+
+  def create_with_xbar(*args)
+    owner.reload_connection_safe { create_without_xbar(*args) }
+  end
+
+  def create_with_xbar!(*args)
+    owner.reload_connection_safe { create_without_xbar!(*args) }
+  end
+
+  def build_with_xbar(*args)
+    owner.reload_connection_safe { build_without_xbar(*args) }
+  end
+
+end
+
+ActiveRecord::Associations::SingularAssociation.send(:include, XBar::SingularAssociation)

data/lib/xbar/scope_proxy.rb

@@ -0,0 +1,55 @@
+class XBar::ScopeProxy
+  attr_accessor :shard, :klass
+
+  def initialize(shard, klass)
+    @shard = shard
+    @klass = klass
+  end
+
+  def using(shard)
+    unless @klass.connection.shards[shard]
+      raise "Nonexistent Shard Name: #{shard}"
+    end 
+    @shard = shard
+    return self
+  end
+  
+  # Transaction Method send all queries to a specified shard.
+  def transaction(options = {}, &block)
+    @klass.connection.run_queries_on_shard(@shard) do
+      @klass = @klass.connection.transaction(options, &block)
+    end
+  end
+
+  def connection
+    @klass.connection.current_shard = @shard
+    @klass.connection
+  end
+
+  def method_missing(method, *args, &block)
+  
+    @klass.connection.current_model = @klass # XXX
+    if XBar.debug
+      # puts "Connection proxy klass proxy is #{@klass.connection.class.name}"
+      # puts "Scope proxy assigned current_model #{@klass.name}"
+      # puts "Block given is #{block_given?}"
+    end
+    @klass.connection.run_queries_on_shard(@shard, true) do
+      #puts "ScopeProxy, method missing, sending query to shard: #{@shard}, klass is #{@klass}"
+      #puts "ScropeProxy, has response: method = #{method.to_s}, #{@klass.respond_to? method}"
+      # puts Thread.current.backtrace
+      #puts "ScopeProxy, connection class = #{@klass.connection.class.name}"
+      
+      @klass = @klass.send(method, *args, &block)
+      #puts "After invocation..."
+    end
+
+    return @klass if @klass.is_a?(ActiveRecord::Base) or @klass.is_a?(Array) or @klass.is_a?(Fixnum) or @klass.nil?
+    return self
+  end
+
+  def ==(other)
+    @shard == other.shard
+    @klass == other.klass
+  end
+end