decisiv-sharded_database 0.1.1

data/README.rdoc

@@ -0,0 +1,54 @@
+= sharded_database
+
+== Overview
+
+This ActiveRecord plugin is the end result of a real world problem - First, how to aggregate table data from across multiple databases for the purpose of pagination / searching. Second, how to transparently have one production instance have read/write access to a set of aggregated records pulled from a global view.
+
+The first step was solved by creating a SQL view to pool all production records together for a given databse. This SQL view resides on a central database accessible by all instances of our application. From there, we query against this view to return a set of Aggregate* records. The second step, which this plugin handles, is proxying to the original database/record for the 'aggregate' instance and transparently interacting with the returned aggregate record as if it were the original object.
+
+
+== Uses
+
+Currently, sharded_database works by reflecting on a set of returned records, and taking the result of #determine_connection to delegate the connection to - for each instance. The path we chose had us creating a view that would return aggregate records, with an added 'source' column specifying the originating database. We then rigged our #determine_connection to return an abstract connection model depending on the value. Other ideas for implementation include an after_create callback that writes the source database / ID to a centralized table.
+
+
+== Example and Usage
+
+=== Setup
+
+Setup your Aggregate model, inheriting from a class that establishes a connection to your global datastore.
+
+  class AggregateFoo < ActiveRecord::Base
+    include ShardedDatabase::Aggregate
+
+    def determine_connection
+      Connections[source.to_sym]
+    end
+  end
+  
+The #determine_connection class above expects that there is a Connections constant that is a hash of key/value pairs of 'source' symbols and abstract models that connect to various databases. This method can be implemented in any way you desire to determine the connection source.
+
+
+=== Loading Aggregate Records
+
+Assuming that AggregateFoo is a model that is bound to your aggregate view/table, you can now #find against this model as you would any other ActiveRecord class.
+
+  AggregateFoo.all   # => [#<Foo(Connection::One) ..>, #<Foo(Connection::Two) ..>, #<Foo(Connection::One) ..>]
+  
+Updating an attribute on the first record in the above array would update the corresponding record located in the database that Connection::One connects to.
+
+
+=== Associations
+
+Associations are also taken into account, and any associations that are defined on a Foo model will be sourced correctly. I have only tested this one level deep, though.
+
+
+=== Accessing 'raw' AggregateFoo objects
+
+Easily done. Simply add a :raw option set to true to your finder queries.
+
+  AggreateFoo.all(:raw => true)
+
+
+
+Copyright &copy; 2008, Brennan Dunn, Decisiv Inc. Released under the MIT license.

data/Rakefile

@@ -0,0 +1,22 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+
+desc 'Default: run unit tests.'
+task :default => :test
+
+desc 'Test the sharded_database plugin.'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+
+desc 'Generate documentation for the sharded_database plugin.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'ShardedDatabase'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/init.rb

@@ -0,0 +1 @@
+require 'sharded_database'

data/lib/sharded_database/aggregate.rb

@@ -0,0 +1,61 @@
+module ShardedDatabase
+  class NoConnectionError < StandardError ; end
+    
+  module Aggregate
+
+    def self.included(klass)
+      klass.extend         ClassMethods
+      klass.send :include, InstanceMethods
+      klass.class_eval do
+        cattr_accessor :connection_field, :source_class
+        @connection_field = :oem
+
+        class << self
+          alias_method_chain :find, :raw
+        end
+      end
+    end
+
+    module ClassMethods
+
+      def find_with_raw(*args)
+        @raw = args.last.is_a?(Hash) && args.last.delete(:raw)
+        @raw ? temporarily_remove(:after_find) { find_without_raw(*args) } : find_without_raw(*args)
+      end
+
+    end
+
+    module InstanceMethods
+      
+      def determine_connection
+        # stub method - implement your own!
+      end
+      
+      def after_find
+        @klass      = determine_connection || raise(ShardedDatabase::NoConnectionError, 'Cannot determine connection class')
+        @connection = @klass.respond_to?(:connection) ? @klass.connection : raise(ShardedDatabase::NoConnectionError, 'Connection class does not respond to :connection')
+        @foreign_id = foreign_id
+
+        metaclass.delegate :connection, :to => @klass
+
+        class << self
+          alias_method :proxy_class, :class
+
+          include AggregateProxy
+          instance_methods.each do |m|
+            undef_method(m) unless m =~ /^__|proxy_|inspect|foreign_id/
+          end
+        end
+        
+        self.class.reflect_on_all_associations.each do |a| 
+          metaclass.send :alias_method, "proxy_#{a.name}".to_sym, a.name.to_sym
+          metaclass.send :undef_method, a.name
+        end
+        
+      end
+      
+    end
+
+  end
+
+end

data/lib/sharded_database/aggregate_proxy.rb

@@ -0,0 +1,51 @@
+module ShardedDatabase
+  module AggregateProxy
+    
+    def ===(other)
+      other === load_target
+    end
+
+    def inspect
+      load_target.inspect.gsub(/#<([\w\:]+)\s(.*?)>/) { "#<#{$1}(#{@klass.name}) #{$2}>" }
+    end
+
+
+    private
+
+    def load_target
+      @target ||= begin
+                    klass = (self.proxy_class.source_class || self.proxy_class.name.gsub('Aggregate','')).constantize
+                    borrow_connection(klass, @klass) { |k| k.find(@foreign_id) }
+                  end
+    end
+
+    def method_missing(method, *args, &block)
+      if association_method?(method)
+        apply_connection_to_association(method)
+      end
+      load_target.respond_to?(method) ? load_target.send(method, *args, &block) : super
+    end
+
+    def association_method?(method)
+      load_target.class.reflect_on_all_associations.any? { |a| a.name == method.to_sym }
+    end
+
+    def borrow_connection(requesting_class, target_class, &block)
+      eigen = requesting_class.metaclass
+      eigen.delegate :connection, :to => target_class
+      yield(requesting_class)
+    end
+    
+    def apply_connection_to_association(method)
+      load_target.metaclass.send(:attr_accessor, :source_class)
+      load_target.source_class = @klass
+      load_target.class_eval %{
+        def #{method}(*args)
+          proxy_#{method}.proxy_reflection.klass.metaclass.delegate :connection, :to => self.source_class
+          proxy_#{method}
+        end
+      }
+    end
+
+  end
+end

data/lib/sharded_database/core_extensions.rb

@@ -0,0 +1,11 @@
+class Object
+  
+  def temporarily_remove(method, &block)
+    alias_method "original_#{method}", method
+    undef_method(method)
+    return yield
+  ensure
+    alias_method method, "original_#{method}"
+  end
+  
+end

data/lib/sharded_database.rb

@@ -0,0 +1,9 @@
+require 'sharded_database/aggregate_proxy'
+require 'sharded_database/aggregate'
+require 'sharded_database/core_extensions'
+
+module ShardedDatabase
+  
+  VERSION = '0.1'
+  
+end

data/test/helper.rb

@@ -0,0 +1,26 @@
+require 'lib/boot'
+require 'lib/test_case'
+require 'sharded_database'
+require 'lib/models'
+
+module ShardedDatabase
+  class TestCase < Test::Unit::TestCase
+    
+    self.new_backtrace_silencer(:shoulda) { |line| line.include? 'lib/shoulda' }
+    self.new_backtrace_silencer(:mocha) { |line| line.include? 'lib/mocha' }
+    self.backtrace_silencers << :shoulda << :mocha
+    
+
+    def assert_connection(configuration, *objects)
+      expected_db = ::ActiveRecord::Base.configurations[configuration.to_s]['database']
+      
+      objects.each do |object|
+        object_db   = object.respond_to?(:connection) ? object.connection.current_database : nil
+        msg = "Expected #{object.inspect} to be connected to :#{expected_db}, but was :#{object_db}"
+        assert_equal expected_db, object_db, msg
+      end
+    end
+    
+  end
+end
+

data/test/lib/boot.rb

@@ -0,0 +1,20 @@
+require 'rubygems'
+require 'yaml'
+require 'test/unit'
+require 'active_record'
+require 'active_support'
+require 'fileutils'
+require 'shoulda'
+require 'quietbacktrace'
+
+ActiveRecord::Base.logger = Logger.new(File.dirname(__FILE__)+'/../debug.log')
+ActiveRecord::Base.configurations = $config = YAML::load(IO.read(File.dirname(__FILE__) + '/database.yml'))
+
+class ActiveRecord::ConnectionAdapters::MysqlAdapter < ActiveRecord::ConnectionAdapters::AbstractAdapter
+  def log_info(sql, name, runtime)
+    if @logger && @logger.debug?
+      name = "#{name.nil? ? "SQL" : name} (DB:#{@config[:database]}) (#{sprintf("%f", runtime)})"
+      @logger.debug format_log_entry(name, sql.squeeze(' '))
+    end
+  end
+end

data/test/lib/database.yml

@@ -0,0 +1,20 @@
+master:
+  adapter: mysql
+  database: database_master
+  username: root
+  password: 
+  host: localhost
+
+one_db:
+  adapter: mysql
+  database: database_mck
+  username: root
+  password: 
+  host: localhost
+  
+two_db:
+  adapter: mysql
+  database: database_vlv
+  username: root
+  password: 
+  host: localhost

data/test/lib/models.rb

@@ -0,0 +1,35 @@
+module Connection ; end
+
+class Connection::One < ActiveRecord::Base
+  establish_connection :one_db
+  self.abstract_class = true
+end
+
+class Connection::Two < ActiveRecord::Base
+  establish_connection :two_db
+  self.abstract_class = true
+end
+
+Connections = { :one => Connection::One, :two => Connection::Two }
+
+class GlobalConnection < ActiveRecord::Base
+  establish_connection :master
+  self.abstract_class = true
+end
+
+class AggregateEstimate < GlobalConnection
+  include ShardedDatabase::Aggregate
+  
+  def determine_connection
+    Connections[source.to_sym]
+  end
+  
+end
+
+class Estimate < ActiveRecord::Base  
+  has_many :items
+end
+
+class Item < ActiveRecord::Base
+  belongs_to :estimate
+end

data/test/lib/test_case.rb

@@ -0,0 +1,73 @@
+module ShardedDatabase
+  class TestCase < Test::Unit::TestCase
+    
+    def setup_environment(options={})
+      setup_database
+      setup_models
+    end
+    
+    def breakdown_environment
+
+    end
+    
+    def test_truth ; end
+    
+    
+    private
+    
+    def setup_database
+      create_db_file = lambda do |file|
+        File.delete(file) if File.exist?(file)
+        SQLite3::Database.new(file)
+      end
+      
+      # setup aggregate table
+      ::ActiveRecord::Base.establish_connection :master
+      ::ActiveRecord::Base.class_eval do
+        silence do
+          connection.create_table :aggregate_estimates, :force => true do |t|
+            t.string    :source
+            t.integer   :foreign_id
+            t.timestamp :created_at
+          end
+        end
+      end
+      
+      # setup NUM_db.estimates
+      %w(one two).each do |num|
+        ::ActiveRecord::Base.establish_connection "#{num}_db".to_sym
+        ::ActiveRecord::Base.class_eval do
+          silence do
+            connection.create_table :estimates, :force => true do |t|
+              t.string      :name
+            end
+            
+            connection.create_table :items, :force => true do |t|
+              t.belongs_to  :estimate
+              t.string      :name
+            end
+          end
+        end
+      end
+      
+    end
+    
+    def setup_models
+      one_estimate = Class.new(Connection::One) { set_table_name 'estimates' ; has_many(:items) }
+      @one_1 = one_estimate.create :name => 'One Estimate'
+      
+      two_estimate = Class.new(Connection::Two) { set_table_name 'estimates' ; has_many(:items) }
+      @two_1 = two_estimate.create :name => 'One Estimate 1'
+      @two_2 = two_estimate.create :name => 'Two Estimate 2'
+      
+      one_item = Class.new(Connection::One) { set_table_name 'items' ; belongs_to(:estimate) }
+      one_item.create :name => 'One Test Item', :estimate_id => @one_1.id
+      
+      AggregateEstimate.create :source => 'one', :foreign_id => @one_1.id
+      AggregateEstimate.create :source => 'two', :foreign_id => @two_1.id
+      AggregateEstimate.create :source => 'two', :foreign_id => @two_2.id
+    end
+    
+  end
+end
+

data/test/sharded_database/association_test.rb

@@ -0,0 +1,24 @@
+require File.dirname(__FILE__) + '/../helper'
+
+class AssociationTest < ShardedDatabase::TestCase
+  def setup ; setup_environment ; end
+  
+  
+  context 'Connection delegation on has_many associations' do
+    
+    setup do
+      @parent = AggregateEstimate.find_by_source('one')
+    end
+    
+    should 'fetch items from the parent instance connection' do
+      assert ! @parent.items.empty?
+      assert_connection :one_db, @parent.items.first
+    end
+    
+    should 'keep its connection when bubbling up to an associations parent' do
+      assert_equal @parent, @parent.items.first.estimate
+    end
+    
+  end
+  
+end

data/test/sharded_database/connection_test.rb

@@ -0,0 +1,22 @@
+require File.dirname(__FILE__) + '/../helper'
+
+class ConnectionTest < ShardedDatabase::TestCase
+  def setup ; setup_environment ; end
+  
+  
+  should 'foundationally support instances having a different connection than their parent class' do
+    assert_not_equal AggregateEstimate.first.connection, AggregateEstimate.connection
+  end
+  
+  should 'have instances of the same source share the same connection' do
+    first, second = AggregateEstimate.find_all_by_source('two', :limit => 2)
+    
+    assert_connection :two_db, first, second
+    assert_equal first.connection.object_id, second.connection.object_id  # ensure that delegation is working correctly
+  end
+  
+  should 'display instance connection when inspecting' do
+    assert_match %{(Connection::One)}, AggregateEstimate.find_by_source('one').inspect
+  end
+    
+end

data/test/sharded_database/instance_test.rb

@@ -0,0 +1,35 @@
+require File.dirname(__FILE__) + '/../helper'
+
+class InstanceTest < ShardedDatabase::TestCase
+  def setup ; setup_environment ; end
+  
+  
+  context 'Loading raw aggregate objects' do
+    
+    setup do
+      @aggregates = AggregateEstimate.all(:raw => true)
+    end
+    
+    should 'all be AggregateEstimate instances' do
+      assert @aggregates.all? { |a| a.is_a?(AggregateEstimate) }
+    end
+    
+  end
+  
+  context "A transformed aggregate instance" do
+    
+    setup do
+      @estimate = AggregateEstimate.first
+    end
+
+    should 'channel calls to #class to the proxy class' do
+      assert @estimate.is_a?(Estimate)
+    end
+    
+    should 'have the same attribute fields as the proxy class' do
+      assert_same_elements @estimate.attributes.keys, Estimate.column_names
+    end
+    
+  end
+  
+end

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification 
+name: decisiv-sharded_database
+version: !ruby/object:Gem::Version 
+  version: 0.1.1
+platform: ruby
+authors: 
+- Brennan Dunn
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-01-11 00:00:00 -08:00
+default_executable: 
+dependencies: []
+
+description: Allows for connection handling at the instance level.
+email: me@brennandunn.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- Rakefile
+- README.rdoc
+- init.rb
+- lib/sharded_database.rb
+- lib/sharded_database/aggregate.rb
+- lib/sharded_database/aggregate_proxy.rb
+- lib/sharded_database/core_extensions.rb
+has_rdoc: true
+homepage: http://github.com/brennandunn/sharded_database/
+post_install_message: 
+rdoc_options: 
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Allows for connection handling at the instance level.
+test_files: 
+- test/helper.rb
+- test/sharded_database/association_test.rb
+- test/sharded_database/connection_test.rb
+- test/sharded_database/instance_test.rb
+- test/lib/boot.rb
+- test/lib/database.yml
+- test/lib/models.rb
+- test/lib/test_case.rb