data_fabric 1.1.0 → 1.2.0

data/CHANGELOG

@@ -1,5 +1,9 @@
 DataFabric changelog
 
+v1.2.0 - 2008-12-25
+
+- Now supports ActiveRecord 2.2.
+
 v1.1.0 - 2008-11-22
 
 - Cache connections so we don't have to reconnect constantly. (Justin Balthrop, Geni)

data/Manifest

@@ -19,9 +19,15 @@ example/config/initializers/inflections.rb
 example/config/initializers/mime_types.rb
 example/config/initializers/new_rails_defaults.rb
 example/config/routes.rb
+example/db/development.sqlite3
 example/db/migrate/20080702154628_create_accounts.rb
 example/db/migrate/20080702154820_create_figments.rb
+example/db/s0_development.sqlite3
+example/db/s0_test.sqlite3
+example/db/s1_development.sqlite3
+example/db/s1_test.sqlite3
 example/db/schema.rb
+example/db/test.sqlite3
 example/public/404.html
 example/public/422.html
 example/public/500.html
@@ -55,7 +61,70 @@ example/test/fixtures/accounts.yml
 example/test/functional/accounts_controller_test.rb
 example/test/integration/account_figments_test.rb
 example/test/test_helper.rb
+example/vendor/plugins/data_fabric/init.rb
+example/vendor/plugins/data_fabric/lib/data_fabric/ar20.rb
+example/vendor/plugins/data_fabric/lib/data_fabric/ar22.rb
+example/vendor/plugins/data_fabric/lib/data_fabric/version.rb
+example/vendor/plugins/data_fabric/lib/data_fabric.rb
+example22/app/controllers/accounts_controller.rb
+example22/app/controllers/application.rb
+example22/app/controllers/figments_controller.rb
+example22/app/helpers/application_helper.rb
+example22/app/models/account.rb
+example22/app/models/figment.rb
+example22/app/views/accounts/index.html.erb
+example22/app/views/layouts/application.html.erb
+example22/config/boot.rb
+example22/config/database.yml
+example22/config/environment.rb
+example22/config/environments/development.rb
+example22/config/environments/production.rb
+example22/config/environments/test.rb
+example22/config/initializers/inflections.rb
+example22/config/initializers/mime_types.rb
+example22/config/initializers/new_rails_defaults.rb
+example22/config/locales/en.yml
+example22/config/routes.rb
+example22/db/migrate/20080702154628_create_accounts.rb
+example22/db/migrate/20080702154820_create_figments.rb
+example22/public/404.html
+example22/public/422.html
+example22/public/500.html
+example22/public/dispatch.cgi
+example22/public/dispatch.fcgi
+example22/public/dispatch.rb
+example22/public/favicon.ico
+example22/public/images/rails.png
+example22/public/index.html
+example22/public/javascripts/application.js
+example22/public/javascripts/controls.js
+example22/public/javascripts/dragdrop.js
+example22/public/javascripts/effects.js
+example22/public/javascripts/prototype.js
+example22/public/robots.txt
+example22/Rakefile
+example22/script/about
+example22/script/console
+example22/script/dbconsole
+example22/script/destroy
+example22/script/generate
+example22/script/performance/benchmarker
+example22/script/performance/profiler
+example22/script/performance/request
+example22/script/plugin
+example22/script/process/inspector
+example22/script/process/reaper
+example22/script/process/spawner
+example22/script/runner
+example22/script/server
+example22/test/fixtures/accounts.yml
+example22/test/functional/accounts_controller_test.rb
+example22/test/integration/account_figments_test.rb
+example22/test/performance/browsing_test.rb
+example22/test/test_helper.rb
 init.rb
+lib/data_fabric/ar20.rb
+lib/data_fabric/ar22.rb
 lib/data_fabric/version.rb
 lib/data_fabric.rb
 Manifest
@@ -68,4 +137,8 @@ test/database_test.rb
 test/shard_test.rb
 test/test_helper.rb
 test/thread_test.rb
+test/vr_austin_master.db
+test/vr_austin_slave.db
+test/vr_dallas_master.db
+test/vr_dallas_slave.db
 TESTING.rdoc

data/README.rdoc

@@ -62,7 +62,7 @@ ActionController around_filter based on the user as follows:
 == Warnings
 
 * Sharded models should never be placed in the session store or you will get "Shard not set" errors when the session is persisted.
-* ActiveRecord's allow_concurrency = true is NOT supported in this version of data_fabric.
+* DataFabric does not support running with ActiveRecord's allow_concurrency = true in AR 2.0 and 2.1.  allow_concurrency is gone in AR 2.2.
 
 == Testing and Bug Reports
 

data/Rakefile

@@ -1,8 +1,6 @@
 require 'rubygems'
 require 'echoe'
 
-#gem 'rails', '=2.0.2'
-
 require File.dirname(__FILE__) << "/lib/data_fabric/version"
 
 Echoe.new 'data_fabric' do |p|
@@ -10,16 +8,23 @@ Echoe.new 'data_fabric' do |p|
   p.author = "Mike Perham"
   p.email  = 'mperham@gmail.com'
   p.project = 'fiveruns'
-  p.summary = 'Sharding and replication support for ActiveRecord 2.0 and 2.1'
+  p.summary = 'Sharding and replication support for ActiveRecord 2.x'
   p.url = "http://github.com/fiveruns/data_fabric"
-#  p.dependencies = ['activerecord >=2.0.2']
   p.development_dependencies = []
   p.rubygems_version = nil
   p.include_rakefile = true
+  p.test_pattern = 'test/*_test.rb'
 end
 
 task :test => [:pretest]
 
+desc "Test all versions of ActiveRecord installed locally"
+task :test_all do
+  Gem.source_index.search(Gem::Dependency.new('activerecord', '>=2.0')).each do |spec|
+    puts `rake test AR_VERSION=#{spec.version}`
+  end
+end
+
 task :pretest do
   setup(false)
 end
@@ -28,10 +33,6 @@ task :create_db do
   setup(true)
 end
 
-task :changelog do
-  `git log | grep -v git-svn-id > History.txt`
-end
-
 def load_database_yml
   filename = "test/database.yml"
   if !File.exist?(filename)
@@ -87,3 +88,28 @@ def create_mysql(create, db_name)
   execute "create table the_whole_burritos (id integer not null auto_increment, name varchar(30) not null, primary key(id))"
   execute "insert into the_whole_burritos (id, name) values (1, '#{db_name}')"
 end
+
+# Test coverage
+gem 'spicycode-rcov' rescue nil
+begin
+  require 'rcov/rcovtask'
+
+  desc "Generate coverage numbers for all locally installed versions of ActiveRecord"
+  task :cover_all do
+    Gem.source_index.search(Gem::Dependency.new('activerecord', '>=2.0')).each do |spec|
+      puts `rake cover AR_VERSION=#{spec.version}`
+    end
+  end
+
+  task :cover => [:pretest, :rcov_impl]
+
+  Rcov::RcovTask.new('rcov_impl') do |t|
+    t.libs << "test"
+    t.test_files = FileList["test/*_test.rb"]
+    t.output_dir = "coverage/#{ENV['AR_VERSION']}"
+    t.verbose = true
+    t.rcov_opts = ['--text-report', '--exclude', "test,Library,#{ENV['GEM_HOME']}", '--sort', 'coverage']
+  end
+rescue LoadError => e
+  puts 'Test coverage support requires \'gem install spicycode-rcov\''
+end

data/data_fabric.gemspec

@@ -2,23 +2,23 @@
 
 Gem::Specification.new do |s|
   s.name = %q{data_fabric}
-  s.version = "1.1.0"
+  s.version = "1.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Mike Perham"]
-  s.date = %q{2008-11-22}
-  s.description = %q{Sharding and replication support for ActiveRecord 2.0 and 2.1}
+  s.date = %q{2008-12-01}
+  s.description = %q{Sharding and replication support for ActiveRecord 2.x}
   s.email = %q{mperham@gmail.com}
-  s.extra_rdoc_files = ["CHANGELOG", "lib/data_fabric/version.rb", "lib/data_fabric.rb", "README.rdoc"]
-  s.files = ["CHANGELOG", "example/app/controllers/accounts_controller.rb", "example/app/controllers/application.rb", "example/app/controllers/figments_controller.rb", "example/app/helpers/accounts_helper.rb", "example/app/helpers/application_helper.rb", "example/app/helpers/figments_helper.rb", "example/app/models/account.rb", "example/app/models/figment.rb", "example/app/views/accounts/index.html.erb", "example/app/views/layouts/application.html.erb", "example/config/boot.rb", "example/config/database.yml", "example/config/environment.rb", "example/config/environments/development.rb", "example/config/environments/production.rb", "example/config/environments/test.rb", "example/config/initializers/inflections.rb", "example/config/initializers/mime_types.rb", "example/config/initializers/new_rails_defaults.rb", "example/config/routes.rb", "example/db/migrate/20080702154628_create_accounts.rb", "example/db/migrate/20080702154820_create_figments.rb", "example/db/schema.rb", "example/public/404.html", "example/public/422.html", "example/public/500.html", "example/public/dispatch.cgi", "example/public/dispatch.fcgi", "example/public/dispatch.rb", "example/public/favicon.ico", "example/public/images/rails.png", "example/public/javascripts/application.js", "example/public/javascripts/controls.js", "example/public/javascripts/dragdrop.js", "example/public/javascripts/effects.js", "example/public/javascripts/prototype.js", "example/public/robots.txt", "example/Rakefile", "example/script/about", "example/script/console", "example/script/dbconsole", "example/script/destroy", "example/script/generate", "example/script/performance/benchmarker", "example/script/performance/profiler", "example/script/performance/request", "example/script/plugin", "example/script/process/inspector", "example/script/process/reaper", "example/script/process/spawner", "example/script/runner", "example/script/server", "example/test/fixtures/accounts.yml", "example/test/functional/accounts_controller_test.rb", "example/test/integration/account_figments_test.rb", "example/test/test_helper.rb", "init.rb", "lib/data_fabric/version.rb", "lib/data_fabric.rb", "Manifest", "Rakefile", "README.rdoc", "test/connection_test.rb", "test/database.yml", "test/database.yml.mysql", "test/database_test.rb", "test/shard_test.rb", "test/test_helper.rb", "test/thread_test.rb", "TESTING.rdoc", "data_fabric.gemspec"]
+  s.extra_rdoc_files = ["CHANGELOG", "lib/data_fabric/ar20.rb", "lib/data_fabric/ar22.rb", "lib/data_fabric/version.rb", "lib/data_fabric.rb", "README.rdoc"]
+  s.files = ["CHANGELOG", "example/app/controllers/accounts_controller.rb", "example/app/controllers/application.rb", "example/app/controllers/figments_controller.rb", "example/app/helpers/accounts_helper.rb", "example/app/helpers/application_helper.rb", "example/app/helpers/figments_helper.rb", "example/app/models/account.rb", "example/app/models/figment.rb", "example/app/views/accounts/index.html.erb", "example/app/views/layouts/application.html.erb", "example/config/boot.rb", "example/config/database.yml", "example/config/environment.rb", "example/config/environments/development.rb", "example/config/environments/production.rb", "example/config/environments/test.rb", "example/config/initializers/inflections.rb", "example/config/initializers/mime_types.rb", "example/config/initializers/new_rails_defaults.rb", "example/config/routes.rb", "example/db/development.sqlite3", "example/db/migrate/20080702154628_create_accounts.rb", "example/db/migrate/20080702154820_create_figments.rb", "example/db/s0_development.sqlite3", "example/db/s0_test.sqlite3", "example/db/s1_development.sqlite3", "example/db/s1_test.sqlite3", "example/db/schema.rb", "example/db/test.sqlite3", "example/public/404.html", "example/public/422.html", "example/public/500.html", "example/public/dispatch.cgi", "example/public/dispatch.fcgi", "example/public/dispatch.rb", "example/public/favicon.ico", "example/public/images/rails.png", "example/public/javascripts/application.js", "example/public/javascripts/controls.js", "example/public/javascripts/dragdrop.js", "example/public/javascripts/effects.js", "example/public/javascripts/prototype.js", "example/public/robots.txt", "example/Rakefile", "example/script/about", "example/script/console", "example/script/dbconsole", "example/script/destroy", "example/script/generate", "example/script/performance/benchmarker", "example/script/performance/profiler", "example/script/performance/request", "example/script/plugin", "example/script/process/inspector", "example/script/process/reaper", "example/script/process/spawner", "example/script/runner", "example/script/server", "example/test/fixtures/accounts.yml", "example/test/functional/accounts_controller_test.rb", "example/test/integration/account_figments_test.rb", "example/test/test_helper.rb", "example/vendor/plugins/data_fabric/init.rb", "example/vendor/plugins/data_fabric/lib/data_fabric/ar20.rb", "example/vendor/plugins/data_fabric/lib/data_fabric/ar22.rb", "example/vendor/plugins/data_fabric/lib/data_fabric/version.rb", "example/vendor/plugins/data_fabric/lib/data_fabric.rb", "example22/app/controllers/accounts_controller.rb", "example22/app/controllers/application.rb", "example22/app/controllers/figments_controller.rb", "example22/app/helpers/application_helper.rb", "example22/app/models/account.rb", "example22/app/models/figment.rb", "example22/app/views/accounts/index.html.erb", "example22/app/views/layouts/application.html.erb", "example22/config/boot.rb", "example22/config/database.yml", "example22/config/environment.rb", "example22/config/environments/development.rb", "example22/config/environments/production.rb", "example22/config/environments/test.rb", "example22/config/initializers/inflections.rb", "example22/config/initializers/mime_types.rb", "example22/config/initializers/new_rails_defaults.rb", "example22/config/locales/en.yml", "example22/config/routes.rb", "example22/db/migrate/20080702154628_create_accounts.rb", "example22/db/migrate/20080702154820_create_figments.rb", "example22/public/404.html", "example22/public/422.html", "example22/public/500.html", "example22/public/dispatch.cgi", "example22/public/dispatch.fcgi", "example22/public/dispatch.rb", "example22/public/favicon.ico", "example22/public/images/rails.png", "example22/public/index.html", "example22/public/javascripts/application.js", "example22/public/javascripts/controls.js", "example22/public/javascripts/dragdrop.js", "example22/public/javascripts/effects.js", "example22/public/javascripts/prototype.js", "example22/public/robots.txt", "example22/Rakefile", "example22/script/about", "example22/script/console", "example22/script/dbconsole", "example22/script/destroy", "example22/script/generate", "example22/script/performance/benchmarker", "example22/script/performance/profiler", "example22/script/performance/request", "example22/script/plugin", "example22/script/process/inspector", "example22/script/process/reaper", "example22/script/process/spawner", "example22/script/runner", "example22/script/server", "example22/test/fixtures/accounts.yml", "example22/test/functional/accounts_controller_test.rb", "example22/test/integration/account_figments_test.rb", "example22/test/performance/browsing_test.rb", "example22/test/test_helper.rb", "init.rb", "lib/data_fabric/ar20.rb", "lib/data_fabric/ar22.rb", "lib/data_fabric/version.rb", "lib/data_fabric.rb", "Manifest", "Rakefile", "README.rdoc", "test/connection_test.rb", "test/database.yml", "test/database.yml.mysql", "test/database_test.rb", "test/shard_test.rb", "test/test_helper.rb", "test/thread_test.rb", "test/vr_austin_master.db", "test/vr_austin_slave.db", "test/vr_dallas_master.db", "test/vr_dallas_slave.db", "TESTING.rdoc", "data_fabric.gemspec"]
   s.has_rdoc = true
   s.homepage = %q{http://github.com/fiveruns/data_fabric}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Data_fabric", "--main", "README.rdoc"]
   s.require_paths = ["lib"]
   s.rubyforge_project = %q{fiveruns}
   s.rubygems_version = %q{1.3.1}
-  s.summary = %q{Sharding and replication support for ActiveRecord 2.0 and 2.1}
-  s.test_files = ["test/connection_test.rb", "test/database_test.rb", "test/shard_test.rb", "test/test_helper.rb", "test/thread_test.rb"]
+  s.summary = %q{Sharding and replication support for ActiveRecord 2.x}
+  s.test_files = ["test/connection_test.rb", "test/database_test.rb", "test/shard_test.rb", "test/thread_test.rb"]
 
   if s.respond_to? :specification_version then
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION

data/example/app/models/figment.rb

@@ -1,4 +1,4 @@
 class Figment < ActiveRecord::Base
-	connection_topology :shard_by => 'shard', :replicated => false
+	data_fabric :shard_by => 'shard', :replicated => false
 	belongs_to :account
 end

data/example/vendor/plugins/data_fabric/init.rb

@@ -0,0 +1 @@
+DataFabric.init

data/example/vendor/plugins/data_fabric/lib/data_fabric.rb

@@ -0,0 +1,106 @@
+require 'active_record'
+require 'active_record/version'
+require 'data_fabric/version'
+
+# DataFabric adds a new level of flexibility to ActiveRecord connection handling.
+# You need to describe the topology for your database infrastructure in your model(s).  As with ActiveRecord normally, different models can use different topologies.
+# 
+# class MyHugeVolumeOfDataModel < ActiveRecord::Base
+#   data_fabric :replicated => true, :shard_by => :city
+# end
+# 
+# There are four supported modes of operation, depending on the options given to the data_fabric method.  The plugin will look for connections in your config/database.yml with the following convention:
+# 
+# No connection topology:
+# #{environment} - this is the default, as with ActiveRecord, e.g. "production"
+# 
+# data_fabric :replicated => true
+# #{environment}_#{role} - no sharding, just replication, where role is "master" or "slave", e.g. "production_master"
+# 
+# data_fabric :shard_by => :city
+# #{group}_#{shard}_#{environment} - sharding, no replication, e.g. "city_austin_production"
+# 
+# data_fabric :replicated => true, :shard_by => :city
+# #{group}_#{shard}_#{environment}_#{role} - sharding with replication, e.g. "city_austin_production_master"
+# 
+# 
+# When marked as replicated, all write and transactional operations for the model go to the master, whereas read operations go to the slave.
+# 
+# Since sharding is an application-level concern, your application must set the shard to use based on the current request or environment.  The current shard for a group is set on a thread local variable.  For example, you can set the shard in an ActionController around_filter based on the user as follows:
+# 
+# class ApplicationController < ActionController::Base
+#   around_filter :select_shard
+#   
+#   private
+#   def select_shard(&action_block)
+#     DataFabric.activate_shard(:city => @current_user.city, &action_block)
+#   end
+# end
+module DataFabric
+
+  # Set this logger to log DataFabric operations.
+  # The logger should quack like a standard Ruby Logger.
+  mattr_accessor :logger
+
+  def self.init
+    logger = ActiveRecord::Base.logger unless logger
+    log { "Loading data_fabric #{DataFabric::Version::STRING} with ActiveRecord #{ActiveRecord::VERSION::STRING}" }
+
+    if ActiveRecord::VERSION::STRING < '2.2.0'
+      require 'data_fabric/ar20'
+    else
+      require 'data_fabric/ar22'
+    end
+    ActiveRecord::Base.send(:include, DataFabric::Extensions)
+  end
+  
+  def self.activate_shard(shards, &block)
+    ensure_setup
+
+    # Save the old shard settings to handle nested activation
+    old = Thread.current[:shards].dup
+
+    shards.each_pair do |key, value|
+      Thread.current[:shards][key.to_s] = value.to_s
+    end
+    if block_given?
+      begin
+        yield
+      ensure
+        Thread.current[:shards] = old
+      end
+    end
+  end
+  
+  # For cases where you can't pass a block to activate_shards, you can
+  # clean up the thread local settings by calling this method at the
+  # end of processing
+  def self.deactivate_shard(shards)
+    ensure_setup
+    shards.each do |key, value|
+      Thread.current[:shards].delete(key.to_s)
+    end
+  end
+  
+  def self.active_shard(group)
+    raise ArgumentError, 'No shard has been activated' unless Thread.current[:shards]
+
+    returning(Thread.current[:shards][group.to_s]) do |shard|
+      raise ArgumentError, "No active shard for #{group}" unless shard
+    end
+  end
+  
+  def self.shard_active_for?(group)
+    return true unless group
+    Thread.current[:shards] and Thread.current[:shards][group.to_s]
+  end
+
+  def self.ensure_setup
+    Thread.current[:shards] = {} unless Thread.current[:shards]
+  end
+
+  def self.log(level=Logger::INFO, &block)
+    logger && logger.add(level, &block)
+  end
+
+end

data/example/vendor/plugins/data_fabric/lib/data_fabric/ar20.rb

@@ -0,0 +1,135 @@
+module DataFabric
+  module Extensions
+    def self.included(model)
+      # Wire up ActiveRecord::Base
+      model.extend ClassMethods
+    end
+
+    # Class methods injected into ActiveRecord::Base
+    module ClassMethods
+      def data_fabric(options)
+        proxy = DataFabric::ConnectionProxy.new(self, options)
+        ActiveRecord::Base.active_connections[name] = proxy
+
+        raise ArgumentError, "data_fabric does not support ActiveRecord's allow_concurrency = true" if allow_concurrency
+        DataFabric.log { "Creating data_fabric proxy for class #{name}" }
+      end
+    end
+  end
+
+  class ConnectionProxy
+    def initialize(model_class, options)
+      @model_class = model_class      
+      @replicated  = options[:replicated]
+      @shard_group = options[:shard_by]
+      @prefix      = options[:prefix]
+      @role        = 'slave' if @replicated
+
+      @model_class.send :include, ActiveRecordConnectionMethods if @replicated
+    end
+
+    delegate :insert, :update, :delete, :create_table, :rename_table, :drop_table, :add_column, :remove_column, 
+      :change_column, :change_column_default, :rename_column, :add_index, :remove_index, :initialize_schema_information,
+      :dump_schema_information, :execute, :execute_ignore_duplicate, :to => :master
+
+    def cache(&block)
+      connection.cache(&block)
+    end
+
+    def transaction(start_db_transaction = true, &block)
+      with_master { connection.transaction(start_db_transaction, &block) }
+    end
+
+    def method_missing(method, *args, &block)
+      DataFabric.log(Logger::DEBUG) { "Calling #{method} on #{connection}" }
+      connection.send(method, *args, &block)
+    end
+
+    def connection_name
+      connection_name_builder.join('_')
+    end
+
+    def disconnect!
+      if connected?
+        connection.disconnect! 
+        cached_connections[connection_name] = nil
+      end
+    end
+
+    def verify!(arg)
+      connection.verify!(arg) if connected?
+    end
+
+    def with_master
+      # Allow nesting of with_master.
+      old_role = @role
+      set_role('master')
+      yield
+    ensure
+      set_role(old_role)
+    end
+
+  private
+
+    def cached_connections
+      @cached_connections ||= {}
+    end
+
+    def connection_name_builder
+      @connection_name_builder ||= begin
+        clauses = []
+        clauses << @prefix if @prefix
+        clauses << @shard_group if @shard_group
+        clauses << StringProxy.new { DataFabric.active_shard(@shard_group) } if @shard_group
+        clauses << RAILS_ENV
+        clauses << StringProxy.new { @role } if @replicated
+        clauses
+      end
+    end
+
+    def connection
+      name = connection_name
+      if not connected?
+        config = ActiveRecord::Base.configurations[name]
+        raise ArgumentError, "Unknown database config: #{name}, have #{ActiveRecord::Base.configurations.inspect}" unless config
+        DataFabric.log { "Connecting to #{name}" }
+        @model_class.establish_connection(config)
+        cached_connections[name] = @model_class.connection
+        @model_class.active_connections[@model_class.name] = self
+      end
+      cached_connections[name].verify!(3600)
+      cached_connections[name]
+    end
+
+    def connected?
+      DataFabric.shard_active_for?(@shard_group) and cached_connections[connection_name]
+    end
+
+    def set_role(role)
+      @role = role if @replicated
+    end
+
+    def master
+      with_master { return connection }
+    end
+  end
+
+  module ActiveRecordConnectionMethods
+    def self.included(base)
+      base.alias_method_chain :reload, :master
+    end
+
+    def reload_with_master(*args, &block)
+      connection.with_master { reload_without_master }
+    end
+  end
+
+  class StringProxy
+    def initialize(&block)
+      @proc = block
+    end
+    def to_s
+      @proc.call
+    end
+  end
+end