data_fabric 1.0.0

data/example/public/robots.txt

@@ -0,0 +1,5 @@
+# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
+#
+# To ban all spiders from the entire site uncomment the next two lines:
+# User-Agent: *
+# Disallow: /

data/example/script/about

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../config/boot'
+require 'commands/about'

data/example/script/console

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../config/boot'
+require 'commands/console'

data/example/script/dbconsole

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../config/boot'
+require 'commands/dbconsole'

data/example/script/destroy

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../config/boot'
+require 'commands/destroy'

data/example/script/generate

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../config/boot'
+require 'commands/generate'

data/example/script/performance/benchmarker

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../../config/boot'
+require 'commands/performance/benchmarker'

data/example/script/performance/profiler

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../../config/boot'
+require 'commands/performance/profiler'

data/example/script/performance/request

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../../config/boot'
+require 'commands/performance/request'

data/example/script/plugin

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../config/boot'
+require 'commands/plugin'

data/example/script/process/inspector

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../../config/boot'
+require 'commands/process/inspector'

data/example/script/process/reaper

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../../config/boot'
+require 'commands/process/reaper'

data/example/script/process/spawner

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../../config/boot'
+require 'commands/process/spawner'

data/example/script/runner

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../config/boot'
+require 'commands/runner'

data/example/script/server

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../config/boot'
+require 'commands/server'

data/example/test/fixtures/accounts.yml

@@ -0,0 +1,7 @@
+# Read about fixtures at http://ar.rubyonrails.org/classes/Fixtures.html
+
+<% %w(fiveruns google yahoo microsoft).each_with_index do |name, idx| %>
+<%= name %>:
+  name: <%= name %> 
+  shard: <%= idx % 2 %>
+<% end %>

data/example/test/functional/accounts_controller_test.rb

@@ -0,0 +1,12 @@
+require 'test_helper'
+
+class AccountsControllerTest < ActionController::TestCase
+  fixtures :accounts
+  
+  def test_index
+    get :index
+    assert_response :success
+    assert assigns(:accounts)
+    assert_equal 4, assigns(:accounts).size
+  end
+end

data/example/test/integration/account_figments_test.rb

@@ -0,0 +1,95 @@
+require 'test_helper'
+
+class AccountFigmentsTest < ActionController::IntegrationTest
+
+  def test_create_account_and_figments
+    conn0 = db_connection "shard_0_test"
+    conn1 = db_connection "shard_1_test"
+    conn0.clear 'figments'
+    conn1.clear 'figments'
+    assert_equal 0, conn0.count_for('figments')
+    assert_equal 0, conn1.count_for('figments')
+
+    new_session do |user|
+      user.goes_home
+      mike = user.creates_account('mike', '0')
+      user.selects_account(mike)
+      before = mike.figments.size
+      user.creates_figment(14)
+      mike.figments.reload
+      assert_equal before + 1, mike.figments.size
+      assert_equal 14, mike.figments.first.value
+    end
+
+    # Bypass data_fabric and verify the figment went to shard 0.
+    assert_equal 1, conn0.count_for('figments')
+    assert_equal 0, conn1.count_for('figments')
+
+    new_session do |user|
+      user.goes_home
+      bob = user.creates_account('bob', '1')
+      user.selects_account(bob)
+      before = bob.figments.size
+      user.creates_figment(66)
+      bob.figments.reload
+      assert_equal before + 1, bob.figments.size
+      assert_equal 66, bob.figments.first.value
+    end
+
+    # Bypass data_fabric and verify the figment went to shard 1.
+    assert_equal 1, conn0.count_for('figments')
+    assert_equal 1, conn1.count_for('figments')
+  end
+
+  private
+  
+  def db_connection(conf)
+    conn = ActiveRecord::Base.sqlite3_connection(HashWithIndifferentAccess.new(ActiveRecord::Base.configurations[conf]))
+    def conn.count_for(table)
+      Integer(execute("select count(*) as c from #{table}")[0]['c'])
+    end
+    def conn.clear(table)
+      execute("delete from #{table}")
+    end
+    conn
+  end
+
+  def new_session
+    open_session do |sess|
+      sess.extend(Operations)
+      yield sess if block_given?
+    end
+  end
+
+  module Operations
+    def goes_home
+      get accounts_path
+      assert_response :success
+    end
+
+    def creates_account(name, shard)
+      post accounts_path, :acct => { :name => name, :shard => shard }
+      assert_response :redirect
+      follow_redirect!
+      assert_response :success
+      assert_template "accounts/index"
+      Account.find_by_name(name)
+    end
+
+    def selects_account(account)
+      get choose_account_path(account)
+      assert_response :redirect
+      follow_redirect!
+      assert_response :success
+      assert_template "accounts/index"
+    end
+    
+    def creates_figment(value)
+      post figments_path, :figment => { :value => value }
+      assert_response :redirect
+      follow_redirect!
+      assert_response :success
+      assert_template "accounts/index"
+    end
+  end
+end

data/example/test/test_helper.rb

@@ -0,0 +1,41 @@
+ENV["RAILS_ENV"] = "test"
+require File.expand_path(File.dirname(__FILE__) + "/../config/environment")
+require 'test_help'
+
+ActiveRecord::Base.logger = Logger.new(STDOUT)
+ActiveRecord::Base.logger.level = Logger::DEBUG
+
+class Test::Unit::TestCase
+  # Transactional fixtures accelerate your tests by wrapping each test method
+  # in a transaction that's rolled back on completion.  This ensures that the
+  # test database remains unchanged so your fixtures don't have to be reloaded
+  # between every test method.  Fewer database queries means faster tests.
+  #
+  # Read Mike Clark's excellent walkthrough at
+  #   http://clarkware.com/cgi/blosxom/2005/10/24#Rails10FastTesting
+  #
+  # Every Active Record database supports transactions except MyISAM tables
+  # in MySQL.  Turn off transactional fixtures in this case; however, if you
+  # don't care one way or the other, switching from MyISAM to InnoDB tables
+  # is recommended.
+  #
+  # The only drawback to using transactional fixtures is when you actually 
+  # need to test transactions.  Since your test is bracketed by a transaction,
+  # any transactions started in your code will be automatically rolled back.
+  self.use_transactional_fixtures = true
+
+  # Instantiated fixtures are slow, but give you @david where otherwise you
+  # would need people(:david).  If you don't want to migrate your existing
+  # test cases which use the @david style and don't mind the speed hit (each
+  # instantiated fixtures translates to a database query per test method),
+  # then set this back to true.
+  self.use_instantiated_fixtures  = false
+
+  # Setup all fixtures in test/fixtures/*.(yml|csv) for all tests in alphabetical order.
+  #
+  # Note: You'll currently still have to declare fixtures explicitly in integration tests
+  # -- they do not yet inherit this setting
+  fixtures :all
+
+  # Add more helper methods to be used by all tests here...
+end

data/example/vendor/plugins/data_fabric/init.rb

@@ -0,0 +1 @@
+DataFabric.init

data/example/vendor/plugins/data_fabric/lib/data_fabric.rb

@@ -0,0 +1,231 @@
+require 'active_record'
+require 'active_record/version'
+
+# DataFabric adds a new level of flexibility to ActiveRecord connection handling.
+# You need to describe the topology for your database infrastructure in your model(s).  As with ActiveRecord normally, different models can use different topologies.
+# 
+# class MyHugeVolumeOfDataModel < ActiveRecord::Base
+#   connection_topology :replicated => true, :shard_by => :city
+# end
+# 
+# There are four supported modes of operation, depending on the options given to the connection_topology method.  The plugin will look for connections in your config/database.yml with the following convention:
+# 
+# No connection topology:
+# #{environment} - this is the default, as with ActiveRecord, e.g. "production"
+# 
+# connection_topology :replicated => true
+# #{environment}_#{role} - no sharding, just replication, where role is "master" or "slave", e.g. "production_master"
+# 
+# connection_topology :shard_by => :city
+# #{group}_#{shard}_#{environment} - sharding, no replication, e.g. "city_austin_production"
+# 
+# connection_topology :replicated => true, :shard_by => :city
+# #{group}_#{shard}_#{environment}_#{role} - sharding with replication, e.g. "city_austin_production_master"
+# 
+# 
+# When marked as replicated, all write and transactional operations for the model go to the master, whereas read operations go to the slave.
+# 
+# Since sharding is an application-level concern, your application must set the shard to use based on the current request or environment.  The current shard for a group is set on a thread local variable.  For example, you can set the shard in an ActionController around_filter based on the user as follows:
+# 
+# class ApplicationController < ActionController::Base
+#   around_filter :select_shard
+#   
+#   private
+#   def select_shard(&action_block)
+#     DataFabric.activate_shard(:city => @current_user.city, &action_block)
+#   end
+# end
+module DataFabric
+  VERSION = "1.0.0"
+  
+  def self.logger
+    ActiveRecord::Base.logger
+  end
+
+  def self.init
+    logger.info "Loading data_fabric #{VERSION} with ActiveRecord #{ActiveRecord::VERSION::STRING}"
+    ActiveRecord::Base.send(:include, self)
+  end
+  
+  def self.activate_shard(shards, &block)
+    ensure_setup
+    shards.each do |key, value|
+      Thread.current[:shards][key.to_s] = value.to_s
+    end
+    if block_given?
+      begin
+        yield
+      ensure
+        shards.each do |key, value|
+          Thread.current[:shards].delete(key.to_s)
+        end
+      end
+    end
+  end
+  
+  # For cases where you can't pass a block to activate_shards, you can
+  # clean up the thread local settings by calling this method at the
+  # end of processing
+  def self.deactivate_shard(shards)
+    ensure_setup
+    shards.each do |key, value|
+      Thread.current[:shards].delete(key.to_s)
+    end
+  end
+  
+  def self.active_shard(group)
+    raise ArgumentError, 'No shard has been activated' unless Thread.current[:shards]
+
+    returning(Thread.current[:shards][group.to_s]) do |shard|
+      raise ArgumentError, "No active shard for #{group}" unless shard
+    end
+  end
+  
+  def self.included(model)
+    # Wire up ActiveRecord::Base
+    model.extend ClassMethods
+  end
+
+  def self.ensure_setup
+    Thread.current[:shards] = {} unless Thread.current[:shards]
+  end
+  
+  # Class methods injected into ActiveRecord::Base
+  module ClassMethods
+    def connection_topology(options)
+      proxy = DataFabric::ConnectionProxy.new(self, options)
+      ActiveRecord::Base.active_connections[name] = proxy
+      
+      raise ArgumentError, "data_fabric does not support ActiveRecord's allow_concurrency = true" if allow_concurrency
+      DataFabric.logger.info "Creating data_fabric proxy for class #{name}"
+    end
+  end
+  
+  class StringProxy
+    def initialize(&block)
+      @proc = block
+    end
+    def to_s
+      @proc.call
+    end
+  end
+
+  class ConnectionProxy
+    def initialize(model_class, options)
+      @model_class = model_class      
+      @replicated = options[:replicated]
+      @shard_group = options[:shard_by]
+      @prefix = options[:prefix]
+      @current_role = 'slave' if @replicated
+      @current_connection_name_builder = connection_name_builder
+      @cached_connection = nil
+      @current_connection_name = nil
+      @role_changed = false
+
+      @model_class.send :include, ActiveRecordConnectionMethods if @replicated
+    end
+
+    delegate :insert, :update, :delete, :create_table, :rename_table, :drop_table, :add_column, :remove_column, 
+      :change_column, :change_column_default, :rename_column, :add_index, :remove_index, :initialize_schema_information,
+      :dump_schema_information, :to => :master
+    
+    def transaction(start_db_transaction = true, &block)
+      with_master { raw_connection.transaction(start_db_transaction, &block) }
+    end
+
+    def method_missing(method, *args, &block)
+      unless @cached_connection and !@role_changed
+        raw_connection
+        @role_changed = false
+      end
+      if logger.debug?
+        logger.debug("Calling #{method} on #{@cached_connection}")
+      end
+      @cached_connection.send(method, *args, &block) 
+    end
+    
+    def connection_name
+      @current_connection_name_builder.join('_')
+    end
+    
+    def disconnect!
+      @cached_connection.disconnect! if @cached_connection
+      @cached_connection = nil
+    end
+    
+    def verify!(arg)
+      @cached_connection.verify!(0) if @cached_connection
+    end
+    
+    def with_master
+      set_role('master')
+      yield
+    ensure
+      set_role('slave')
+    end
+
+    private
+    
+    def connection_name_builder
+      clauses = []
+      clauses << @prefix if @prefix
+      clauses << @shard_group if @shard_group
+      clauses << StringProxy.new { DataFabric.active_shard(@shard_group) } if @shard_group
+      clauses << RAILS_ENV
+      clauses << StringProxy.new { @current_role } if @replicated
+      clauses
+    end
+    
+    def raw_connection
+      conn_name = connection_name
+      unless already_connected_to? conn_name 
+        @cached_connection = begin 
+          config = ActiveRecord::Base.configurations[conn_name]
+          raise ArgumentError, "Unknown database config: #{conn_name}, have #{ActiveRecord::Base.configurations.inspect}" unless config
+          @model_class.establish_connection config
+          if logger.debug?
+            logger.debug "Switching from #{@current_connection_name} to #{conn_name}"
+          end
+          @current_connection_name = conn_name
+          conn = @model_class.connection
+          conn.verify! 0
+          conn
+        end
+        @model_class.active_connections[@model_class.name] = self
+      end
+      @cached_connection
+    end
+    
+    def already_connected_to?(conn_name)
+      conn_name == @current_connection_name and @cached_connection
+    end
+    
+    def set_role(role)
+      if @replicated and @current_role != role
+        @current_role = role
+        @role_changed = true
+      end
+    end
+    
+    def master
+      set_role('master')
+      return raw_connection
+    ensure
+      set_role('slave')
+    end
+    
+    def logger
+      DataFabric.logger
+    end
+  end
+
+  module ActiveRecordConnectionMethods
+    def self.included(base)
+      base.alias_method_chain :reload, :master
+    end
+
+    def reload_with_master(*args, &block)
+      connection.with_master { reload_without_master }
+    end
+  end
+end