lookout-data_fabric 1.5.0

data/example30/config/locales/en.yml

@@ -0,0 +1,5 @@
+# Sample localization file for English. Add more files in this directory for other locales.
+# See http://github.com/svenfuchs/rails-i18n/tree/master/rails%2Flocale for starting points.
+
+en:
+  hello: "Hello world"

data/example30/config/routes.rb

@@ -0,0 +1,65 @@
+Example30::Application.routes.draw do
+  resources :figments
+  resources :accounts do
+    member do 
+      get :choose
+    end
+  end
+  root :to => 'accounts#index'
+  # The priority is based upon order of creation:
+  # first created -> highest priority.
+
+  # Sample of regular route:
+  #   match 'products/:id' => 'catalog#view'
+  # Keep in mind you can assign values other than :controller and :action
+
+  # Sample of named route:
+  #   match 'products/:id/purchase' => 'catalog#purchase', :as => :purchase
+  # This route can be invoked with purchase_url(:id => product.id)
+
+  # Sample resource route (maps HTTP verbs to controller actions automatically):
+  #   resources :products
+
+  # Sample resource route with options:
+  #   resources :products do
+  #     member do
+  #       get 'short'
+  #       post 'toggle'
+  #     end
+  #
+  #     collection do
+  #       get 'sold'
+  #     end
+  #   end
+
+  # Sample resource route with sub-resources:
+  #   resources :products do
+  #     resources :comments, :sales
+  #     resource :seller
+  #   end
+
+  # Sample resource route with more complex sub-resources
+  #   resources :products do
+  #     resources :comments
+  #     resources :sales do
+  #       get 'recent', :on => :collection
+  #     end
+  #   end
+
+  # Sample resource route within a namespace:
+  #   namespace :admin do
+  #     # Directs /admin/products/* to Admin::ProductsController
+  #     # (app/controllers/admin/products_controller.rb)
+  #     resources :products
+  #   end
+
+  # You can have the root of your site routed with "root"
+  # just remember to delete public/index.html.
+  # root :to => "welcome#index"
+
+  # See how all your routes lay out with "rake routes"
+
+  # This is a legacy wild controller route that's not recommended for RESTful applications.
+  # Note: This route will make all actions in every controller accessible via GET requests.
+  # match ':controller(/:action(/:id(.:format)))'
+end

data/example30/db/migrate/20080702154628_create_accounts.rb

@@ -0,0 +1,14 @@
+class CreateAccounts < ActiveRecord::Migration
+  def self.up
+    create_table :accounts do |t|
+      t.string :name
+      t.string :shard
+
+      t.timestamps
+    end
+  end
+
+  def self.down
+    drop_table :accounts
+  end
+end

data/example30/db/migrate/20080702154820_create_figments.rb

@@ -0,0 +1,14 @@
+class CreateFigments < ActiveRecord::Migration
+  def self.up
+    create_table :figments do |t|
+      t.integer :account_id
+      t.integer :value
+
+      t.timestamps
+    end
+  end
+
+  def self.down
+    drop_table :figments
+  end
+end

data/example30/db/seeds.rb

@@ -0,0 +1,7 @@
+# This file should contain all the record creation needed to seed the database with its default values.
+# The data can then be loaded with the rake db:seed (or created alongside the db with db:setup).
+#
+# Examples:
+#
+#   cities = City.create([{ :name => 'Chicago' }, { :name => 'Copenhagen' }])
+#   Mayor.create(:name => 'Daley', :city => cities.first)

data/example30/script/rails

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# This command will automatically be run when you run "rails" with Rails 3 gems installed from the root of your application.
+
+APP_PATH = File.expand_path('../../config/application',  __FILE__)
+require File.expand_path('../../config/boot',  __FILE__)
+require 'rails/commands'

data/example30/test/fixtures/accounts.yml

@@ -0,0 +1,7 @@
+# Read about fixtures at http://ar.rubyonrails.org/classes/Fixtures.html
+
+<% %w(fiveruns google yahoo microsoft).each_with_index do |name, idx| %>
+<%= name %>:
+  name: <%= name %> 
+  shard: <%= idx % 2 %>
+<% end %>

data/example30/test/functional/accounts_controller_test.rb

@@ -0,0 +1,12 @@
+require 'test_helper'
+
+class AccountsControllerTest < ActionController::TestCase
+  fixtures :accounts
+  
+  def test_index
+    get :index
+    assert_response :success
+    assert assigns(:accounts)
+    assert_equal 4, assigns(:accounts).size
+  end
+end

data/example30/test/integration/account_figments_test.rb

@@ -0,0 +1,97 @@
+require 'test_helper'
+
+class AccountFigmentsTest < ActionController::IntegrationTest
+
+  def test_create_account_and_figments
+    conn0 = db_connection "shard_0_test"
+    conn1 = db_connection "shard_1_test"
+    conn0.clear 'figments'
+    conn1.clear 'figments'
+    assert_equal 0, conn0.count_for('figments')
+    assert_equal 0, conn1.count_for('figments')
+
+    new_session(0) do |user|
+      user.goes_home
+      mike = user.creates_account('mike', '0')
+      user.selects_account(mike)
+      before = mike.figments.size
+      user.creates_figment(14)
+      mike.figments.reload
+      assert_equal before + 1, mike.figments.size
+      assert_equal 14, mike.figments.first.value
+    end
+
+    # Bypass data_fabric and verify the figment went to shard 0.
+    assert_equal 1, conn0.count_for('figments')
+    assert_equal 0, conn1.count_for('figments')
+
+    new_session(1) do |user|
+      user.goes_home
+      bob = user.creates_account('bob', '1')
+      user.selects_account(bob)
+      before = bob.figments.size
+      user.creates_figment(66)
+      bob.figments.reload
+      assert_equal before + 1, bob.figments.size
+      assert_equal 66, bob.figments.first.value
+    end
+
+    # Bypass data_fabric and verify the figment went to shard 1.
+    assert_equal 1, conn0.count_for('figments')
+    assert_equal 1, conn1.count_for('figments')
+  end
+
+  private
+  
+  def db_connection(conf)
+    conn = ActiveRecord::Base.sqlite3_connection(HashWithIndifferentAccess.new(ActiveRecord::Base.configurations[conf]))
+    def conn.count_for(table)
+      Integer(execute("select count(*) as c from #{table}")[0]['c'])
+    end
+    def conn.clear(table)
+      execute("delete from #{table}")
+    end
+    conn
+  end
+
+  def new_session(shard)
+    open_session do |sess|
+      DataFabric.activate_shard :shard => shard do
+        sess.extend(Operations)
+        yield sess if block_given?
+      end
+    end
+  end
+
+  module Operations
+    def goes_home
+      get accounts_path
+      assert_response :success
+    end
+
+    def creates_account(name, shard)
+      post accounts_path, :acct => { :name => name, :shard => shard }
+      assert_response :redirect
+      follow_redirect!
+      assert_response :success
+      assert_template "accounts/index"
+      Account.find_by_name(name)
+    end
+
+    def selects_account(account)
+      get choose_account_path(account)
+      assert_response :redirect
+      follow_redirect!
+      assert_response :success
+      assert_template "accounts/index"
+    end
+    
+    def creates_figment(value)
+      post figments_path, :figment => { :value => value }
+      assert_response :redirect
+      follow_redirect!
+      assert_response :success
+      assert_template "accounts/index"
+    end
+  end
+end

data/example30/test/performance/browsing_test.rb

@@ -0,0 +1,9 @@
+require 'test_helper'
+require 'rails/performance_test_help'
+
+# Profiling results for each test method are written to tmp/performance.
+class BrowsingTest < ActionDispatch::PerformanceTest
+  def test_homepage
+    get '/'
+  end
+end

data/example30/test/test_helper.rb

@@ -0,0 +1,13 @@
+ENV["RAILS_ENV"] = "test"
+require File.expand_path('../../config/environment', __FILE__)
+require 'rails/test_help'
+
+class ActiveSupport::TestCase
+  # Setup all fixtures in test/fixtures/*.(yml|csv) for all tests in alphabetical order.
+  #
+  # Note: You'll currently still have to declare fixtures explicitly in integration tests
+  # -- they do not yet inherit this setting
+  fixtures :all
+
+  # Add more helper methods to be used by all tests here...
+end

data/lib/data_fabric.rb

@@ -0,0 +1,107 @@
+require 'active_record'
+require 'active_record/version'
+require 'active_record/connection_adapters/abstract/connection_pool'
+require 'active_record/connection_adapters/abstract/connection_specification'
+require 'data_fabric/version'
+
+# DataFabric adds a new level of flexibility to ActiveRecord connection handling.
+# You need to describe the topology for your database infrastructure in your model(s).  As with ActiveRecord normally, different models can use different topologies.
+# 
+# class MyHugeVolumeOfDataModel < ActiveRecord::Base
+#   data_fabric :replicated => true, :shard_by => :city
+# end
+# 
+# There are four supported modes of operation, depending on the options given to the data_fabric method.  The plugin will look for connections in your config/database.yml with the following convention:
+# 
+# No connection topology:
+# #{environment} - this is the default, as with ActiveRecord, e.g. "production"
+# 
+# data_fabric :replicated => true
+# #{environment}_#{role} - no sharding, just replication, where role is "master" or "slave", e.g. "production_master"
+# 
+# data_fabric :shard_by => :city
+# #{group}_#{shard}_#{environment} - sharding, no replication, e.g. "city_austin_production"
+# 
+# data_fabric :replicated => true, :shard_by => :city
+# #{group}_#{shard}_#{environment}_#{role} - sharding with replication, e.g. "city_austin_production_master"
+# 
+# 
+# When marked as replicated, all write and transactional operations for the model go to the master, whereas read operations go to the slave.
+# 
+# Since sharding is an application-level concern, your application must set the shard to use based on the current request or environment.  The current shard for a group is set on a thread local variable.  For example, you can set the shard in an ActionController around_filter based on the user as follows:
+# 
+# class ApplicationController < ActionController::Base
+#   around_filter :select_shard
+#   
+#   private
+#   def select_shard(&action_block)
+#     DataFabric.activate_shard(:city => @current_user.city, &action_block)
+#   end
+# end
+module DataFabric
+
+  def self.logger
+    @logger ||= ActiveRecord::Base.logger || default_logger 
+  end
+  
+  def self.logger=(log)
+    @logger = log
+  end
+  
+  def self.activate_shard(shards, &block)
+    ensure_setup
+
+    # Save the old shard settings to handle nested activation
+    old = Thread.current[:shards].dup
+
+    shards.each_pair do |key, value|
+      Thread.current[:shards][key.to_s] = value.to_s
+    end
+    if block_given?
+      begin
+        yield
+      ensure
+        Thread.current[:shards] = old
+      end
+    end
+  end
+  
+  # For cases where you can't pass a block to activate_shards, you can
+  # clean up the thread local settings by calling this method at the
+  # end of processing
+  def self.deactivate_shard(shards)
+    ensure_setup
+    shards.each do |key, value|
+      Thread.current[:shards].delete(key.to_s)
+    end
+  end
+  
+  def self.active_shard(group)
+    raise ArgumentError, 'No shard has been activated' unless Thread.current[:shards]
+
+    shard = Thread.current[:shards][group.to_s]
+    raise ArgumentError, "No active shard for #{group}" unless shard
+    shard
+  end
+  
+  def self.shard_active_for?(group)
+    return true unless group
+    Thread.current[:shards] and Thread.current[:shards][group.to_s]
+  end
+
+  def self.ensure_setup
+    Thread.current[:shards] = {} unless Thread.current[:shards]
+  end
+
+  private
+  def self.default_logger
+    devnull = RUBY_PLATFORM =~ /w32/ ? 'nul' : '/dev/null'
+    l = Logger.new(devnull)
+    l.level = Logger::INFO
+    l
+  end
+
+end
+
+require 'data_fabric/extensions'
+ActiveRecord::Base.send(:include, DataFabric::Extensions)

data/lib/data_fabric/connection_proxy.rb

@@ -0,0 +1,176 @@
+module DataFabric
+  module ActiveRecordConnectionMethods
+    def self.included(base)
+      unless base.method_defined? :reload_without_master
+        base.alias_method_chain :reload, :master
+      end
+    end
+
+    def reload_with_master(*args, &block)
+      connection.with_master { reload_without_master }
+    end
+  end
+
+  class StringProxy
+    def initialize(&block)
+      @proc = block
+    end
+    def to_s
+      @proc.call
+    end
+  end
+
+  class PoolProxy
+    def initialize(proxy)
+      @proxy = proxy
+    end
+
+    def connection
+      @proxy
+    end
+
+    def spec
+      @proxy.current_pool.spec
+    end
+
+    def with_connection
+      yield @proxy
+    end
+
+    def connected?
+      @proxy.connected?
+    end
+
+    %w(disconnect! release_connection clear_reloadable_connections! clear_stale_cached_connections! verify_active_connections!).each do |name|
+      define_method(name.to_sym) do
+        @proxy.shard_pools.values.each do |pool|
+          pool.send(name.to_sym)
+        end
+      end
+    end
+
+    %w(columns column_defaults columns_hash table_exists? primary_keys).each do |name|
+      define_method(name.to_sym) do |*args|
+        @proxy.current_pool.send(name.to_sym, *args)
+      end
+    end
+
+    def method_missing(name, *args)
+      DataFabric.logger.warn "Add '#{name}' to DataFabric::PoolProxy for performance"
+      @proxy.current_pool.send(name, *args)
+    end
+  end
+
+  class ConnectionProxy
+    cattr_accessor :shard_pools
+
+    def initialize(model_class, options)
+      @model_class = model_class      
+      @replicated  = options[:replicated]
+      @shard_group = options[:shard_by]
+      @prefix      = options[:prefix]
+      set_role('slave') if @replicated
+
+      @model_class.send :include, ActiveRecordConnectionMethods if @replicated
+    end
+
+    delegate :insert, :update, :delete, :create_table, :rename_table, :drop_table, :add_column, :remove_column, 
+      :change_column, :change_column_default, :rename_column, :add_index, :remove_index, :initialize_schema_information,
+      :dump_schema_information, :execute, :execute_ignore_duplicate, :to => :master
+
+    delegate :insert_many, :to => :master # ar-extensions bulk insert support
+
+    def transaction(start_db_transaction = true, &block)
+      with_master do
+        connection.transaction(start_db_transaction, &block)
+      end
+    end
+
+    def respond_to?(method)
+      super || connection.respond_to?(method)
+    end
+
+    def method_missing(method, *args, &block)
+      DataFabric.logger.debug { "Calling #{method} on #{connection}" }
+      connection.send(method, *args, &block)
+    end
+
+    def connection_name
+      connection_name_builder.join('_')
+    end
+
+    def with_master
+      # Allow nesting of with_master.
+      old_role = current_role
+      set_role('master')
+      yield
+    ensure
+      set_role(old_role)
+    end
+
+    def connected?
+      current_pool.connected?
+    end
+
+    def connection
+      current_pool.connection
+    end
+
+    def current_pool
+      name = connection_name
+      self.class.shard_pools[name] ||= begin
+        config = ActiveRecord::Base.configurations[name]
+        raise ArgumentError, "Unknown database config: #{name}" unless config
+        ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec_for(config))
+      end
+    end
+
+    private
+
+    def spec_for(config)
+      config = config.symbolize_keys
+      adapter_method = "#{config[:adapter]}_connection"
+      initialize_adapter(config[:adapter])
+      ActiveRecord::Base::ConnectionSpecification.new(config, adapter_method)
+    end
+
+    def initialize_adapter(adapter)
+      begin
+        require 'rubygems'
+        gem "activerecord-#{adapter}-adapter"
+        require "active_record/connection_adapters/#{adapter}_adapter"
+      rescue LoadError
+        begin
+          require "active_record/connection_adapters/#{adapter}_adapter"
+        rescue LoadError
+          raise "Please install the #{adapter} adapter: `gem install activerecord-#{adapter}-adapter` (#{$!})"
+        end
+      end
+    end
+
+    def connection_name_builder
+      @connection_name_builder ||= begin
+        clauses = []
+        clauses << @prefix if @prefix
+        clauses << @shard_group if @shard_group
+        clauses << StringProxy.new { DataFabric.active_shard(@shard_group) } if @shard_group
+        clauses << Rails.env
+        clauses << StringProxy.new { current_role } if @replicated
+        clauses
+      end
+    end
+
+    def set_role(role)
+      Thread.current[:data_fabric_role] = role
+    end
+
+    def current_role
+      Thread.current[:data_fabric_role] || 'slave'
+    end
+
+    def master
+      with_master { return connection }
+    end
+  end
+
+end