xbar 0.0.1 → 0.4.0

data/lib/xbar/shard.rb

@@ -0,0 +1,95 @@
+# For debugging.
+require File.expand_path(File.join(File.dirname(__FILE__),  'colors'))
+
+module XBar
+  class Shard
+    include Colors
+    
+    # Methods that we invoke on the proxy.
+    [:current_model, :in_block_scope?].each do |meth|
+      define_method(meth) {@proxy.send(meth)}
+    end
+
+    attr_reader :shard_name, :master, :proxy, :slaves
+      
+    def initialize(proxy, name, master, slaves)
+      @master = master # a connection pool
+      @shard_name = name
+      @slaves = slaves # an array of connection pools
+      @proxy = proxy # our parent proxy
+      @slave_index = 0
+    end
+    
+    def run_queries(method, *args, &block)
+      if XBar.debug
+        puts("Shard##{BLUE_TEXT}run_queries#{RESET_COLORS}: " + 
+          "method = #{RED_TEXT}#{method}#{RESET_COLORS}, " +
+          "shard= #{shard_name}, " +
+          "block_scope = #{in_block_scope?}")
+      end
+      if method.to_s =~ /select/ && !in_block_scope? # XXX needed???, eventual flag??
+        # OK to send the query to a slave
+        run_queries_on_slave(method, *args, &block)
+      else
+        # Use the master.
+        run_queries_on_replica(master, method, *args, &block)
+      end
+    end
+    
+    def transaction(options, &block)
+      if XBar.debug
+        config = master.spec.config
+        puts("\nShard##{BLUE_TEXT}transaction#{RESET_COLORS}: " +
+          "shard_name=master, shard=#{shard_name}, " +
+          "Host=#{config[:host]}, Port=#{config[:port]}, Database=#{config[:database]}")
+      end
+      proxy.enter_statistics(@shard_name, master.spec.config, 'transaction')
+      prepare_connection_pool(master)
+      master.connection.transaction(options, &block)
+    end
+
+    def schema_cache
+      # defined by attr_reader in AbstractAdapter class.
+      master.connection.schema_cache
+    end
+
+    def quote_table_name(table_name)
+      master.connection.quote_table_name(table_name)
+    end
+        
+    private
+
+    def prepare_connection_pool(pool)
+      pool.automatic_reconnect = true if XBar.rails31?
+      pool.verify_active_connections! if Mapper.options[:verify_connection]
+    end
+    
+    def run_queries_on_replica(replica, method, *args, &block)
+      if XBar.debug
+        config = replica.spec.config
+        puts("Shard##{BLUE_TEXT}run_queries_on_replica#{RESET_COLORS}: " +
+          "shard_name=#{shard_name}, " +
+          "Host=#{config[:host]}, Port=#{config[:port]}, " +
+          "Database=#{config[:database]}")
+      end
+      # proxy.enter_statistics(@shard_name, replica.spec.config, method) changes the shard
+      prepare_connection_pool(replica)
+      replica.connection.send(method, *args, &block)
+    end
+    
+    def run_queries_on_slave(method, *args, &block)
+      if current_model.unreplicated_model? || @slaves.empty?
+        replica = @master
+      else
+        if XBar.debug
+          puts "Shard##{BLUE_TEXT}run_queries_on_slave#{RESET_COLORS}: " +
+            "slave_index=#{@slave_index}"
+        end
+        replica = @slaves[@slave_index]
+        @slave_index = (@slave_index + 1) % @slaves.length
+      end
+      run_queries_on_replica(replica, method, *args, &block) # return sql
+    end
+    
+  end
+end

data/lib/xbar/version.rb

@@ -1,3 +1,3 @@
-module Xbar
-  VERSION = "0.0.1"
+module XBar
+  VERSION = '0.4.0'
 end

data/lib/xbar.rb

@@ -1,5 +1,124 @@
+if ActiveRecord::VERSION::MAJOR == 3 && ActiveRecord::VERSION::MINOR < 2
+  require 'active_support/core_ext/class/inheritable_attributes' # Removed in Rails 3.2.beta
+else
+  require 'active_support/core_ext/class/attribute' # maybe not needed
+  ActiveRecord::Base.send(:class_attribute, :_unreplicated,
+    :_establish_connection, :_reset_table_name)
+end
+
+require "yaml"
+require "erb"
+require 'active_support/hash_with_indifferent_access'
+
+module XBar
+  
+  class ConfigError < StandardError; end
+  
+  class << self
+    attr_accessor :debug
+  end
+  
+  # There is no corresponding 'setter' method because this is not
+  # how the Rails environment is set.
+  def self.rails_env
+    defined?(Rails) ? Rails.env.to_s : nil
+  end
+  
+  def self.config
+    XBar::Mapper.config
+  end
+  
+  # Returns the Rails.root_to_s when you are using rails
+  # Running the current directory in a generic Ruby process
+  def self.directory
+    @directory ||= defined?(Rails) ?  Rails.root.to_s : Dir.pwd
+  end
+  
+  def self.directory=(dir)
+    @directory = dir
+  end
+  
+  def self.enable_stats
+    @stats = true
+  end
+  
+  def self.disable_stats
+    @stats = false
+  end
+  
+  def self.collect_stats?
+    @stats ||= false
+  end
+
+  # This is the default way to do XBar Setup
+
+  def self.setup
+    yield self
+  end
+
+  def self.environments=(environments)
+    @environments = environments.map { |element| element.to_s }
+  end
+
+  def self.environments
+    @environments || ['production', 'test', 'development']
+  end
+
+  def self.rails3?
+    ActiveRecord::VERSION::MAJOR == 3
+  end
+
+  def self.rails31?
+    ActiveRecord::VERSION::MAJOR == 3 && ActiveRecord::VERSION::MINOR >= 1
+  end
+  
+  def self.rails32?
+    ActiveRecord::VERSION::MAJOR == 3 && ActiveRecord::VERSION::MINOR >= 2
+  end
+
+  def self.rails?
+    defined?(Rails)
+  end
+
+  def self.shards=(shards)
+    XBar::Mapper.shards = shards
+  end
+
+  def self.using(shard, &block)
+    conn = ActiveRecord::Base.connection
+    if conn.is_a?(XBar::Proxy)
+      conn.run_queries_on_shard(shard, &block)
+    else
+      yield
+    end
+  end
+end
+
 require "xbar/version"
+require "xbar/mapper"
+require "xbar/model"
+require "xbar/migration"
+require "xbar/association_collection"
+require "xbar/has_and_belongs_to_many_association"
+require "xbar/association"
 
-module Xbar
-  # Your code goes here...
+if XBar.rails3?
+  require "xbar/rails3/association"
+  require "xbar/rails3/persistence"
+  require "xbar/rails3/arel"
+else
+  require "xbar/rails2/association"
+  require "xbar/rails2/persistence"
 end
+
+if XBar.rails31?
+  require "xbar/rails3.1/singular_association"
+end
+
+require "xbar/shard"
+require "xbar/proxy"
+require "xbar/scope_proxy"
+require "xbar/logger"
+
+ActiveRecord::Base.send(:include, XBar::Model)
+class XBarModel < ActiveRecord::Base; end; # used only in migrations

data/run

@@ -0,0 +1,27 @@
+#!/bin/bash
+#RUBY=/usr/local/rvm/rubies/ruby-1.9.3-p0/bin/ruby
+RDEBUG=/usr/local/rvm/gems/ruby-1.9.3-p0@edge/bin/rdebug
+RSPEC=/usr/local/rvm/gems/ruby-1.9.3-p0@edge/bin/rspec
+SPEC_DIR="spec/xbar"
+
+export BUNDLE_GEMFILE=/Volumes/Opt/projects/xbar/gemfiles/rails32.gemfile
+
+SPEC=$1
+LINE=$2
+
+if [ -n "$SPEC"  ]
+then
+  TEST="$SPEC_DIR/${SPEC}_spec.rb"
+else
+  echo "run.sh spec [line]"
+  exit 1
+fi
+
+if [ -n "$LINE" ]
+then
+  TEST="${TEST}:${LINE}"
+fi
+
+echo $TEST
+
+bundle exec ${RUBY} ${RSPEC} -fd -c ${TEST}

data/spec/config/acme.json

@@ -0,0 +1,53 @@
+{
+    "__COMMENT": "Acme Corporation Environments",
+
+    "connections": {
+        "mysql_m": "mysql2://root@localhost:3306/master",
+
+        "inventory_1": "postgres://postgres@127.0.0.1/russia_1",      
+        "inventory_2": "postgres://postgres@127.0.0.1/russia_2",      
+        "inventory_3": "postgres://postgres@127.0.0.1/russia_3",
+
+        "sales_1": "mysql2://root:graceling@deimos.thirdmode.com:3307/canada",      
+        "sales_2": "mysql2://root:graceling@deimos.thirdmode.com:3308/canada",      
+        "sales_3": "mysql2://root:graceling@deimos.thirdmode.com:3309/canada",
+        
+        "common_1": "mysql2://root:graceling@deimos.thirdmode.com:3307/brazil",
+        "common_2": "mysql2://root:graceling@deimos.thirdmode.com:3308/brazil",      
+        "common_3": "mysql2://root:graceling@deimos.thirdmode.com:3309/brazil"
+    },
+
+    "environments": {
+        "test": {
+            "shards": {
+                "master": "mysql_m",
+                "sales": "sales_1",
+                "inventory": "inventory_1",
+                "common": ["common_1", "common_2", "common_3"]
+            }
+        },
+        "development": {
+            "shards": {
+                "master": "mysql_m",
+                "sales": "sales_2",
+                "inventory": "inventory_2",
+                "common": ["common_1", "common_2", "common_3"]
+            }
+        },
+        "production": {
+            "shards": {
+                "master": "mysql_m",
+                "sales": "sales_3",
+                "inventory": "inventory_3",
+                "common": ["common_1", "common_2", "common_3"]
+            }
+        },
+				 "staging": {
+	            "shards": {
+	                "master": "mysql_m",
+	                "inventory": "inventory_3",
+	                "common": ["common_1", "common_2", "common_3"]
+	            }
+	        }
+    }
+}

data/spec/config/connection.rb

@@ -0,0 +1,2 @@
+ActiveRecord::Base.establish_connection(:adapter => "mysql2", :database => "master",
+  :host => "localhost", :port => 3306, :username => "root", :password => "")

data/spec/config/default.json

@@ -0,0 +1,160 @@
+{
+    "__COMMENT": "Shards for the default meta-environment.",
+
+    "connections": {
+        "mysql_m": "mysql2://root@localhost:3306/master",
+        "rogue_s": "mysql2://root@localhost:3306/rogue",
+        "london_s": "mysql2://root:graceling@deimos.thirdmode.com:3307/london",
+        "moscow_s": "postgres://postgres@127.0.0.1/moscow",
+        "paris_s": { "adapter": "sqlite3", "database": "/tmp/paris.sqlite3"},
+
+        "russia_1": "postgres://postgres@127.0.0.1/russia_1",      
+        "russia_2": "postgres://postgres@127.0.0.1/russia_2",      
+        "russia_3": "postgres://postgres@127.0.0.1/russia_3",
+
+        "canada_1": "mysql2://root:graceling@deimos.thirdmode.com:3307/canada",      
+        "canada_2": "mysql2://root:graceling@deimos.thirdmode.com:3308/canada",      
+        "canada_3": "mysql2://root:graceling@deimos.thirdmode.com:3309/canada",
+        
+        "brazil_1": "mysql2://root:graceling@deimos.thirdmode.com:3307/brazil",
+        "brazil_2": "mysql2://root:graceling@deimos.thirdmode.com:3308/brazil",      
+        "brazil_3": "mysql2://root:graceling@deimos.thirdmode.com:3309/brazil",
+
+        "__COMMENT": "The china database is setup in the MySQL configuration",
+        "__COMMENT": "to NOT be replicated.",
+        "china_1": "mysql2://root:graceling@deimos.thirdmode.com:3307/china",      
+        "china_2": "mysql2://root:graceling@deimos.thirdmode.com:3308/china"
+    },
+
+    "environments": {
+        "test": {
+						"any_option": true,
+            "shards": {
+                "master": "mysql_m",
+                
+                "__COMMENT": "MySQL singleton shard.",
+                "london": "london_s",
+
+                "__COMMENT": "SQLite singleton shard.",
+                "paris": "paris_s",
+
+                "__COMMENT": "Postgres singleton shard.",
+                "moscow": "moscow_s",
+                                                              
+                "__COMMENT": "Postgres unreplicated shard (for now)",
+		"__COMMENT": "One server has three different databases.",
+                "russia": ["russia_1", "russia_2", "russia_3"],
+
+                "__COMMENT": "Convenience to get at the shard replicas as shards.",
+                "russia_east": "russia_1",
+                "russia_central": "russia_2",
+                "russia_west": "russia_3",
+                                                                
+                "__COMMENT": "MySQL Fully replicated shard",
+                "canada": ["canada_1", "canada_2", "canada_3"],
+
+               "__COMMENT": "Convenience to get at the shard replicas as shards.",
+                "canada_east": "canada_1",
+                "canada_central": "canada_2",
+                "canada_west": "canada_3",
+
+                "__COMMENT": "MySQL fully relicated shard",
+                "brazil": ["brazil_1", "brazil_2", "brazil_3"],
+                                                                
+                "__COMMENT": "MySQL unreplicated shard",
+                "china": ["china_1", "china_2"],
+                                                                
+                "__COMMENT": "Convenience to get at the shard replicas as shards.",
+		"__COMMENT": "Two different servers have the same-named database.",
+                "china_east": "china_1",
+                "china_west": "china_2"
+            }
+        },
+        "development": {
+			      "verify_connection": true,
+				    "favorite_color": "blue",
+            "shards": {
+                "master": "mysql_m",
+
+                "__COMMENT": "MySQL singleton shard.",
+                "london": "london_s",
+
+                "__COMMENT": "SQLite singleton shard.",
+                "paris": "paris_s",
+
+                "__COMMENT": "Postgres singleton shard.",
+                "moscow": "moscow_s",
+
+                "__COMMENT": "Postgres unreplicated shard (for now)",
+		            "__COMMENT": "One server has three different databases.",
+                "russia": ["russia_1", "russia_2", "russia_3"],
+
+                "__COMMENT": "Convenience to get at the shard replicas as shards.",
+                "russia_east": "russia_1",
+                "russia_central": "russia_2",
+                "russia_west": "russia_3",
+
+                "__COMMENT": "MySQL Fully replicated shard",
+                "canada": ["canada_1", "canada_2", "canada_3"],
+
+               "__COMMENT": "Convenience to get at the shard replicas as shards.",
+                "canada_east": "canada_1",
+                "canada_central": "canada_2",
+                "canada_west": "canada_3",
+
+                "__COMMENT": "MySQL fully relicated shard",
+                "brazil": ["brazil_1", "brazil_2", "brazil_3"],
+
+                "__COMMENT": "MySQL unreplicated shard",
+                "china": ["china_1", "china_2"],
+
+                "__COMMENT": "Convenience to get at the shard replicas as shards.",
+		"__COMMENT": "Two different servers have the same-named database.",
+                "china_east": "china_1",
+                "china_west": "china_2"
+            }
+        },
+        "staging": {
+            "shards": {
+                "master": "mysql_m",
+                
+                "__COMMENT": "MySQL singleton shard.",
+                "london": "london_s",
+
+                "__COMMENT": "SQLite singleton shard.",
+                "paris": "paris_s",
+
+                "__COMMENT": "Postgres singleton shard.",
+                "moscow": "moscow_s",
+                                                              
+                "__COMMENT": "Postgres unreplicated shard (for now)",
+		"__COMMENT": "One server has three different databases.",
+                "russia": ["russia_1", "russia_2", "russia_3"],
+
+                "__COMMENT": "Convenience to get at the shard replicas as shards.",
+                "russia_east": "russia_1",
+                "russia_central": "russia_2",
+                "russia_west": "russia_3",
+                                                                
+                "__COMMENT": "MySQL Fully replicated shard",
+                "canada": ["canada_1", "canada_2", "canada_3"],
+
+               "__COMMENT": "Convenience to get at the shard replicas as shards.",
+                "canada_east": "canada_1",
+                "canada_central": "canada_2",
+                "canada_west": "canada_3",
+
+                "__COMMENT": "MySQL fully relicated shard",
+                "brazil": ["brazil_1", "brazil_2", "brazil_3"],
+                                                                
+                "__COMMENT": "MySQL unreplicated shard",
+                "china": ["china_1", "china_2"],
+                                                                
+                "__COMMENT": "Convenience to get at the shard replicas as shards.",
+		"__COMMENT": "Two different servers have the same-named database.",
+                "china_east": "china_1",
+                "china_west": "china_2"
+            }
+        }
+    }
+}

data/spec/config/duplicate_shard.json

@@ -0,0 +1,21 @@
+{
+    "__COMMENT": "Acme Corporation Environments with a duplicate shard error",
+
+    "connections": {
+        "mysql_m": "mysql2://root@localhost:3306/master",
+        "inventory_1": "postgres://postgres@127.0.0.1/russia_1",      
+        "inventory_2": "postgres://postgres@127.0.0.1/russia_2",
+        "sales_1": "mysql2://root:graceling@deimos.thirdmode.com:3307/canada"   
+    },
+
+    "environments": {
+        "test": {
+            "shards": {
+                "master": "mysql_m",
+                "sales": "sales_1",
+                "inventory": "inventory_1",
+								"inventory": "inventory_2"
+            }
+        }
+    }
+}

data/spec/config/missing_key.json

@@ -0,0 +1,20 @@
+{
+    "__COMMENT": "Acme Corporation Environments with a missing key error",
+
+    "connections": {
+        "mysql_m": "mysql2://root@localhost:3306/master",
+        "inventory_1": "postgres://postgres@127.0.0.1/russia_1",      
+        "inventory_2": "postgres://postgres@127.0.0.1/russia_2",
+        "sales_1": "mysql2://root:graceling@deimos.thirdmode.com:3307/canada"   
+    },
+
+    "environments": {
+        "test": {
+            "shards": {
+                "master": "mysql_m",
+                "sales": "sales_1",
+                "inventory": "inventory_3"
+            }
+        }
+    }
+}

data/spec/config/new_shards.json

@@ -0,0 +1,29 @@
+{
+  "connections": {
+     "p1": "postgres://user:password@alpha.server.com:5432/mydatabase?encoding=unicode&pool=5",
+     "p2": "postgres://user:password@beta.server.com:5432/mydatabase?encoding=unicode&pool=5",   
+     "m1": "mysql://user:password@alpha.server.com:3306/mydatabase?pool=10",     
+     "m2": "postgres://user:password@beta.server.com:3306/mydatabase?pool=10",
+     "m3": {"database": "xbar_shard5", "adapter": "mysql2", "host": "localhost" }
+  }, 
+  "environments": {
+    "development": {
+      "shards": [
+        {"name": "shard_one", "connections": ["p1", "p2"]},
+        {"name": "shard_one", "connections": ["p1", "p2"]}
+      ] 
+    },
+		"test": {
+      "shards": [
+        {"name": "shard_one", "connections": ["p1", "p2"]},
+        {"name": "shard_one", "connections": ["p1", "p2"]}
+      ]
+    },
+    "production": {
+      "shards": [
+        {"name": "shard_one", "connections": ["p1", "p2"]},
+        {"name": "shard_one", "connections": ["p1", "p2"]}
+      ]
+    }
+  }
+}

data/spec/config/no_master_shard.json

@@ -0,0 +1,19 @@
+{
+    "__COMMENT": "Acme Corporation Environments with a duplicate shard error",
+
+    "connections": {
+        "mysql_m": "mysql2://root@localhost:3306/master",
+        "inventory_1": "postgres://postgres@127.0.0.1/russia_1",      
+        "inventory_2": "postgres://postgres@127.0.0.1/russia_2",
+        "sales_1": "mysql2://root:graceling@deimos.thirdmode.com:3307/canada"     
+    },
+
+    "environments": {
+        "test": {
+            "shards": {
+                "sales": "sales_1",
+                "inventory": ["inventory_1", "inventory_2"]
+            }
+        }
+    }
+}

data/spec/config/not_entire_sharded.json

@@ -0,0 +1,23 @@
+{
+	"__COMMENT": "Not Entire Sharded",
+	
+	"entire_sharded": false,
+		
+	"connections": {
+    "europe": {"database": "xbar_shard2", "adapter": "mysql2", "host": "localhost"},
+    "canada": {"database": "xbar_shard3", "adapter": "mysql2", "host": "localhost"},
+    "brazil": {"database": "xbar_shard4", "adapter": "mysql2", "host": "localhost"},
+    "russia": {"database": "xbar_shard5", "adapter": "mysql2", "host": "localhost"}
+	},
+
+	"environments": {
+		"shards": {
+		  "shards": {
+		    "europe": "europe",
+		    "canada": "canada",
+		    "brazil": "brazil",
+		    "russia": "russia"
+		  }
+		}
+	}
+}

data/spec/config/octopus.json

@@ -0,0 +1,27 @@
+{
+    "__COMMENT": "Shards for the XBar meta-environment.",
+    
+    "connections": {
+        "alone_shard": "mysql2://localhost/xbar_shard5",
+        "aug2009": {"database": "xbar_shard2", "adapter": "mysql2", "host": "localhost"},
+        "aug2010": {"database": "xbar_shard3", "adapter": "mysql2", "host": "localhost"},
+        "aug2011": {"database": "xbar_shard4", "adapter": "mysql2", "host": "localhost"},
+        "postgresql_shard": {"adapter": "postgresql", "username": "postgres", "password": null, 
+          "database": "xbar_shard1", "encoding": "unicode"},
+            "sqlite_shard": { "adapter": "sqlite3", "database": "/tmp/database.sqlite3"},
+            "canada": "mysql2://localhost/xbar_shard2",
+        "brazil": {"database": "xbar_shard3", "adapter": "mysql2", "host": "localhost"},
+        "russia": {"database": "xbar_shard4", "adapter": "mysql2", "host": "localhost"}
+    },
+    "environments": {
+        "shards": {
+            "shards": {
+                "alone_shard": "alone_shard",
+                "postgresql_shard": "postgresql_shard",
+                "sqlite_shard": "sqlite_shard",
+                "history_shard": ["aug2009", "aug2010", "aug2011"],
+                "country_shard": ["canada", "brazil", "russia"]
+            }    
+        }
+    }
+}

data/spec/config/octopus_rails.json

@@ -0,0 +1,25 @@
+{
+	"__COMMENT": "XBar Rails Mapperuration",
+	"replicated": true,
+  "verify_connection": true,
+	"connections": {
+		"slave1": {"database": "xbar_shard2", "adapter": "mysql2", "host": "localhost"},
+    "slave2": {"database": "xbar_shard3", "adapter": "mysql2", "host": "localhost"},
+    "slave3": {"database": "xbar_shard4", "adapter": "mysql2", "host": "localhost"},
+    "slave4": {"database": "xbar_shard5", "adapter": "mysql2", "host": "localhost"}
+	},
+	"environments": {			  
+		"staging": {
+		  "shards": {
+		    "slave1": "slave1",
+		    "slave2": "slave2"
+		  }
+		},
+		"production": {
+			"shards": {
+		    "slave3": "slave3",
+		    "slave4": "slave4"
+		  }
+		}
+	}
+}

data/spec/config/production_fully_replicated.json

@@ -0,0 +1,21 @@
+{
+	"__COMMENT": "Production Fully Replicated",
+	"replicated": true,
+	"connections": {
+    "slave1": {"database": "xbar_shard2", "adapter": "mysql2", "host": "localhost"},
+    "slave2": {"database": "xbar_shard3", "adapter": "mysql2", "host": "localhost"},
+    "slave3": {"database": "xbar_shard4", "adapter": "mysql2", "host": "localhost"},
+    "slave4": {"database": "xbar_shard5", "adapter": "mysql2", "host": "localhost"}
+	},
+
+	"environments": {
+		"shards": {
+		  "shards": {
+		    "slave1": "slave1",
+		    "slave2": "slave2",
+		    "slave3": "slave3",
+		    "slave4": "slave4"
+		  }
+		}
+	}
+}