ar-octopus 0.0.29 → 0.0.30

data/README.mkdn

@@ -18,6 +18,9 @@ When using replication, all writes queries will be sent to master, and read quer
 ### Sharding
  When using sharding, you need to specify what shard the query will be sent. Octopus support selecting the shard inside a controller, or manually in each object. More could be found at <a href="http://wiki.github.com/tchandy/octopus/sharding"> Wiki</a> 
 
+### Replication + Sharding
+ Replication + Sharding isn't supported yet. This is in our todo, and will be done ASAP. if you need, feel free to fork and implement it.
+
 ## Install 
 
 ### Rails 2.x 
@@ -115,7 +118,7 @@ If you want to send a specified action, or all actions from a controller, to a s
 </pre>
 
  To see the complete list of features and syntax, please check out our <a href="http://wiki.github.com/tchandy/octopus/"> Wiki</a>
-Wanna see sample rails applications using octopus features? please check it out: <a href="http://github.com/tchandy/octopus_sharding_example">Sharding Example</a> and <a href="http://github.com/tchandy/octopus_replication_example">Replication Example</a> 
+Wanna see sample rails applications using octopus features? please check it out: <a href="http://github.com/tchandy/octopus_sharding_example">Sharding Example</a> and <a href="http://github.com/tchandy/octopus_replication_example">Replication Example</a>. Also, we have a example that shows how to use Octopus without Rails: <a href="http://github.com/tchandy/octopus_sinatra"> Octopus + Sinatra Example</a>.
 
 
 ## Contributing with Octopus

data/Rakefile

@@ -37,7 +37,7 @@ begin
     gem.add_development_dependency "jeweler", ">= 1.4"
     gem.add_development_dependency "actionpack", ">= 2.3"
     gem.add_dependency('activerecord', '>= 2.3')
-    gem.version = "0.0.29"
+    gem.version = "0.0.30"
   end
   Jeweler::GemcutterTasks.new
 rescue LoadError

data/TODO.txt

@@ -0,0 +1,6 @@
+- Find the best way to move data along shards.
+- Improve documentation.
+- Automatic configuration file for shards.
+- Support specifics cases of sharding/replication.
+- Basic algorithm to do load balancing between slaves server (handling servers down)
+- Support replication + sharding

data/ar-octopus.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{ar-octopus}
-  s.version = "0.0.29"
+  s.version = "0.0.30"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Thiago Pradi", "Mike Perham"]
-  s.date = %q{2010-07-30}
+  s.date = %q{2010-08-14}
   s.description = %q{This gem allows you to use sharded databases with ActiveRecord. this also provides a interface for replication and for running migrations with multiples shards.}
   s.email = %q{tchandy@gmail.com}
   s.extra_rdoc_files = [
@@ -22,12 +22,8 @@ Gem::Specification.new do |s|
      "Gemfile.lock",
      "README.mkdn",
      "Rakefile",
-     "VERSION",
+     "TODO.txt",
      "ar-octopus.gemspec",
-     "doc/api.textile",
-     "doc/features.textile",
-     "doc/libraries.textile",
-     "doc/shards.yml",
      "init.rb",
      "lib/octopus.rb",
      "lib/octopus/association.rb",

data/lib/octopus/model.rb

@@ -12,7 +12,7 @@ module Octopus::Model
 
     def using(shard)
       return self if defined?(::Rails) && !Octopus.enviroments.include?(Rails.env.to_s)
-
+      
       clean_table_name()
       hijack_initializer()
 

data/lib/octopus/proxy.rb

@@ -37,7 +37,11 @@ class Octopus::Proxy
 
   def initialize_replication(config)
     @replicated = true
-    @entire_replicated = config["entire_replicated"]
+    if config.has_key?("fully_replicated")
+      @fully_replicated = config["fully_replicated"]
+    else
+      @fully_replicated = true
+    end
     @slaves_list = @shards.keys.map {|sym| sym.to_s}.sort 
     @slaves_list.delete('master')   
   end
@@ -105,6 +109,16 @@ class Octopus::Proxy
       ActiveRecord::Base.using(shard).connection.initialize_schema_migrations_table 
     end
   end
+  
+  def transaction(options = {}, &block)
+    if @replicated && (current_model.read_inheritable_attribute(:replicated) || @fully_replicated)
+      self.run_queries_on_shard(:master) do
+        select_connection.transaction(options, &block)
+      end
+    else
+      select_connection.transaction(options, &block)
+    end
+  end
 
   def method_missing(method, *args, &block)
     if should_clean_connection?(method)
@@ -142,16 +156,14 @@ class Octopus::Proxy
 
   def send_queries_to_selected_slave(method, *args, &block)        
     old_shard = self.current_shard
-
-    if current_model.read_inheritable_attribute(:replicated) || @entire_replicated
-      if !using_enabled
-        self.current_shard = @slaves_list.shift.to_sym
-        @slaves_list << self.current_shard
-      end
+    
+    if current_model.read_inheritable_attribute(:replicated) || @fully_replicated
+      self.current_shard = @slaves_list.shift.to_sym
+      @slaves_list << self.current_shard
     else
       self.current_shard = :master
     end
-
+    
     sql = select_connection().send(method, *args, &block)     
     self.current_shard = old_shard
     @using_enabled = nil

data/spec/config/shards.yml

@@ -12,7 +12,6 @@ octopus:
       database: octopus_shard1
       encoding: unicode
     
-    #TODO - FIX THIS
     sqlite_shard:
       adapter: sqlite3
       database: /tmp/database.sqlite3
@@ -62,6 +61,8 @@ production_raise_error:
         
 production_replicated:
   replicated: true
+  fully_replicated: false
+  
   shards:
     slave1:
       adapter: mysql
@@ -81,9 +82,8 @@ production_replicated:
       database: octopus_shard5
 
 
-production_entire_replicated:
+production_fully_replicated:
   replicated: true
-  entire_replicated: true
   shards:
     slave1:
       adapter: mysql
@@ -128,7 +128,6 @@ octopus_rails:
       host: localhost
       database: octopus_shard5
 
-
 not_entire_sharded:
   entire_sharded: false
   shards:

data/spec/octopus/controller_spec.rb

@@ -9,10 +9,8 @@ describe "Rails Controllers" do
         render :nothing => true
       end
       
-      def select_shard
-        Octopus.using(:brazil) do
-          yield
-        end
+      def select_shard(&block)
+        Octopus.using(:brazil, &block)
       end
       
       def self._routes

data/spec/octopus/migration_spec.rb

@@ -78,7 +78,6 @@ describe Octopus::Migration do
     it "should run writes on master when you use replication" do
       using_enviroment :production_replicated do 
         migrating_to_version 10 do 
-          Cat.using(:master).find_by_name("Replication").should_not be_nil
           Cat.find_by_name("Replication").should be_nil
         end
       end
@@ -88,10 +87,8 @@ describe Octopus::Migration do
       using_enviroment :production_replicated do 
         migrating_to_version 11 do 
           [:slave4, :slave1, :slave2, :slave3].each do |sym|
-            Cat.using(sym).find_by_name("Slaves").should_not be_nil
+            Cat.find_by_name("Slaves").should_not be_nil
           end
-
-          Cat.using(:master).find_by_name("Slaves").should be_nil
         end
       end
     end

data/spec/octopus/model_spec.rb

@@ -36,6 +36,18 @@ describe Octopus::Model do
       User.create!(:name => 'oi')
       User.count.should == 1
     end
+    
+    it "should ensure that the connection will be cleaned" do
+      ActiveRecord::Base.connection.current_shard.should == :master
+      begin
+        Octopus.using(:canada) do 
+          raise "Some Exception"
+        end
+      rescue
+      end
+
+      ActiveRecord::Base.connection.current_shard.should == :master
+    end
 
     it "should allow creating more than one user" do
       User.using(:canada).create([{ :name => 'America User 1' }, { :name => 'America User 2' }])

data/spec/octopus/replication_specs.rb

@@ -1,100 +1,54 @@
 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
 describe "when the database is replicated" do
-  before(:each) do
-    Octopus.stub!(:env).and_return("production_replicated")
-    clean_connection_proxy()
-  end
-
   it "should send all writes/reads queries to master when you have a non replicated model" do
-    u = User.create!(:name => "Replicated")
-    User.count.should == 1
-    User.find(u.id).should == u
+    using_enviroment :production_replicated do 
+      u = User.create!(:name => "Replicated")
+      User.count.should == 1
+      User.find(u.id).should == u
+    end
   end
-  
+
   it "should send all writes queries to master" do
-    u = Cat.using(:slave4).create!(:name => "Slave Cat")    
-    u1 = Cat.using(:slave4).first
-    u1.name = "Miau"
-    u1.save()
-    Cat.using(:slave4).first.name.should == "Slave Cat"
-  end
-  
-  it "should allow to send some queries to a selected slave" do
-    Cat.using(:slave4).create!([{:name => "Slave Cat 1"}, {:name => "Slave Cat 2"}])  
-    Cat.using(:slave4).count.should == 2
+    using_enviroment :production_replicated do 
+      Cat.create!(:name => "Slave Cat")    
+      Cat.find_by_name("Slave Cat").should be_nil
+      Client.create!(:name => "Slave Client")
+      Client.find_by_name("Slave Client").should_not be_nil
+    end
   end
 
-  it "should send read queries to slaves, when you have a replicated model, using a round robin algorithm" do
-    u = Cat.create!(:name => "master")
-    c = Client.create!(:name => "client_master")
-
-    [:slave4, :slave1, :slave2, :slave3].each do |sym|
-      Cat.using(sym).create!(:name => "Replicated_#{sym}")
+  it "should allow to create multiple models on the master" do
+    using_enviroment :production_replicated do 
+      Cat.create!([{:name => "Slave Cat 1"}, {:name => "Slave Cat 2"}])  
+      Cat.find_by_name("Slave Cat 1").should be_nil
+      Cat.find_by_name("Slave Cat 2").should be_nil
     end
+  end
 
-    Client.find(:first).should_not be_nil
-    Cat.find(:first).name.should == "Replicated_slave1"
-    Client.find(:first).should_not be_nil
-    Cat.find(:first).name.should == "Replicated_slave2"
-    Client.find(:first).should_not be_nil
-    Cat.find(:first).name.should == "Replicated_slave3"
-    Client.find(:first).should_not be_nil
-    Cat.find(:first).name.should == "Replicated_slave4"
-    Client.find(:first).should_not be_nil
-    Cat.find(:first).name.should == "Replicated_slave1"
-    Client.find(:first).should_not be_nil
-
-    [:slave4, :slave1, :slave2, :slave3].each do |sym|
-      Cat.using(sym).find_by_name("master").should be_false
-    end
+  it "should send the count query to a slave" do
+    pending()
+    # using_enviroment :production_replicated do 
+    #       Cat.create!(:name => "Slave Cat")    
+    #       Cat.count.should == 0
+    #     end
   end
 end
 
 
 describe "when the database is replicated and the entire application is replicated" do
   before(:each) do
-    Octopus.stub!(:env).and_return("production_entire_replicated")
+    Octopus.stub!(:env).and_return("production_fully_replicated")
     clean_connection_proxy()
   end
-  
-  it "should send all writes queries to master" do
-    u = Client.using(:slave4).create!(:name => "Slave Client")    
-    u1 = Client.using(:slave4).first
-    u1.name = "Client"
-    u1.save()
-    Client.using(:slave4).first.name.should == "Slave Client"
-  end
-
-  it "should send read queries to slaves,to all models, using a round robin algorithm" do
-    u = Cat.create!(:name => "master")
-    c = Client.create!(:name => "client_master")
-
-    [:slave4, :slave1, :slave2, :slave3].each do |sym|
-      Cat.using(sym).create!(:name => "Replicated_#{sym}")
-    end
-
-    [:slave4, :slave1, :slave2, :slave3].each do |sym|
-      Client.using(sym).create!(:name => "Replicated_#{sym}")
-    end
-
-    Client.find(:first).name.should == "Replicated_slave1"
-    Cat.find(:first).name.should == "Replicated_slave2"
-    Client.find(:first).name.should == "Replicated_slave3"
-    Cat.find(:first).name.should == "Replicated_slave4"
-    Cat.find(:first).name.should == "Replicated_slave1"
-    Client.find(:first).name.should == "Replicated_slave2"
-    Cat.find(:first).name.should == "Replicated_slave3"
-    Client.find(:first).name.should == "Replicated_slave4"
-
 
-    [:slave4, :slave1, :slave2, :slave3].each do |sym|
-      Cat.using(sym).find_by_name("master").should be_false
-    end
-    
-    [:slave4, :slave1, :slave2, :slave3].each do |sym|
-      Client.using(sym).find_by_name("master").should be_false
+  it "should send all writes queries to master" do
+    using_enviroment :production_fully_replicated do 
+      Cat.create!(:name => "Slave Cat")    
+      Cat.find_by_name("Slave Cat").should be_nil
+      Client.create!(:name => "Slave Client")
+      Client.find_by_name("Slave Client").should be_nil
     end
-  end
+  end  
 end
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: ar-octopus
 version: !ruby/object:Gem::Version 
-  hash: 37
+  hash: 35
   prerelease: false
   segments: 
   - 0
   - 0
-  - 29
-  version: 0.0.29
+  - 30
+  version: 0.0.30
 platform: ruby
 authors: 
 - Thiago Pradi
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-07-30 00:00:00 -03:00
+date: 2010-08-14 00:00:00 -03:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -145,12 +145,8 @@ files:
 - Gemfile.lock
 - README.mkdn
 - Rakefile
-- VERSION
+- TODO.txt
 - ar-octopus.gemspec
-- doc/api.textile
-- doc/features.textile
-- doc/libraries.textile
-- doc/shards.yml
 - init.rb
 - lib/octopus.rb
 - lib/octopus/association.rb

data/VERSION

@@ -1 +0,0 @@
-0.0.1

data/doc/api.textile

@@ -1,98 +0,0 @@
-h1. Api Design
-
-* The API design should be simple as possible, so they will just have one method to change the shards, the method using, which will be different in the context that him get called. In controller, they will send all queries in the action or controller to a specified shard. So, if you want to get all of your application sharded, you need to do a before_filter in the application controller.
-* The other method will be sharded_by, which selects the shard using a attribute of the model. This method is just accessible from class method on ActiveRecord::Base.
-* The using method also could be called from models, as a scope, or receiving a block, that will run all the methods inside a specific shard.
-* Also, the using method could be called from migrations. For default, migrations should run only in the master database, if you want to run in a different database, you should specify where you wish that migration run. like this:
-<pre>
-# This will run in all, master and slaves
-  class AddNameFieldToUser < ActiveRecord::Migration
-    using(:all)
-    #or
-    using_all
-    
-    def self.up
-      add_column  :users, :name, :string
-    end
-
-    def self.down
-      remove_column  :users, :name, :string    
-    end
-  end
-  
-  
-# This will run just in all slaves
-  class AddNameFieldToUser < ActiveRecord::Migration
-    using(:slaves)
-    or 
-    using_slaves()
-
-    def self.up
-      add_column  :users, :name, :string
-    end
-
-    def self.down
-      remove_column  :users, :name, :string    
-    end
-  end
-  
-# This will run in all shards, slaves or not
-  class AddNameFieldToUser < ActiveRecord::Migration
-    using(:shards)
-
-    def self.up
-      add_column  :users, :name, :string
-    end
-
-    def self.down
-      remove_column  :users, :name, :string    
-    end
-  end
-  
-# This will run in specific shard
-  class AddNameFieldToUser < ActiveRecord::Migration
-    using(:canada)
-
-    def self.up
-      add_column  :users, :name, :string
-    end
-
-    def self.down
-      remove_column  :users, :name, :string    
-    end
-  end
-  
-# This will run in the specified shards, this accepts two or more shards.
-  class AddNameFieldToUser < ActiveRecord::Migration
-    using(:canada, :brazil)
-
-    def self.up
-      add_column  :users, :name, :string
-    end
-
-    def self.down
-      remove_column  :users, :name, :string    
-    end
-  end
-</pre>
-* The default behavior on finds will be: if you find a object from a shard, like: 
-
-# This will find all users in brazil shard and iterate over it, saving all of them in the brazil shard
-<pre>
-User.using(:brazil).find(:all).each do |user|
-  user.name = "Brazil"
-  user.save()
-end
-</pre>
-
-# This will find all users in brazil shard and iterate over it, saving all of them in the canada shard
-<pre>
-User.using(:brazil).find(:all).each do |user|
-  user.name = "Brazil"
-  user.using(:canada).save()
-end
-</pre>
-
-
-
-

data/doc/features.textile

@@ -1,74 +0,0 @@
-h1. Octopus Features List:
-
-* Support for replicated databases, with one master and multiple slaves. Multi-master support could be added later. The writes will be sended to master and the reads to slave, but this could be modified.
-* Support database sharding, with a nice and clean syntax like:
-  <pre> 
-  User.using(:awesome_shard).all() or User.using_awesome_shard.all()
-  
-  class User < ActiveRecord::Base
-    sharded_by :code
-    
-    def awesome_method
-      #All queries inside the block will go to awesome_shard
-      using(:awesome_shard) do
-        Foo.all
-        Bar.all
-      end
-    end
-  end
-  
-  class ApplicationController < ActionController::Base
-    around_filter :select_shard
-    
-    def select_shard(&block)
-      using(current_user.city) do
-        yield
-      end
-    end
-  end 
-  </pre>
-* Running migrations between shards, example:
-  <pre>
-  class MyAwesomeMigration < ActiveRecord::Migration
-    using :awesome_shard
-    
-    def self.up
-      create_table :users do |t|
-        t.string :name
-      end
-    end
-
-    def self.down
-      drop_table :users
-    end
-  end
-  </pre>
-* The sharding config will be separated from the database.yml file, given a more cleaner and nice configuration file.
-* An initial generator will be integrated in to the project, to help people using octopus, this will generate the config file: db/shards.yml, and a initializer to help the configuration. The example of this configuration is on this directory.
-* Like the others implementation, the sharding will be selected using a Proxy class for ActiveRecord, where the class will require the connection, and the query will be executed in the selected shard.
-* After this essentials feature, I will add a task to capistrano that allows you to generate the configuration and start the replication with one line command, reading the db/shards.yml configuration and running commands on the servers.
-* In replication, all slaves will be trated as shards, but you will have to specify what should be used as slave. For default, replication will balance your read queries between the slaves, and your writes queries will goes to master. if you don't want this feature, just set the load_balancing to false, and specify what queries you want to goes to slaves, and what slave. with replication, you will have methods like:
-<pre>
-# This sends the queries to a random shard (support just read queries, not writes)
-User.using_slaves().all()
-# This sends all queries to master(read/write)
-User.using_master().all()
-# Same thing, but all users queries will be sent to master
-class User < ActiveRecord::Base
-  using_master()
-end
-# Same thing, but all read queries will be sent to slaves
-class User < ActiveRecord::Base
-  using_slaves()
-end
-
-#or 
-
-class User < ActiveRecord::Base
-  using(:master)
-end
-
-#Tip: you cannot name a shard as master or slaves, they are reserved words used for replication.
-</pre>
-
-* If you have multiple slaves, the load balancing will be did using round robin algorithm, sending the queries to the databases available. (This isn't the better algorithm, but it's easy to implement and works)

data/doc/libraries.textile

@@ -1,70 +0,0 @@
-h1. Masochism
-
-p. Features:
-* Support Multiple Database Support
-* The config is stored in database.yml
-* You could have a master and multiples slaves, but you couldn't change on the fly the shard. Ex: User.using(:awesome_shard)
-
-p. Pros:
-* Easy to use
-* Test Coverage
-
-p. Cons:
-* Outdated (Lastest commit in January 12, 2009)
-* Don't support running migrations on different shards.
-* Don't support changing the shard on the fly
-
-
-h1. DataFabric 
-
-p. Features:
-* Support Multiple Database Support
-* The config is stored in database.yml
-* You could have data that are just sharded, not replicated.
-* Support on the fly sharding selecting, with blocks
-
-p. Pros:
-* Easy to use and config
-* Test Coverage
-* Support just sharded, not replicated data
-
-p. Cons:
-* Don't support running migrations between shards.
-* Don't support changing the sharding on the model, example: User.using(:awesome_shard)
-
-
-
-h1. DbCharmer
-
-p. Features:
-* Support Multiple Database Support
-* The config is stored in database.yml
-* You could have a master and multiples slaves
-* You could change the shard on the fly, with this syntax: User.switch_connection_to(:awesome_shard)
-* You could run migrations over shards
-* You could specify configurations of shards using ruby code
-
-
-p. Pros:
-* Support replication and sharding
-* Support migrations between shards
-* Supports on the Fly changing on the model
-
-p. Cons:
-* Didn't have test coverage in the plugin project, the tests are in another project.
-* Weird and complicated syntax.
-* Code are much more complicated than in the others project.
-
-
-
-h1. DataMapper Sharding 
-
-p. Features:
-* Support Multiple Database Support
-* Syntax: DataMapper.setup(:external, 'mysql://someother_host/dm_core_test'); repository(:external) { Person.first }
-
-h1. Multi-DB (http://github.com/schoefmax/multi_db)
-
-p. Features:
-* Support replication, with multiple slaves
-* Load balancing between slaves

data/doc/shards.yml

@@ -1,76 +0,0 @@
-# The master database is the database settend in config/database.yml
-# This supports both sharding and replication, you could select the shards, and for default just replicated
-# database will be used, the master database is in database.yml and the slave will only be awesome_slave.
-# Setting Load balancing for default will be true, octopus will send all your read queries to slaves, and writes queries to 
-# master. if you just want some queries to send to slaves, set it to false, and use the sintax User.using(:slave).
-production:
-  replicated: true
-    
-  shards:
-    awesome_slave:
-      adapter: mysql
-      username: user
-      password: pass
-      database: awesome_slave
-      host: 192.321.321.21
-   
-    not_slave:
-      adapter: mysql
-      username: user
-      password: pass
-      database: awesome_slave
-      host: 192.321.321.18
-  
-# This is another example, not replicated.
-production:
-  shards:
-    us:
-      adapter: mysql
-      username: user
-      password: pass
-      database: shard1
-      host: 192.321.321.19
-    canada:
-      adapter: mysql
-      username: user
-      password: pass
-      database: shard1
-      host: 192.321.321.17
-    brazil:
-      adapter: mysql
-      username: user
-      password: pass
-      database: shard1
-      host: 192.321.321.90
-      
-# This is a example using a group of shards:
-  production:  
-    shards:
-      history_shards:
-        aug2009:
-          adapter: mysql
-          username: user
-          password: pass
-          database: aug2009
-          host: 192.321.321.21
-        oct2009:
-          adapter: mysql
-          username: user
-          password: pass
-          database: oct2009
-          host: 192.321.321.18
-
-      country_shards:
-        brazil:
-          adapter: mysql
-          username: user
-          password: pass
-          database: brazil
-          host: 192.321.321.21
-          slave: true
-        canada:
-          adapter: mysql
-          username: user
-          password: pass
-          database: canada
-          host: 192.321.321.18