ar-octopus 0.0.1

data/README.mkdn

@@ -0,0 +1,49 @@
+<h1> Octopus  - Easy Database Sharding for ActiveRecord</h1>
+
+<p> Octopus is a better way to do Database Sharding in ActiveRecord. Sharding allows multiple databases in the same rails application. While there are several projects that implement Sharding (e.g. DbCharmer, DataFabric), each project has its own limitations. The main goal of octopus project is to provide a nice and clean way of doing Database Sharding.</p>
+
+<h2>Feature list: </h2>
+<p> The design of the api is made to be simple as possible. Octopus is focusing in the end user, giving the power of multiple databases, but with reliable code and flexibility. Octopus is focused on Rails 3,  but will be soon compatible with Rails 2.x.</p>
+
+<p> Octopus supports: </p>
+
+- Sharding (with multiple shards, and grouped shards).
+- Replication (Master/slave support, with multiple slaves).
+- Moving data between shards with migrations.
+- Tools to manage database configurations. (soon)
+
+<p> To see the complete list of features and syntax, please check out our <a href="http://wiki.github.com/tchandy/octopus/"> Wiki</a> 
+  
+<h2>Thanks</h2>
+
+This project is sponsored by the <a href="http://www.rubysoc.org">Ruby Summer of Code</a>,
+and my mentors <a href="http://github.com/mperham">Mike Perham</a> and <a href="http://github.com/amitagarwal">Amit Agarwal</a>.
+
+
+<h3>If you are writing code that smells using octopus, the EVIL Octopus is going behind you, be careful.</h3>
+<pre>
+                                _________________
+                      ___       | I'm so EVIL!  |
+                   .-'   `'.    |______________ |
+                  /         \   /
+                  |         ;  /
+                  |         | /         ___.--,
+         _.._     |0) ~ (0) |    _.---'`__.-( (_.
+  __.--'`_.. '.__.\    '--. \_.-' ,.--'`     `""`
+ ( ,.--'`   ',__ /./;   ;, '.__.'`    __
+ _`) )  .---.__.' / |   |\   \__..--""  """--.,_
+`---' .'.''-._.-'`_./  /\ '.  \ _.-~~~````~~~-._`-.__.'
+      | |  .' _.-' |  |  \  \  '.               `~---`
+       \ \/ .'     \  \   '. '-._)
+        \/ /        \  \    `=.__`~-.
+        / /\         `) )    / / `"".`\
+  , _.-'.'\ \        / /    ( (     / /
+   `--~`   ) )    .-'.'      '.'.  | (
+          (/`    ( (`          ) )  '-;
+           `      '-;         (-'
+</pre>
+
+
+<h2>Copyright</h2>
+
+Copyright (c) 2010 Thiago Pradi, released under the MIT license.

data/Rakefile

@@ -0,0 +1,106 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), 'lib'))
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "ar-octopus"
+    gem.summary = "Easy Database Sharding for ActiveRecord"
+    gem.description = "This gem allows you to use sharded databases with ActiveRecord. this also provides a interface for replication and for running migrations with multiples shards."
+    gem.email = "tchandy@gmail.com"
+    gem.homepage = "http://github.com/tchandy/octopus"
+    gem.authors = ["Thiago Pradi", "Mike Perham", "Amit Agarwal"]
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    gem.version = "0.0.1"
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "octopus #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+namespace :db do
+  desc 'Build the MySQL test databases'
+  task :build_databases do
+    mysql_user = ENV['MYSQL_USER'] || "root"
+    postgres_user = ENV['POSTGRES_USER'] || "postgres"
+    (1..5).each do |idx|
+      %x( echo "create DATABASE octopus_shard#{idx} DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_unicode_ci " | mysql --user=#{mysql_user})
+    end
+    
+    %x( createdb -E UTF8 -U #{postgres_user} octopus_shard1 )
+  end
+
+  desc 'Drop the MySQL test databases'
+  task :drop_databases do
+    mysql_user = ENV['MYSQL_USER'] || "root"
+    postgres_user = ENV['POSTGRES_USER'] || "postgres"
+    (1..5).each do |idx|
+      %x( mysqladmin --user=#{mysql_user} -f drop octopus_shard#{idx} )
+    end
+    
+    %x( dropdb -U #{postgres_user} octopus_shard1 )
+  end
+
+  desc 'Create tables on mysql databases'
+  task :create_tables do
+    Dir.chdir(File.expand_path(File.dirname(__FILE__) + "/spec"))
+    require "database_connection"
+    require "octopus"
+    [:master, :brazil, :canada, :russia, :alone_shard, :postgresql_shard].each do |shard_symbol|
+      ActiveRecord::Base.using(shard_symbol).connection.create_table(:users) do |u|
+        u.string :name
+      end
+      
+      ActiveRecord::Base.using(shard_symbol).connection.create_table(:clients) do |u|
+        u.string :country
+        u.string :name
+      end
+      
+      ActiveRecord::Base.using(shard_symbol).connection.create_table(:cats) do |u|
+        u.string :name
+      end
+      
+      ActiveRecord::Base.using(shard_symbol).connection.create_table(:items) do |u|
+        u.string :name
+        u.integer :client_id
+      end
+      
+      ActiveRecord::Base.using(shard_symbol).connection.create_table(:schema_migrations) do |u|
+        u.string :version, :unique => true, :null => false
+      end
+    end
+  end
+  
+  desc 'Prepare the MySQL test databases'
+  task :prepare => [:drop_databases, :build_databases, :create_tables]
+end
+
+
+

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/doc/api.textile

@@ -0,0 +1,98 @@
+h1. Api Design
+
+* The API design should be simple as possible, so they will just have one method to change the shards, the method using, which will be different in the context that him get called. In controller, they will send all queries in the action or controller to a specified shard. So, if you want to get all of your application sharded, you need to do a before_filter in the application controller.
+* The other method will be sharded_by, which selects the shard using a attribute of the model. This method is just accessible from class method on ActiveRecord::Base.
+* The using method also could be called from models, as a scope, or receiving a block, that will run all the methods inside a specific shard.
+* Also, the using method could be called from migrations. For default, migrations should run only in the master database, if you want to run in a different database, you should specify where you wish that migration run. like this:
+<pre>
+# This will run in all, master and slaves
+  class AddNameFieldToUser < ActiveRecord::Migration
+    using(:all)
+    #or
+    using_all
+    
+    def self.up
+      add_column  :users, :name, :string
+    end
+
+    def self.down
+      remove_column  :users, :name, :string    
+    end
+  end
+  
+  
+# This will run just in all slaves
+  class AddNameFieldToUser < ActiveRecord::Migration
+    using(:slaves)
+    or 
+    using_slaves()
+
+    def self.up
+      add_column  :users, :name, :string
+    end
+
+    def self.down
+      remove_column  :users, :name, :string    
+    end
+  end
+  
+# This will run in all shards, slaves or not
+  class AddNameFieldToUser < ActiveRecord::Migration
+    using(:shards)
+
+    def self.up
+      add_column  :users, :name, :string
+    end
+
+    def self.down
+      remove_column  :users, :name, :string    
+    end
+  end
+  
+# This will run in specific shard
+  class AddNameFieldToUser < ActiveRecord::Migration
+    using(:canada)
+
+    def self.up
+      add_column  :users, :name, :string
+    end
+
+    def self.down
+      remove_column  :users, :name, :string    
+    end
+  end
+  
+# This will run in the specified shards, this accepts two or more shards.
+  class AddNameFieldToUser < ActiveRecord::Migration
+    using(:canada, :brazil)
+
+    def self.up
+      add_column  :users, :name, :string
+    end
+
+    def self.down
+      remove_column  :users, :name, :string    
+    end
+  end
+</pre>
+* The default behavior on finds will be: if you find a object from a shard, like: 
+
+# This will find all users in brazil shard and iterate over it, saving all of them in the brazil shard
+<pre>
+User.using(:brazil).find(:all).each do |user|
+  user.name = "Brazil"
+  user.save()
+end
+</pre>
+
+# This will find all users in brazil shard and iterate over it, saving all of them in the canada shard
+<pre>
+User.using(:brazil).find(:all).each do |user|
+  user.name = "Brazil"
+  user.using(:canada).save()
+end
+</pre>
+
+
+
+

data/doc/features.textile

@@ -0,0 +1,74 @@
+h1. Octopus Features List:
+
+* Support for replicated databases, with one master and multiple slaves. Multi-master support could be added later. The writes will be sended to master and the reads to slave, but this could be modified.
+* Support database sharding, with a nice and clean syntax like:
+  <pre> 
+  User.using(:awesome_shard).all() or User.using_awesome_shard.all()
+  
+  class User < ActiveRecord::Base
+    sharded_by :code
+    
+    def awesome_method
+      #All queries inside the block will go to awesome_shard
+      using(:awesome_shard) do
+        Foo.all
+        Bar.all
+      end
+    end
+  end
+  
+  class ApplicationController < ActionController::Base
+    around_filter :select_shard
+    
+    def select_shard(&block)
+      using(current_user.city) do
+        yield
+      end
+    end
+  end 
+  </pre>
+* Running migrations between shards, example:
+  <pre>
+  class MyAwesomeMigration < ActiveRecord::Migration
+    using :awesome_shard
+    
+    def self.up
+      create_table :users do |t|
+        t.string :name
+      end
+    end
+
+    def self.down
+      drop_table :users
+    end
+  end
+  </pre>
+* The sharding config will be separated from the database.yml file, given a more cleaner and nice configuration file.
+* An initial generator will be integrated in to the project, to help people using octopus, this will generate the config file: db/shards.yml, and a initializer to help the configuration. The example of this configuration is on this directory.
+* Like the others implementation, the sharding will be selected using a Proxy class for ActiveRecord, where the class will require the connection, and the query will be executed in the selected shard.
+* After this essentials feature, I will add a task to capistrano that allows you to generate the configuration and start the replication with one line command, reading the db/shards.yml configuration and running commands on the servers.
+* In replication, all slaves will be trated as shards, but you will have to specify what should be used as slave. For default, replication will balance your read queries between the slaves, and your writes queries will goes to master. if you don't want this feature, just set the load_balancing to false, and specify what queries you want to goes to slaves, and what slave. with replication, you will have methods like:
+<pre>
+# This sends the queries to a random shard (support just read queries, not writes)
+User.using_slaves().all()
+# This sends all queries to master(read/write)
+User.using_master().all()
+# Same thing, but all users queries will be sent to master
+class User < ActiveRecord::Base
+  using_master()
+end
+# Same thing, but all read queries will be sent to slaves
+class User < ActiveRecord::Base
+  using_slaves()
+end
+
+#or 
+
+class User < ActiveRecord::Base
+  using(:master)
+end
+
+#Tip: you cannot name a shard as master or slaves, they are reserved words used for replication.
+</pre>
+
+* If you have multiple slaves, the load balancing will be did using round robin algorithm, sending the queries to the databases available. (This isn't the better algorithm, but it's easy to implement and works)

data/doc/libraries.textile

@@ -0,0 +1,70 @@
+h1. Masochism
+
+p. Features:
+* Support Multiple Database Support
+* The config is stored in database.yml
+* You could have a master and multiples slaves, but you couldn't change on the fly the shard. Ex: User.using(:awesome_shard)
+
+p. Pros:
+* Easy to use
+* Test Coverage
+
+p. Cons:
+* Outdated (Lastest commit in January 12, 2009)
+* Don't support running migrations on different shards.
+* Don't support changing the shard on the fly
+
+
+h1. DataFabric 
+
+p. Features:
+* Support Multiple Database Support
+* The config is stored in database.yml
+* You could have data that are just sharded, not replicated.
+* Support on the fly sharding selecting, with blocks
+
+p. Pros:
+* Easy to use and config
+* Test Coverage
+* Support just sharded, not replicated data
+
+p. Cons:
+* Don't support running migrations between shards.
+* Don't support changing the sharding on the model, example: User.using(:awesome_shard)
+
+
+
+h1. DbCharmer
+
+p. Features:
+* Support Multiple Database Support
+* The config is stored in database.yml
+* You could have a master and multiples slaves
+* You could change the shard on the fly, with this syntax: User.switch_connection_to(:awesome_shard)
+* You could run migrations over shards
+* You could specify configurations of shards using ruby code
+
+
+p. Pros:
+* Support replication and sharding
+* Support migrations between shards
+* Supports on the Fly changing on the model
+
+p. Cons:
+* Didn't have test coverage in the plugin project, the tests are in another project.
+* Weird and complicated syntax.
+* Code are much more complicated than in the others project.
+
+
+
+h1. DataMapper Sharding 
+
+p. Features:
+* Support Multiple Database Support
+* Syntax: DataMapper.setup(:external, 'mysql://someother_host/dm_core_test'); repository(:external) { Person.first }
+
+h1. Multi-DB (http://github.com/schoefmax/multi_db)
+
+p. Features:
+* Support replication, with multiple slaves
+* Load balancing between slaves

data/doc/shards.yml

@@ -0,0 +1,76 @@
+# The master database is the database settend in config/database.yml
+# This supports both sharding and replication, you could select the shards, and for default just replicated
+# database will be used, the master database is in database.yml and the slave will only be awesome_slave.
+# Setting Load balancing for default will be true, octopus will send all your read queries to slaves, and writes queries to 
+# master. if you just want some queries to send to slaves, set it to false, and use the sintax User.using(:slave).
+production:
+  replicated: true
+    
+  shards:
+    awesome_slave:
+      adapter: mysql
+      username: user
+      password: pass
+      database: awesome_slave
+      host: 192.321.321.21
+   
+    not_slave:
+      adapter: mysql
+      username: user
+      password: pass
+      database: awesome_slave
+      host: 192.321.321.18
+  
+# This is another example, not replicated.
+production:
+  shards:
+    us:
+      adapter: mysql
+      username: user
+      password: pass
+      database: shard1
+      host: 192.321.321.19
+    canada:
+      adapter: mysql
+      username: user
+      password: pass
+      database: shard1
+      host: 192.321.321.17
+    brazil:
+      adapter: mysql
+      username: user
+      password: pass
+      database: shard1
+      host: 192.321.321.90
+      
+# This is a example using a group of shards:
+  production:  
+    shards:
+      history_shards:
+        aug2009:
+          adapter: mysql
+          username: user
+          password: pass
+          database: aug2009
+          host: 192.321.321.21
+        oct2009:
+          adapter: mysql
+          username: user
+          password: pass
+          database: oct2009
+          host: 192.321.321.18
+
+      country_shards:
+        brazil:
+          adapter: mysql
+          username: user
+          password: pass
+          database: brazil
+          host: 192.321.321.21
+          slave: true
+        canada:
+          adapter: mysql
+          username: user
+          password: pass
+          database: canada
+          host: 192.321.321.18