activerecord-turntable 1.0.0

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.gitignore

@@ -0,0 +1,25 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.DS_Store
+doc
+.yardoc
+
+# For TextMate
+#*.tmproj
+#tmtags
+
+# For emacs:
+*~
+\#*
+.\#*
+
+# For vim:
+*.swp
+
+# For redcar:
+#.redcar
+
+# For rubinius:
+#*.rbc

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format=documentation
+

data/Gemfile

@@ -0,0 +1,25 @@
+source :rubygems
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+gem 'activerecord', ">= 3.1.0"
+gem 'activesupport', ">=3.1.0"
+gem "sql_tree", "=0.2.0"
+gem "bsearch", "~>1.5"
+gem "httpclient"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development, :test do
+  gem "rake", "~>0.9.2"
+  gem "rspec"
+  gem "rr"
+  gem "mysql2"
+  gem "fabrication"
+  gem "faker"
+  gem "activerecord-import"
+  gem "pry"
+  gem "guard-rspec"
+  gem "growl"
+end

data/Guardfile

@@ -0,0 +1,9 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'rspec', :version => 2 do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end
+

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Drecom Co.,Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,290 @@
+= ActiveRecord::Turntable
+
+ActiveRecord::Turntable is a database sharding extension for ActiveRecord.
+
+== Dependencies
+activerecord(>=3.0.0)
+
+== Supported Database
+Currently supports mysql only.
+
+== Installation
+
+Add 'activerecord-turntable' to Gemfile:
+
+    gem 'activerecord-turntable', '1.0.0'
+
+Run a bundle install:
+
+    bundle install
+
+Run install generator:
+
+    bundle exec rails g active_record:turntable:install
+
+generator creates Rails.root/config/turntable.yml
+
+== ActiveRecord::Turntable terms
+
+*Shard
+*Cluster - a cluster of shards.
+*Master - default ActiveRecord::Base's connection
+*Sequencer - a module that creates global sequence number
+
+== Example
+
+=== Example Databases Structure
+one main database(default ActiveRecord::Base connection) and
+three user databases sharded by user_id.
+
+                         +-------+
+                         |  App  |
+                         +-------+
+                             |
+       +---------+-----------+---------+---------+
+       |         |           |         |         |
+  +--------+ +---------+ +-------+ +-------+ +-------+
+  | Master | |Sequencer| |UserDB1| |UserDB2| |UserDB3|
+  +--------+ +---------+ +-------+ +-------+ +-------+
+
+=== Example Configuration
+
+Edit turntable.yml and database.yml. See below example config.
+
+example turntable.yml:
+
+    development:
+      clusters:
+        user_cluster: # <-- cluster name
+          algorithm: range # <-- only range or range_bsearch sharding is supported currently.
+          shards:
+            - connection: user_shard_1 # <-- shard name
+              less_than: 100           # <-- shard range(like mysql partitioning)
+            - connection: user_shard_2
+              less_than: 200
+            - connection: user_shard_3
+              less_than: 2000000000
+
+database.yml:
+    connection_spec: &spec
+      adapter: mysql2
+      encoding: utf8
+      reconnect: false
+      pool: 5
+      username: root
+      password: root
+      socket: /tmp/mysql.sock
+
+    development:
+      <<: *spec
+      database: sample_app_development
+      seq: # <-- sequence definition
+        user_seq_1:
+          <<: *spec
+          database: sample_app_user_seq_development
+      shards: # <-- shards definition
+        user_shard_1:
+          <<: *spec
+          database: sample_app_user1_development
+        user_shard_2:
+          <<: *spec
+          database: sample_app_user2_development
+        user_shard_3:
+          <<: *spec
+          database: sample_app_user3_development
+
+
+=== Migration Example
+
+Generate a model:
+
+  bundle exec rails g model user name:string
+
+Edit migration file:
+
+  class CreateUsers < ActiveRecord::Migration
+    clusters :user_cluster # <-- the cluster that executes migration
+
+    def change
+      create_table :users do |t|
+        t.string :name
+        t.timestamps
+      end
+      create_sequence_for(:users)
+    end
+  end
+
+then please execute rake tasks:
+
+  bundle exec rake db:create
+  bundle exec rake db:migrate
+
+Those rake tasks would be executed to shards too.
+
+=== Model Example
+add turntable [shard_key_name] to the model class:
+
+  class User < ActiveRecord::Base
+    turntable :user_cluster, :id
+    sequencer
+    has_one :status
+  end
+
+  class Status < ActiveRecord::Base
+    turntable :user_cluster, :user_id
+    sequencer
+    belongs_to :user
+  end
+
+== Usage
+
+=== Create
+    > User.create(name: "hoge")
+      (0.0ms) [Shard: user_seq_1] BEGIN
+      (0.3ms) [Shard: user_seq_1] UPDATE `users_id_seq` SET id=LAST_INSERT_ID(id+1)
+      (0.8ms) [Shard: user_seq_1] COMMIT
+      (0.1ms) [Shard: user_seq_1] SELECT LAST_INSERT_ID()
+      (0.1ms) [Shard: user_shard_1] BEGIN
+    [ActiveRecord::Turntable] Sending method: insert, sql: #<Arel::InsertManager:0x007f8503685b48>, shards: ["user_shard_1"]
+      SQL (0.8ms) [Shard: user_shard_1] INSERT INTO `users` (`created_at`, `id`, `name`, `updated_at`) VALUES ('2012-04-10 03:59:42', 2, 'hoge', '2012-04-10 03:59:42')
+      (0.4ms) [Shard: user_shard_1] COMMIT
+    => #<User id: 2, name: "hoge", created_at: "2012-04-10 03:59:42", updated_at: "2012-04-10 03:59:42">
+
+=== Retrieve
+    > user = User.find(2)
+    [ActiveRecord::Turntable] Sending method: select_all, sql: #<Arel::SelectManager:0x007f850466e668>, shards: ["user_shard_1"]
+      User Load (0.3ms) [Shard: user_shard_1] SELECT `users`.* FROM `users` WHERE `users`.`id` = 2 LIMIT 1
+    => #<User id: 2, name: "hoge", created_at: "2012-04-10 03:59:42", updated_at: "2012-04-10 03:59:42">
+
+=== Update
+    > user.update_attributes(name: "hogefoo")
+      (0.1ms) [Shard: user_shard_1] BEGIN
+    [ActiveRecord::Turntable] Sending method: update, sql: UPDATE `users` SET `name` = 'hogefoo', `updated_at` = '2012-04-10 04:07:52' WHERE `users`.`id` = 2, shards: ["user_shard_1"]
+      (0.3ms) [Shard: user_shard_1] UPDATE `users` SET `name` = 'hogefoo', `updated_at` = '2012-04-10 04:07:52' WHERE `users`.`id` = 2
+      (0.8ms) [Shard: user_shard_1] COMMIT
+    => true
+
+=== Delete
+    > user.destroy
+      (0.2ms) [Shard: user_shard_1] BEGIN
+    [ActiveRecord::Turntable] Sending method: delete, sql: #<Arel::DeleteManager:0x007f8503677ea8>, shards: ["user_shard_1"]
+      SQL (0.3ms) [Shard: user_shard_1] DELETE FROM `users` WHERE `users`.`id` = 2
+      (1.7ms) [Shard: user_shard_1] COMMIT
+    => #<User id: 2, name: "hogefoo", created_at: "2012-04-10 03:59:42", updated_at: "2012-04-10 04:07:52">
+
+=== Count
+
+    > User.count
+    [ActiveRecord::Turntable] Sending method: select_value, sql: #<Arel::SelectManager:0x007f9e82ccebb0>, shards: ["user_shard_1", "user_shard_2", "user_shard_3"]
+       (0.8ms) [Shard: user_shard_1] SELECT COUNT(*) FROM `users`
+       (0.3ms) [Shard: user_shard_2] SELECT COUNT(*) FROM `users`
+       (0.2ms) [Shard: user_shard_3] SELECT COUNT(*) FROM `users`
+    => 1
+
+=== Sequencer
+Sequencer provides generating global IDs.
+
+Add below to the migration:
+
+      create_sequence_for(:users) # <-- this line creates sequence table
+
+This will creates sequence table.
+
+Next, add sequencer to the model:
+
+  class User < ActiveRecord::Base
+    turntable :id
+    sequencer # <-- this line enables sequencer module
+    has_one :status
+  end
+
+
+=== Transaction
+    > user = User.find(2)
+    > user3 = User.create(name: "hoge3")
+    > User.shards_transaction([user, user3]) do
+    >   user.name  = "hogehoge"
+    >   user3.name = "hogehoge3"
+    >   user.save!
+    >   user3.save!
+    > end
+
+=== Send query to a specific shard.
+use with_shard method:
+
+    AR::Base.connection.with_shard(shard1) do
+      # something queries to shard1
+    end
+
+To access shard objects, use below:
+* AR::Base.connection.shards # \\{shard_name => shard_obj,....}
+* AR::Base#turntable_shard
+* AR::Base.connection.select_shard(shard_key_value) #=> shard
+
+=== Send query to all shards
+use with_all method:
+
+  User.connection.with_all do
+    User.order("created_at DESC").limit(3).all
+  end
+
+
+=== Migration
+If you specify cluster or shard, migration will be executed to the cluster(or shard) and master database.
+
+to specify cluster:
+
+    class CreateUsers < ActiveRecord::Migration
+      clusters :user_cluster
+      ....
+    end
+
+to specify shard:
+
+    class CreateUsers < ActiveRecord::Migration
+      shards :user_shard_01
+      ....
+    end
+
+if you would like to execute a migration to all databases in the environments, use :all symbol:
+
+    class CreateUsers < ActiveRecord::Migration
+      clusters :all
+      ....
+    end
+
+== Connection Management
+Rails's ConnectionManagement middleware keeps ActiveRecord's connection during the process is alive, but Turntable keeps more connections.
+This may cause flooding max connections on your database. So, we made a middleware that disconnects on each request.
+To use turntable's ConnectionManagement middleware, add below line to your initializer.
+
+      app.middleware.swap ActiveRecord::ConnectionAdapters::ConnectionManagement, ActiveRecord::Turntable::Rack::ConnectionManagement
+
+== Inability
+* Queries includes "ORDER BY", "GROUP BY" and "LIMIT" clauses cannot be distributed.
+* "has many through" and "habtm" relationships causes wrong results. ex) User-Friend-User relation
+
+== Thanks to
+ConnectionProxy, Distributed Migration implementation is inspired by Octopus and DataFabric.
+
+== LICENSE
+Copyright (c) 2012 Drecom Co.,Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,101 @@
+require "bundler/gem_tasks"
+require 'rubygems'
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+namespace :turntable do
+  namespace :db do
+    task :rails_env do
+      unless defined? RAILS_ENV
+        RAILS_ENV = ENV['RAILS_ENV'] ||= 'test'
+      end
+    end
+
+    task :load_config => :rails_env do
+      require 'active_record'
+      ActiveRecord::Base.configurations = YAML.load_file(File.join(File.dirname(__FILE__), 'spec/config/database.yml'))
+    end
+
+    desc "create turntable test database"
+    task :create => :load_config do
+      database_configs = [ActiveRecord::Base.configurations[RAILS_ENV]] + ActiveRecord::Base.configurations[RAILS_ENV]["shards"].values + ActiveRecord::Base.configurations[RAILS_ENV]["seq"].values
+      database_configs.each do |dbconf|
+        %x{ echo "CREATE DATABASE #{dbconf["database"]}" | mysql -u #{dbconf["username"]} -p#{dbconf["password"]} -h #{dbconf["host"]} }
+      end
+    end
+
+    desc "migrate turntable test tables"
+    task :migrate => :load_config do
+      ActiveRecord::Base.establish_connection RAILS_ENV
+      require 'active_record/turntable'
+      ActiveRecord::Base.send(:include, ActiveRecord::Turntable)
+      ActiveRecord::ConnectionAdapters::SchemaStatements.send(:include, Turntable::Migration::SchemaStatementsExt)
+      database_configs = [ActiveRecord::Base.configurations[RAILS_ENV]] + ActiveRecord::Base.configurations[RAILS_ENV]["shards"].values + ActiveRecord::Base.configurations[RAILS_ENV]["seq"].values
+
+      database_configs.each do |dbconf|
+        ActiveRecord::Base.establish_connection dbconf
+
+        ActiveRecord::Base.connection.create_table :users do |t|
+          t.string :nickname
+          t.string :thumbnail_url
+          t.datetime :joined_at
+          t.datetime :deleted_at
+          t.timestamps
+        end
+        ActiveRecord::Base.connection.create_sequence_for :users
+
+        ActiveRecord::Base.connection.create_table :user_statuses do |t|
+          t.belongs_to :user, :null => false
+          t.integer    :hp,   :null => false, :default => 0
+          t.integer    :mp,   :null => false, :default => 0
+          t.datetime   :deleted_at, :default => nil
+          t.timestamps
+        end
+        ActiveRecord::Base.connection.create_sequence_for :user_statuses
+
+        ActiveRecord::Base.connection.create_table :cards do |t|
+          t.string :name, :null => false
+          t.integer :hp,  :null => false, :default => 0
+          t.integer :mp,  :null => false, :default => 0
+          t.timestamps
+        end
+        ActiveRecord::Base.connection.create_table :archived_cards do |t|
+          t.string :name, :null => false
+          t.integer :hp,  :null => false, :default => 0
+          t.integer :mp,  :null => false, :default => 0
+          t.timestamps
+          t.datetime :deleted_at, :default => nil
+        end
+
+        ActiveRecord::Base.connection.create_table :cards_users do |t|
+          t.belongs_to :card,    :null => false
+          t.belongs_to :user,    :null => false
+          t.timestamps
+        end
+        ActiveRecord::Base.connection.create_sequence_for :cards_users
+
+        ActiveRecord::Base.connection.create_table :archived_cards_users do |t|
+          t.belongs_to :card,    :null => false
+          t.belongs_to :user,    :null => false
+          t.timestamps
+          t.datetime   :deleted_at, :default => nil
+        end
+        ActiveRecord::Base.connection.create_sequence_for :archived_cards_users
+      end
+    end
+
+    desc "drop turntable test database"
+    task :drop => :load_config do
+      database_configs = [ActiveRecord::Base.configurations[RAILS_ENV]] + ActiveRecord::Base.configurations[RAILS_ENV]["shards"].values + ActiveRecord::Base.configurations[RAILS_ENV]["seq"].values
+      database_configs.each do |dbconf|
+        %x{ echo "DROP DATABASE #{dbconf["database"]}" | mysql -u #{dbconf["username"]} -p#{dbconf["password"]} -h #{dbconf["host"]} }
+      end
+    end
+
+    task :reset => ["turntable:db:drop", "turntable:db:create", "turntable:db:migrate"]
+  end
+end