data_fabric 1.0.0

data/CHANGELOG

@@ -0,0 +1 @@
+http://github.com/fiveruns/data_fabric/commits/master

data/Manifest

@@ -0,0 +1,79 @@
+CHANGELOG
+example/app/controllers/accounts_controller.rb
+example/app/controllers/application.rb
+example/app/controllers/figments_controller.rb
+example/app/helpers/accounts_helper.rb
+example/app/helpers/application_helper.rb
+example/app/helpers/figments_helper.rb
+example/app/models/account.rb
+example/app/models/figment.rb
+example/app/views/accounts/index.html.erb
+example/app/views/layouts/application.html.erb
+example/config/boot.rb
+example/config/database.yml
+example/config/environment.rb
+example/config/environments/development.rb
+example/config/environments/production.rb
+example/config/environments/test.rb
+example/config/initializers/inflections.rb
+example/config/initializers/mime_types.rb
+example/config/initializers/new_rails_defaults.rb
+example/config/routes.rb
+example/db/development.sqlite3
+example/db/migrate/20080702154628_create_accounts.rb
+example/db/migrate/20080702154820_create_figments.rb
+example/db/s0_development.sqlite3
+example/db/s0_test.sqlite3
+example/db/s1_development.sqlite3
+example/db/s1_test.sqlite3
+example/db/schema.rb
+example/db/test.sqlite3
+example/public/404.html
+example/public/422.html
+example/public/500.html
+example/public/dispatch.cgi
+example/public/dispatch.fcgi
+example/public/dispatch.rb
+example/public/favicon.ico
+example/public/images/rails.png
+example/public/javascripts/application.js
+example/public/javascripts/controls.js
+example/public/javascripts/dragdrop.js
+example/public/javascripts/effects.js
+example/public/javascripts/prototype.js
+example/public/robots.txt
+example/Rakefile
+example/script/about
+example/script/console
+example/script/dbconsole
+example/script/destroy
+example/script/generate
+example/script/performance/benchmarker
+example/script/performance/profiler
+example/script/performance/request
+example/script/plugin
+example/script/process/inspector
+example/script/process/reaper
+example/script/process/spawner
+example/script/runner
+example/script/server
+example/test/fixtures/accounts.yml
+example/test/functional/accounts_controller_test.rb
+example/test/integration/account_figments_test.rb
+example/test/test_helper.rb
+example/vendor/plugins/data_fabric/init.rb
+example/vendor/plugins/data_fabric/lib/data_fabric.rb
+init.rb
+lib/data_fabric/version.rb
+lib/data_fabric.rb
+Rakefile
+README.rdoc
+test/connection_test.rb
+test/database.yml
+test/database_test.rb
+test/shard_test.rb
+test/test_helper.rb
+test/thread_test.rb
+TESTING.rdoc
+TODO
+Manifest

data/README.rdoc

@@ -0,0 +1,108 @@
+= data_fabric
+
+DataFabric provides flexible database connection switching for ActiveRecord.
+
+We needed two features to scale our mysql database: application-level sharding and 
+master/slave replication.  Sharding is the process of splitting a dataset across many 
+independent databases.  This often happens based on geographical region (e.g. 
+craigslist) or category (e.g. ebay).  Replication provides a near-real-time copy 
+of a database which can be used for fault tolerance and to reduce load on the master 
+node.  Combined, you get a scalable database solution which does not require huge 
+hardware to scale to huge volumes.  Or: DPAYEIOB - don't put all your eggs in one 
+basket.  :-)
+
+== Installation
+
+  gem install data_fabric
+
+== How does it work?
+
+You describe the topology for your database infrastructure in your model(s).  
+Different models can use different topologies.
+
+  class MyHugeVolumeOfDataModel < ActiveRecord::Base
+    connection_topology :replicated => true, :shard_by => :city
+  end
+
+There are four supported modes of operation, depending on the options given to the 
+connection_topology method.  The plugin will look for connections in your 
+config/database.yml with the following convention:
+
+No connection topology:
+#{environment} - this is the default, as with ActiveRecord, e.g. "production"
+
+  connection_topology :replicated => true
+#{environment}_#{role} - no sharding, just replication, where role is "master" or "slave", e.g. "production_master"
+
+  connection_topology :shard_by => :city
+#{group}_#{shard}_#{environment} - sharding, no replication, e.g. "city_austin_production"
+
+  connection_topology :replicated => true, :shard_by => :city
+#{group}_#{shard}_#{environment}_#{role} - sharding with replication, e.g. "city_austin_production_master"
+
+
+When marked as replicated, all write and transactional operations for the model 
+go to the master, whereas read operations go to the slave.
+
+Since sharding is an application-level concern, your application must set the shard
+to use based on the current request or environment.  The current shard
+is set on a thread local variable.  For example, you can set the shard in an 
+ActionController around_filter based on the user as follows:
+
+  class ApplicationController < ActionController::Base
+    around_filter :select_shard
+
+    private
+    def select_shard(&block)
+      DataFabric.activate_shard(:city => @current_user.city, &block)
+    end
+  end
+
+
+== Warnings
+
+* Sharded models should never be placed in the session store or you will get "Shard not set" errors when the session is persisted.
+* ActiveRecord's allow_concurrency = true is NOT supported in this version of data_fabric.
+
+== Testing and Bug Reports
+
+If you think you've found a problem with data_fabric, please use the example application to reproduce the bug and send me the diff.  The example application is a stock Rails 2.1 application which uses data_fabric in the expected fashion.
+
+
+== Thanks to...
+
+Rick Olsen of Rails Core:: for the Masochism plugin, which showed me how to bend AR's connection handling to my will
+Bradley Taylor of RailsMachine:: for the advice to shard while at acts_as_conference
+FiveRuns:: for paying me to develop this code and allowing its release
+
+
+== Author
+
+Mike Perham <mperham@gmail.com>
+http://mikeperham.com
+
+
+== LICENSE:
+ 
+(The FiveRuns License)
+ 
+Copyright (c) 2008 FiveRuns Corporation
+ 
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+ 
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+ 
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,69 @@
+require 'rubygems'
+require 'echoe'
+
+require File.dirname(__FILE__) << "/lib/data_fabric/version"
+
+Echoe.new 'data_fabric' do |p|
+	p.version = DataFabric::Version::STRING
+	p.author = "Mike Perham"
+	p.email  = 'mperham@gmail.com'
+	p.project = 'fiveruns'
+	p.summary = 'Sharding and replication support for ActiveRecord 2.x'
+	p.url = "http://github.com/fiveruns/data_fabric"
+	p.dependencies = ['activerecord >=2.0.2']
+	p.include_rakefile = true
+end
+
+require 'rake/testtask'
+
+RAILS_ROOT=File.dirname(__FILE__)
+
+Rake::TestTask.new do |t|
+  t.test_files = FileList['test/*_test.rb']
+  t.verbose = true
+end
+
+task :pretest do
+	setup(false)
+end
+
+task :create_db do
+	setup(true)
+end
+
+task :changelog do
+	`git log | grep -v git-svn-id > History.txt`
+end
+
+def setup_connection
+  require 'active_record'
+  ENV['RAILS_ENV'] = 'test'
+
+  ActiveRecord::Base.configurations = { 'test' => { :adapter => 'mysql', :host => 'localhost', :database => 'mysql' } }
+  ActiveRecord::Base.establish_connection 'test'
+end
+
+def using_connection(&block)
+  ActiveRecord::Base.connection.instance_eval(&block)
+end
+
+def setup(create = false)
+  setup_connection
+
+  databases = %w( vr_austin_master vr_austin_slave vr_dallas_master vr_dallas_slave )
+  databases.each do |db|
+    using_connection do
+			if create
+				execute "drop database if exists #{db}"
+				execute "create database #{db}"
+			end
+      execute "use #{db}"
+      execute "drop table if exists the_whole_burritos"
+      execute "drop table if exists enchiladas"
+      execute "create table enchiladas (id integer not null auto_increment, name varchar(30) not null, primary key(id))"
+      execute "insert into enchiladas (id, name) values (1, '#{db}')"
+      execute "create table the_whole_burritos (id integer not null auto_increment, name varchar(30) not null, primary key(id))"
+      execute "insert into the_whole_burritos (id, name) values (1, '#{db}')"
+    end  
+  end
+end

data/TESTING.rdoc

@@ -0,0 +1,34 @@
+Because of what it does, data_fabric is difficult to test easily.
+
+There are two layers of tests: unit tests and integration tests.
+
+
+== Running the Unit Tests
+
+The unit tests test both with and without an actual database.  test/database_test.rb
+tests against a MySQL database.  The other unit tests mock AR so no actual database is 
+required.  You must customize test/database.yml to point to your MySQL database.  
+"rake pretest" will set up MySQL to run the tests but you will need to customize
+the connection settings in the Rakefile pretest task.
+
+
+== Running the Integration Tests
+
+data_fabric ships with an example application which shows an actual Rails application
+using data_fabric.  You can customize config/database.yml but by default it uses a local
+Sqlite3 database so there should be no customization necessary.
+
+Run the Rake task to prepare the application for testing.  This will copy the latest data_fabric code
+to vendor/plugins, delete existing databases and migrate all the databases from scratch.  You should
+then be able to run the example application's tests.
+
+cd example
+rake app:prepare
+rake test
+
+
+== Submitting Bugs
+
+If you think you've found a problem with data_fabric, please write a unit or integration test
+which reproduces the problem.  This should make tracking down and fixing the bug relatively
+easy and provides a regression test for future releases.

data/TODO

@@ -0,0 +1 @@
+Process or Documentation for how to migrate existing data to shards

data/data_fabric.gemspec

@@ -0,0 +1,166 @@
+
+# Gem::Specification for Data_fabric-1.0.0
+# Originally generated by Echoe
+
+--- !ruby/object:Gem::Specification 
+name: data_fabric
+version: !ruby/object:Gem::Version 
+  version: 1.0.0
+platform: ruby
+authors: 
+- Mike Perham
+autorequire: 
+bindir: bin
+
+date: 2008-07-08 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.0.2
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: echoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: Sharding and replication support for ActiveRecord 2.x
+email: mperham@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- CHANGELOG
+- lib/data_fabric/version.rb
+- lib/data_fabric.rb
+- README.rdoc
+- TODO
+files: 
+- CHANGELOG
+- example/app/controllers/accounts_controller.rb
+- example/app/controllers/application.rb
+- example/app/controllers/figments_controller.rb
+- example/app/helpers/accounts_helper.rb
+- example/app/helpers/application_helper.rb
+- example/app/helpers/figments_helper.rb
+- example/app/models/account.rb
+- example/app/models/figment.rb
+- example/app/views/accounts/index.html.erb
+- example/app/views/layouts/application.html.erb
+- example/config/boot.rb
+- example/config/database.yml
+- example/config/environment.rb
+- example/config/environments/development.rb
+- example/config/environments/production.rb
+- example/config/environments/test.rb
+- example/config/initializers/inflections.rb
+- example/config/initializers/mime_types.rb
+- example/config/initializers/new_rails_defaults.rb
+- example/config/routes.rb
+- example/db/development.sqlite3
+- example/db/migrate/20080702154628_create_accounts.rb
+- example/db/migrate/20080702154820_create_figments.rb
+- example/db/s0_development.sqlite3
+- example/db/s0_test.sqlite3
+- example/db/s1_development.sqlite3
+- example/db/s1_test.sqlite3
+- example/db/schema.rb
+- example/db/test.sqlite3
+- example/public/404.html
+- example/public/422.html
+- example/public/500.html
+- example/public/dispatch.cgi
+- example/public/dispatch.fcgi
+- example/public/dispatch.rb
+- example/public/favicon.ico
+- example/public/images/rails.png
+- example/public/javascripts/application.js
+- example/public/javascripts/controls.js
+- example/public/javascripts/dragdrop.js
+- example/public/javascripts/effects.js
+- example/public/javascripts/prototype.js
+- example/public/robots.txt
+- example/Rakefile
+- example/script/about
+- example/script/console
+- example/script/dbconsole
+- example/script/destroy
+- example/script/generate
+- example/script/performance/benchmarker
+- example/script/performance/profiler
+- example/script/performance/request
+- example/script/plugin
+- example/script/process/inspector
+- example/script/process/reaper
+- example/script/process/spawner
+- example/script/runner
+- example/script/server
+- example/test/fixtures/accounts.yml
+- example/test/functional/accounts_controller_test.rb
+- example/test/integration/account_figments_test.rb
+- example/test/test_helper.rb
+- example/vendor/plugins/data_fabric/init.rb
+- example/vendor/plugins/data_fabric/lib/data_fabric.rb
+- init.rb
+- lib/data_fabric/version.rb
+- lib/data_fabric.rb
+- Rakefile
+- README.rdoc
+- test/connection_test.rb
+- test/database.yml
+- test/database_test.rb
+- test/shard_test.rb
+- test/test_helper.rb
+- test/thread_test.rb
+- TESTING.rdoc
+- TODO
+- Manifest
+- data_fabric.gemspec
+has_rdoc: true
+homepage: http://github.com/fiveruns/data_fabric
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --inline-source
+- --title
+- Data_fabric
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - "="
+    - !ruby/object:Gem::Version 
+      version: "1.2"
+  version: 
+requirements: []
+
+rubyforge_project: fiveruns
+rubygems_version: 1.2.0
+specification_version: 2
+summary: Sharding and replication support for ActiveRecord 2.x
+test_files: 
+- test/connection_test.rb
+- test/database_test.rb
+- test/shard_test.rb
+- test/test_helper.rb
+- test/thread_test.rb