lookout-data_fabric 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +15 -0
  2. data/CHANGELOG +79 -0
  3. data/README.rdoc +125 -0
  4. data/Rakefile +93 -0
  5. data/TESTING.rdoc +30 -0
  6. data/example30/Gemfile +5 -0
  7. data/example30/Gemfile.lock +79 -0
  8. data/example30/README +256 -0
  9. data/example30/Rakefile +52 -0
  10. data/example30/app/controllers/accounts_controller.rb +22 -0
  11. data/example30/app/controllers/application_controller.rb +39 -0
  12. data/example30/app/controllers/figments_controller.rb +8 -0
  13. data/example30/app/helpers/application_helper.rb +2 -0
  14. data/example30/app/models/account.rb +3 -0
  15. data/example30/app/models/figment.rb +4 -0
  16. data/example30/app/views/accounts/index.html.erb +47 -0
  17. data/example30/app/views/layouts/application.html.erb +8 -0
  18. data/example30/config.ru +4 -0
  19. data/example30/config/application.rb +42 -0
  20. data/example30/config/boot.rb +13 -0
  21. data/example30/config/database.yml +21 -0
  22. data/example30/config/environment.rb +5 -0
  23. data/example30/config/environments/development.rb +26 -0
  24. data/example30/config/environments/production.rb +49 -0
  25. data/example30/config/environments/test.rb +35 -0
  26. data/example30/config/initializers/backtrace_silencers.rb +7 -0
  27. data/example30/config/initializers/inflections.rb +10 -0
  28. data/example30/config/initializers/mime_types.rb +5 -0
  29. data/example30/config/initializers/secret_token.rb +7 -0
  30. data/example30/config/initializers/session_store.rb +8 -0
  31. data/example30/config/locales/en.yml +5 -0
  32. data/example30/config/routes.rb +65 -0
  33. data/example30/db/migrate/20080702154628_create_accounts.rb +14 -0
  34. data/example30/db/migrate/20080702154820_create_figments.rb +14 -0
  35. data/example30/db/seeds.rb +7 -0
  36. data/example30/script/rails +6 -0
  37. data/example30/test/fixtures/accounts.yml +7 -0
  38. data/example30/test/functional/accounts_controller_test.rb +12 -0
  39. data/example30/test/integration/account_figments_test.rb +97 -0
  40. data/example30/test/performance/browsing_test.rb +9 -0
  41. data/example30/test/test_helper.rb +13 -0
  42. data/lib/data_fabric.rb +107 -0
  43. data/lib/data_fabric/connection_proxy.rb +176 -0
  44. data/lib/data_fabric/extensions.rb +39 -0
  45. data/lib/data_fabric/version.rb +5 -0
  46. data/test/connection_test.rb +142 -0
  47. data/test/database.yml +24 -0
  48. data/test/database.yml.mysql +36 -0
  49. data/test/database_test.rb +51 -0
  50. data/test/shard_test.rb +58 -0
  51. data/test/test_helper.rb +28 -0
  52. data/test/thread_test.rb +83 -0
  53. metadata +172 -0
@@ -0,0 +1,39 @@
1
+ require 'data_fabric/connection_proxy'
2
+
3
+ class ActiveRecord::ConnectionAdapters::ConnectionHandler
4
+ def clear_active_connections_with_data_fabric!
5
+ clear_active_connections_without_data_fabric!
6
+ DataFabric::ConnectionProxy.shard_pools.each_value { |pool| pool.release_connection }
7
+ end
8
+ alias_method_chain :clear_active_connections!, :data_fabric
9
+ end
10
+
11
+ module DataFabric
12
+ module Extensions
13
+ def self.included(model)
14
+ DataFabric.logger.info { "Loading data_fabric #{DataFabric::Version::STRING} with ActiveRecord #{ActiveRecord::VERSION::STRING}" }
15
+
16
+ # Wire up ActiveRecord::Base
17
+ model.extend ClassMethods
18
+ ConnectionProxy.shard_pools = {}
19
+ end
20
+
21
+ # Class methods injected into ActiveRecord::Base
22
+ module ClassMethods
23
+ def data_fabric(options)
24
+ DataFabric.logger.info { "Creating data_fabric proxy for class #{name}" }
25
+ pool_proxy = PoolProxy.new(ConnectionProxy.new(self, options))
26
+ klass_name = name
27
+ connection_handler.instance_eval do
28
+ if @class_to_pool
29
+ # Rails 3.2
30
+ @class_to_pool[klass_name] = pool_proxy
31
+ else
32
+ # <= Rails 3.1
33
+ @connection_pools[klass_name] = pool_proxy
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,5 @@
1
+ module DataFabric
2
+ module Version
3
+ STRING = '1.5.0'
4
+ end
5
+ end
@@ -0,0 +1,142 @@
1
+ require 'test_helper'
2
+ require 'active_record/connection_adapters/mysql2_adapter'
3
+
4
+ class PrefixModel < ActiveRecord::Base
5
+ data_fabric :prefix => 'prefix'
6
+ end
7
+
8
+ class ShardModel < ActiveRecord::Base
9
+ data_fabric :shard_by => :city
10
+ end
11
+
12
+ class TheWholeEnchilada < ActiveRecord::Base
13
+ data_fabric :prefix => 'fiveruns', :replicated => true, :shard_by => :city
14
+ end
15
+
16
+ class AdapterMock < ActiveRecord::ConnectionAdapters::AbstractAdapter
17
+ # Minimum required to perform a find with no results.
18
+ def columns(table_name, name=nil)
19
+ [ActiveRecord::ConnectionAdapters::Column.new('id', 0, :integer, false)]
20
+ end
21
+ def primary_key(name)
22
+ :id
23
+ end
24
+ def adapter_name
25
+ 'mysql2'
26
+ end
27
+ def select(sql, name=nil, bindings=nil)
28
+ []
29
+ end
30
+ def execute(sql, name=nil)
31
+ []
32
+ end
33
+ def tables
34
+ ["enchiladas", "the_whole_burritos"]
35
+ end
36
+ def table_exists?(name)
37
+ true
38
+ end
39
+ def last_inserted_id(result)
40
+ 1
41
+ end
42
+ def method_missing(name, *args)
43
+ raise ArgumentError, "#{self.class.name} missing '#{name}': #{args.inspect}"
44
+ end
45
+
46
+ def self.visitor_for(pool)
47
+ Arel::Visitors::MySQL.new(pool)
48
+ end
49
+ end
50
+
51
+ class RawConnection
52
+ def method_missing(name, *args)
53
+ puts "#{self.class.name} missing '#{name}': #{args.inspect}"
54
+ end
55
+ end
56
+
57
+ class ConnectionTest < MiniTest::Test
58
+ include FlexMock::TestCase
59
+
60
+ def test_should_install_into_arbase
61
+ assert PrefixModel.methods.map(&:to_s).include?('data_fabric')
62
+ end
63
+
64
+ def test_prefix_connection_name
65
+ setup_configuration_for PrefixModel, 'prefix_test'
66
+ assert_equal 'prefix_test', PrefixModel.connection.connection_name
67
+ end
68
+
69
+ def test_shard_connection_name
70
+ setup_configuration_for ShardModel, 'city_austin_test'
71
+ # ensure unset means error
72
+ assert_raises ArgumentError do
73
+ ShardModel.connection.connection_name
74
+ end
75
+ DataFabric.activate_shard(:city => 'austin', :category => 'art') do
76
+ assert_equal 'city_austin_test', ShardModel.connection.connection_name
77
+ end
78
+ # ensure it got unset
79
+ assert_raises ArgumentError do
80
+ ShardModel.connection.connection_name
81
+ end
82
+ end
83
+
84
+ def test_respond_to_connection_methods
85
+ setup_configuration_for ShardModel, 'city_austin_test'
86
+ DataFabric.activate_shard(:city => 'austin', :category => 'art') do
87
+ assert ShardModel.connection.respond_to?(:columns)
88
+ assert ShardModel.connection.respond_to?(:primary_key)
89
+ assert !ShardModel.connection.respond_to?(:nonexistent_method)
90
+ end
91
+ end
92
+
93
+ def test_respond_to_connection_proxy_methods
94
+ setup_configuration_for ShardModel, 'city_austin_test'
95
+ DataFabric.activate_shard(:city => 'austin', :category => 'art') do
96
+ assert ShardModel.connection.respond_to?(:with_master)
97
+ assert !ShardModel.connection.respond_to?(:nonexistent_method)
98
+ end
99
+ end
100
+
101
+ def test_enchilada
102
+ setup_configuration_for TheWholeEnchilada, 'fiveruns_city_dallas_test_slave'
103
+ setup_configuration_for TheWholeEnchilada, 'fiveruns_city_dallas_test_master'
104
+ DataFabric.activate_shard :city => :dallas do
105
+ assert_equal 'fiveruns_city_dallas_test_slave', TheWholeEnchilada.connection.connection_name
106
+
107
+ # Should use the slave
108
+ assert_raises ActiveRecord::RecordNotFound do
109
+ TheWholeEnchilada.find(1)
110
+ end
111
+
112
+ # Should use the master
113
+ mmmm = TheWholeEnchilada.new
114
+ mmmm.instance_variable_set(:@attributes, { 'id' => 1 })
115
+ assert_raises ActiveRecord::RecordNotFound do
116
+ mmmm.reload
117
+ end
118
+ # ...but immediately set it back to default to the slave
119
+ assert_equal 'fiveruns_city_dallas_test_slave', TheWholeEnchilada.connection.connection_name
120
+
121
+ # Should use the master
122
+ TheWholeEnchilada.transaction do
123
+ mmmm.save!
124
+ end
125
+ TheWholeEnchilada.verify_active_connections!
126
+ TheWholeEnchilada.clear_active_connections!
127
+ TheWholeEnchilada.clear_all_connections!
128
+ end
129
+
130
+ TheWholeEnchilada.verify_active_connections!
131
+ TheWholeEnchilada.clear_active_connections!
132
+ TheWholeEnchilada.clear_all_connections!
133
+ end
134
+
135
+ private
136
+
137
+ def setup_configuration_for(clazz, name)
138
+ flexmock(ActiveRecord::Base).should_receive(:mysql2_connection).and_return(AdapterMock.new(RawConnection.new))
139
+ ActiveRecord::Base.configurations ||= HashWithIndifferentAccess.new
140
+ ActiveRecord::Base.configurations[name] = HashWithIndifferentAccess.new({ :adapter => 'mysql2', :database => name, :host => 'localhost'})
141
+ end
142
+ end
@@ -0,0 +1,24 @@
1
+ # The unit tests make use of the data populated in these databases.
2
+ #
3
+ # Notes:
4
+ # - The database identifiers (e.g. "fiveruns_city_austin_test_master") MUST NOT
5
+ # be changed! Everything else may be changed.
6
+ # - The user defined for "fiveruns_city_austin_test_master" MUST have the
7
+ # privilege to create and drop databases and tables.
8
+
9
+
10
+ fiveruns_city_austin_test_master:
11
+ adapter: sqlite3
12
+ database: test/vr_austin_master.db
13
+
14
+ fiveruns_city_austin_test_slave:
15
+ adapter: sqlite3
16
+ database: test/vr_austin_slave.db
17
+
18
+ fiveruns_city_dallas_test_master:
19
+ adapter: sqlite3
20
+ database: test/vr_dallas_master.db
21
+
22
+ fiveruns_city_dallas_test_slave:
23
+ adapter: sqlite3
24
+ database: test/vr_dallas_slave.db
@@ -0,0 +1,36 @@
1
+ # The unit tests make use of the data populated in these databases.
2
+ #
3
+ # Notes:
4
+ # - The database identifiers (e.g. "fiveruns_city_austin_test_master") MUST NOT
5
+ # be changed! Everything else may be changed.
6
+ # - The user defined for "fiveruns_city_austin_test_master" MUST have the
7
+ # privilege to create and drop databases and tables.
8
+
9
+
10
+ fiveruns_city_austin_test_master:
11
+ adapter: mysql2
12
+ host: localhost
13
+ database: vr_austin_master
14
+ username: root
15
+ password:
16
+
17
+ fiveruns_city_austin_test_slave:
18
+ adapter: mysql2
19
+ host: localhost
20
+ database: vr_austin_slave
21
+ username: root
22
+ password:
23
+
24
+ fiveruns_city_dallas_test_master:
25
+ adapter: mysql2
26
+ host: localhost
27
+ database: vr_dallas_master
28
+ username: root
29
+ password:
30
+
31
+ fiveruns_city_dallas_test_slave:
32
+ adapter: mysql2
33
+ host: localhost
34
+ database: vr_dallas_slave
35
+ username: root
36
+ password:
@@ -0,0 +1,51 @@
1
+ require 'test_helper'
2
+
3
+ class TheWholeBurrito < ActiveRecord::Base
4
+ data_fabric :prefix => 'fiveruns', :replicated => true, :shard_by => :city
5
+ end
6
+
7
+ class DatabaseTest < MiniTest::Test
8
+ def setup
9
+ ActiveRecord::Base.configurations = load_database_yml
10
+ DataFabric::ConnectionProxy.shard_pools.clear
11
+ end
12
+
13
+ def test_features
14
+ DataFabric.activate_shard :city => :dallas do
15
+ assert_equal 'fiveruns_city_dallas_test_slave', TheWholeBurrito.connection.connection_name
16
+ assert_equal DataFabric::PoolProxy, TheWholeBurrito.connection_pool.class
17
+ assert !TheWholeBurrito.connected?
18
+
19
+ # Should use the slave
20
+ burrito = TheWholeBurrito.find(1)
21
+ assert_match 'vr_dallas_slave', burrito.name
22
+
23
+ assert TheWholeBurrito.connected?
24
+ end
25
+ end
26
+
27
+ def test_live_burrito
28
+ DataFabric.activate_shard :city => :dallas do
29
+ assert_equal 'fiveruns_city_dallas_test_slave', TheWholeBurrito.connection.connection_name
30
+
31
+ # Should use the slave
32
+ burrito = TheWholeBurrito.find(1)
33
+ assert_match 'vr_dallas_slave', burrito.name
34
+
35
+ # Should use the master
36
+ burrito.reload
37
+ assert_match 'vr_dallas_master', burrito.name
38
+
39
+ # ...but immediately set it back to default to the slave
40
+ assert_equal 'fiveruns_city_dallas_test_slave', TheWholeBurrito.connection.connection_name
41
+
42
+ # Should use the master
43
+ TheWholeBurrito.transaction do
44
+ burrito = TheWholeBurrito.find(1)
45
+ assert_match 'vr_dallas_master', burrito.name
46
+ burrito.name = 'foo'
47
+ burrito.save!
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,58 @@
1
+ require 'test_helper'
2
+
3
+ class ShardTest < MiniTest::Test
4
+
5
+ def test_activation_should_persist_in_thread
6
+ DataFabric.activate_shard(:city => 'austin')
7
+ assert_equal 'austin', DataFabric.active_shard(:city)
8
+ end
9
+
10
+ def test_activation_in_one_thread_does_not_change_another
11
+ assert_raises ArgumentError do
12
+ DataFabric.active_shard(:city)
13
+ end
14
+ DataFabric.activate_shard(:city => 'austin')
15
+
16
+ Thread.new do
17
+ assert_raises ArgumentError do
18
+ DataFabric.active_shard(:city)
19
+ end
20
+ DataFabric.activate_shard(:city => 'dallas')
21
+ assert_equal 'dallas', DataFabric.active_shard(:city)
22
+ end.join
23
+ end
24
+
25
+ def test_activations_can_be_nested
26
+ DataFabric.activate_shard(:name => 'Belldandy') do
27
+ DataFabric.activate_shard(:technique => 'Thousand Years of Pain') do
28
+ DataFabric.activate_shard(:name => 'Lelouche') do
29
+ DataFabric.activate_shard(:technique => 'Shadow Replication') do
30
+ assert_equal 'Shadow Replication', DataFabric.active_shard(:technique)
31
+ assert_equal 'Lelouche', DataFabric.active_shard(:name)
32
+ end
33
+ assert_equal 'Thousand Years of Pain', DataFabric.active_shard(:technique)
34
+ assert_equal 'Lelouche', DataFabric.active_shard(:name)
35
+ end
36
+ assert_equal 'Thousand Years of Pain', DataFabric.active_shard(:technique)
37
+ assert_equal 'Belldandy', DataFabric.active_shard(:name)
38
+ end
39
+ assert_raises ArgumentError do
40
+ DataFabric.active_shard(:technique)
41
+ end
42
+ assert_equal 'Belldandy', DataFabric.active_shard(:name)
43
+ end
44
+ assert_raises ArgumentError do
45
+ DataFabric.active_shard(:technique)
46
+ end
47
+ assert_raises ArgumentError do
48
+ DataFabric.active_shard(:name)
49
+ end
50
+ end
51
+
52
+ def test_activate_shard_returns_result_of_block
53
+ result = DataFabric.activate_shard(:gundam => 'Exia') do
54
+ 1234
55
+ end
56
+ assert_equal 1234, result
57
+ end
58
+ end
@@ -0,0 +1,28 @@
1
+ ENV['RAILS_ENV'] = 'test'
2
+ ROOT_PATH = File.expand_path(File.join(File.dirname(__FILE__), ".."))
3
+ DATABASE_YML_PATH = File.join(ROOT_PATH, "test", "database.yml")
4
+ Dir.chdir(ROOT_PATH)
5
+
6
+ require 'rubygems'
7
+ require 'erb'
8
+ require 'minitest/autorun'
9
+ require 'flexmock'
10
+
11
+ version = ENV['AR_VERSION']
12
+ if version
13
+ puts "Testing ActiveRecord #{version}"
14
+ gem 'activerecord', "=#{version}"
15
+ end
16
+
17
+ require 'rails'
18
+ require 'active_record'
19
+ require 'active_record/version'
20
+ ActiveRecord::Base.logger = Logger.new(STDOUT)
21
+ ActiveRecord::Base.logger.level = Logger::WARN
22
+
23
+ require 'data_fabric'
24
+
25
+ def load_database_yml
26
+ filename = DATABASE_YML_PATH
27
+ YAML::load(ERB.new(IO.read(filename)).result)
28
+ end
@@ -0,0 +1,83 @@
1
+ require 'test_helper'
2
+
3
+ class ThreadTest < MiniTest::Test
4
+
5
+ MUTEX = Mutex.new
6
+
7
+ def test_class_and_instance_connections
8
+ Object.class_eval %{
9
+ class ThreadedEnchilada < ActiveRecord::Base
10
+ set_table_name :enchiladas
11
+ data_fabric :prefix => 'fiveruns', :replicated => true, :shard_by => :city
12
+ end
13
+ }
14
+ ActiveRecord::Base.configurations = load_database_yml
15
+
16
+ cconn = ThreadedEnchilada.connection
17
+ iconn = ThreadedEnchilada.new.connection
18
+ assert_equal cconn, iconn
19
+ end
20
+
21
+ def xtest_threaded_access
22
+ clear_databases
23
+
24
+ filename = File.join(File.dirname(__FILE__), "database.yml")
25
+ ActiveRecord::Base.configurations = load_database_yml
26
+
27
+ counts = {:austin => 0, :dallas => 0}
28
+ threads = []
29
+ 10.times do
30
+ threads << Thread.new do
31
+ begin
32
+ 200.times do
33
+ city = rand(1_000_000) % 2 == 0 ? :austin : :dallas
34
+ DataFabric.activate_shard :city => city do
35
+ #puts Enchilada.connection.to_s
36
+ #assert_equal "fiveruns_city_#{city}_test_slave", Enchilada.connection.connection_name
37
+ ThreadedEnchilada.create!(:name => "#{city}")
38
+ MUTEX.synchronize do
39
+ counts[city] += 1
40
+ end
41
+ end
42
+ end
43
+ rescue => e
44
+ puts e.message
45
+ puts e.backtrace.join("\n\t")
46
+ end
47
+ end
48
+ end
49
+ threads.each { |thread| thread.join }
50
+
51
+ counts.each_pair do |city, count|
52
+ DataFabric.activate_shard(:city => city) do
53
+ # slave should be empty
54
+ #assert_equal "fiveruns_city_#{city}_test_slave", ThreadedEnchilada.connection.connection_name
55
+ assert_equal 0, ThreadedEnchilada.count
56
+ ThreadedEnchilada.transaction do
57
+ #assert_equal "fiveruns_city_#{city}_test_master", Enchilada.connection.connection_name
58
+ # master should have the counts we expect
59
+ assert_equal count, ThreadedEnchilada.count
60
+ end
61
+ end
62
+ end
63
+ end
64
+
65
+ private
66
+
67
+ def clear_databases
68
+ ActiveRecord::Base.configurations = { 'test' => { :adapter => 'mysql', :host => 'localhost', :database => 'mysql' } }
69
+ ActiveRecord::Base.establish_connection 'test'
70
+ databases = %w( vr_austin_master vr_austin_slave vr_dallas_master vr_dallas_slave )
71
+ databases.each do |db|
72
+ using_connection do
73
+ execute "use #{db}"
74
+ execute "delete from the_whole_burritos"
75
+ end
76
+ end
77
+ ActiveRecord::Base.clear_active_connections!
78
+ end
79
+
80
+ def using_connection(&block)
81
+ ActiveRecord::Base.connection.instance_eval(&block)
82
+ end
83
+ end