xbar 0.0.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. data/Appraisals +25 -0
  2. data/README.mkdn +215 -0
  3. data/Rakefile +337 -1
  4. data/examples/README +5 -0
  5. data/examples/config/simple.json +22 -0
  6. data/examples/example1.rb +34 -0
  7. data/examples/migrations/1_create_users.rb +10 -0
  8. data/examples/setup.rb +43 -0
  9. data/gemfiles/rails3.gemfile +8 -0
  10. data/gemfiles/rails3.gemfile.lock +74 -0
  11. data/gemfiles/rails31.gemfile +8 -0
  12. data/gemfiles/rails31.gemfile.lock +83 -0
  13. data/gemfiles/rails32.gemfile +7 -0
  14. data/gemfiles/rails32.gemfile.lock +117 -0
  15. data/gemfiles/rails4.gemfile +9 -0
  16. data/gemfiles/rails4.gemfile.lock +134 -0
  17. data/lib/migrations/1_create_usage_statistics.rb +23 -0
  18. data/lib/xbar/association.rb +49 -0
  19. data/lib/xbar/association_collection.rb +69 -0
  20. data/lib/xbar/colors.rb +32 -0
  21. data/lib/xbar/has_and_belongs_to_many_association.rb +17 -0
  22. data/lib/xbar/logger.rb +14 -0
  23. data/lib/xbar/mapper.rb +304 -0
  24. data/lib/xbar/migration.rb +76 -0
  25. data/lib/xbar/model.rb +165 -0
  26. data/lib/xbar/proxy.rb +249 -0
  27. data/lib/xbar/rails2/association.rb +133 -0
  28. data/lib/xbar/rails2/persistence.rb +39 -0
  29. data/lib/xbar/rails3/arel.rb +13 -0
  30. data/lib/xbar/rails3/association.rb +112 -0
  31. data/lib/xbar/rails3/persistence.rb +37 -0
  32. data/lib/xbar/rails3.1/singular_association.rb +34 -0
  33. data/lib/xbar/scope_proxy.rb +55 -0
  34. data/lib/xbar/shard.rb +95 -0
  35. data/lib/xbar/version.rb +2 -2
  36. data/lib/xbar.rb +121 -2
  37. data/run +27 -0
  38. data/spec/config/acme.json +53 -0
  39. data/spec/config/connection.rb +2 -0
  40. data/spec/config/default.json +160 -0
  41. data/spec/config/duplicate_shard.json +21 -0
  42. data/spec/config/missing_key.json +20 -0
  43. data/spec/config/new_shards.json +29 -0
  44. data/spec/config/no_master_shard.json +19 -0
  45. data/spec/config/not_entire_sharded.json +23 -0
  46. data/spec/config/octopus.json +27 -0
  47. data/spec/config/octopus_rails.json +25 -0
  48. data/spec/config/production_fully_replicated.json +21 -0
  49. data/spec/config/production_raise_error.json +17 -0
  50. data/spec/config/simple.json +22 -0
  51. data/spec/config/single_adapter.json +20 -0
  52. data/spec/console.rb +15 -0
  53. data/spec/migrations/10_create_users_using_replication.rb +12 -0
  54. data/spec/migrations/11_add_field_in_all_slaves.rb +11 -0
  55. data/spec/migrations/12_create_users_using_block.rb +23 -0
  56. data/spec/migrations/13_create_users_using_block_and_using.rb +15 -0
  57. data/spec/migrations/1_create_users_on_master.rb +9 -0
  58. data/spec/migrations/2_create_users_on_canada.rb +11 -0
  59. data/spec/migrations/3_create_users_on_both_shards.rb +11 -0
  60. data/spec/migrations/4_create_users_on_shards_of_a_group.rb +11 -0
  61. data/spec/migrations/5_create_users_on_multiples_groups.rb +11 -0
  62. data/spec/migrations/6_raise_exception_with_invalid_shard_name.rb +11 -0
  63. data/spec/migrations/7_raise_exception_with_invalid_multiple_shard_names.rb +11 -0
  64. data/spec/migrations/8_raise_exception_with_invalid_group_name.rb +11 -0
  65. data/spec/migrations/9_raise_exception_with_multiple_invalid_group_names.rb +11 -0
  66. data/spec/spec_helper.rb +25 -0
  67. data/spec/support/database_models.rb +78 -0
  68. data/spec/support/xbar_helper.rb +42 -0
  69. data/spec/xbar/association_spec.rb +660 -0
  70. data/spec/xbar/controller_spec.rb +40 -0
  71. data/spec/xbar/logger_spec.rb +22 -0
  72. data/spec/xbar/mapper_spec.rb +283 -0
  73. data/spec/xbar/migration_spec.rb +110 -0
  74. data/spec/xbar/model_spec.rb +434 -0
  75. data/spec/xbar/proxy_spec.rb +124 -0
  76. data/spec/xbar/replication_spec.rb +94 -0
  77. data/spec/xbar/scope_proxy_spec.rb +22 -0
  78. data/spec/xbar/shard_spec.rb +36 -0
  79. data/xbar.gemspec +13 -3
  80. metadata +231 -10
@@ -0,0 +1,22 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
2
+
3
+ describe XBar::Logger do
4
+ before(:all) do
5
+ set_xbar_env('default', 'test')
6
+ end
7
+
8
+ before :each do
9
+ @out = StringIO.new
10
+ @log = XBar::Logger.new(@out)
11
+ ActiveRecord::Base.logger = @log
12
+ end
13
+
14
+ after :each do
15
+ ActiveRecord::Base.logger = nil
16
+ end
17
+
18
+ it "should add to the default logger what shard the query was sent" do
19
+ User.using(:canada).create!(:name => "test")
20
+ @out.string.should =~ /Shard: canada/
21
+ end
22
+ end
@@ -0,0 +1,283 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
2
+
3
+ describe XBar::Mapper do
4
+
5
+ describe "creating a new instance" do
6
+ before(:all) do
7
+ set_xbar_env('default', 'test')
8
+ end
9
+
10
+ it "should initialize all shards" do
11
+ XBar::Mapper.shards.keys.to_set.should ==
12
+ ["master", "london", "paris", "moscow", "russia", "russia_east",
13
+ "russia_central", "russia_west", "canada", "canada_east",
14
+ "canada_central", "canada_west", "brazil", "china", "china_east",
15
+ "china_west"].to_set
16
+ end
17
+
18
+ it "should return all environments" do
19
+ XBar::Mapper.environments.should == ["test", "development", "staging"]
20
+ end
21
+
22
+ it "should work with thinking sphinx" do
23
+ master_connection_pool = XBar::Mapper.shards[:master][0]
24
+ master_connection_pool.should be_instance_of(
25
+ ActiveRecord::ConnectionAdapters::ConnectionPool)
26
+ master_connection_pool.spec.config.should ==
27
+ {:adapter=>"mysql2", :username=>"root", :port=>3306,
28
+ :database=>"master", :host=>"localhost"}
29
+ end
30
+
31
+ it 'should create a set with all adapters' do
32
+ adapters = XBar::Mapper.adapters
33
+ adapters.should be_kind_of(Set)
34
+ adapters.to_a.should =~ ["sqlite3", "mysql2", "postgresql"]
35
+ end
36
+
37
+ it "uses second occurrence when there are duplicate shard names" do
38
+ # The JSON parser is discarding the first duplicate key instead of
39
+ # throwing an exception. Just the way JSON works, I guess.
40
+ set_xbar_env("duplicate_shard", "test")
41
+ XBar::Mapper.shards.keys.should == ["master", "sales", "inventory"]
42
+ XBar::Mapper.config["environments"]["test"]["shards"]["inventory"].
43
+ should == "inventory_2"
44
+ config = XBar::Mapper.shards[:inventory][0].spec.config
45
+ config[:database].should == "russia_2"
46
+ end
47
+
48
+ describe "when a config file has a missing connection key" do
49
+ after(:each) do
50
+ set_xbar_env('default') # give something sensible to clean_all_shards
51
+ end
52
+ it "should raise an exception" do
53
+ lambda { set_xbar_env('missing_key', 'test') }.should raise_error(
54
+ XBar::ConfigError, "No connection for key inventory_3")
55
+ end
56
+ end
57
+
58
+ it "should create a master shard if app_env doesn't specify one" do
59
+ set_xbar_env("no_master_shard", "test")
60
+ XBar::Mapper.shards.keys.should == ["sales", "inventory", "master"]
61
+ end
62
+
63
+ describe "should correctly initialize shards when bogus environment is given" do
64
+ before(:each) do
65
+ set_xbar_env('default', 'crazy_environment')
66
+ end
67
+
68
+ it "should initialize just the master shard" do
69
+ XBar::Mapper.shards.keys.should == ["master"]
70
+ end
71
+
72
+ it "should use connection pool that we have configured" do
73
+ XBar::Mapper.shards[:master][0].should ==
74
+ ActiveRecord::Base.connection_pool
75
+ end
76
+
77
+ it "should have one adapter" do
78
+ XBar::Mapper.adapters.size.should ==1
79
+ end
80
+
81
+ it "should have one connection pool for the master shard" do
82
+ XBar::Mapper.shards[:master].size.should == 1
83
+ end
84
+
85
+ end
86
+
87
+ describe "should correctly initialize shards when config file is missing" do
88
+ before(:each) do
89
+ set_xbar_env('missing')
90
+ end
91
+
92
+ it "should initialize just the master shard" do
93
+ XBar::Mapper.shards.keys.should == ["master"]
94
+ end
95
+
96
+ it "should use connection pool that we have configured" do
97
+ XBar::Mapper.shards[:master][0].should ==
98
+ ActiveRecord::Base.connection_pool
99
+ end
100
+
101
+ it "should have one adapter" do
102
+ XBar::Mapper.adapters.size.should ==1
103
+ end
104
+
105
+ it "should have one connection pool for the master shard" do
106
+ XBar::Mapper.shards[:master].size.should == 1
107
+ end
108
+
109
+ end
110
+
111
+ describe "should process environment options correctly" do
112
+ before(:each) do
113
+ set_xbar_env('default', 'development')
114
+ end
115
+
116
+ it "should have correct color" do
117
+ XBar::Mapper.options[:favorite_color].should == 'blue'
118
+ end
119
+
120
+ it "should have correct verify connection option" do
121
+ XBar::Mapper.options[:verify_connection].should == true
122
+ end
123
+
124
+ end
125
+ end
126
+
127
+ describe "When Rails is present" do
128
+ before(:each) do
129
+ Rails = mock
130
+ Rails.stub(:env).and_return('staging')
131
+ end
132
+ after(:each) do
133
+ Object.class_eval {remove_const :Rails}
134
+ end
135
+ it "should not allow app_env to be set" do
136
+ lambda {
137
+ XBar::Mapper.reset(xbar_env: 'anything', app_env: 'anything') }.should raise_error(
138
+ XBar::ConfigError, "Can't change app_env when you have a Rails environment.")
139
+ end
140
+
141
+ it "should use correct environments" do
142
+ XBar.rails_env.should == 'staging'
143
+ XBar::Mapper.app_env.should == 'staging'
144
+ end
145
+
146
+ it "reset should not change any environments" do
147
+ XBar::Mapper.reset
148
+ XBar.rails_env.should == 'staging'
149
+ XBar::Mapper.app_env.should == 'staging'
150
+ XBar::Mapper.xbar_env == 'default'
151
+ end
152
+
153
+ end
154
+
155
+ describe "when a Rails application is not present" do
156
+ before(:each) do
157
+ set_xbar_env('default', 'test')
158
+ end
159
+
160
+ it "should sychronize rails_env and app_env environments" do
161
+ XBar.rails_env.should be_nil
162
+ XBar::Mapper.app_env.should == 'test'
163
+ end
164
+
165
+ it "reset should not change any environments" do
166
+ XBar::Mapper.reset
167
+ XBar.rails_env.should be_nil
168
+ XBar::Mapper.app_env.should == 'test'
169
+ XBar::Mapper.xbar_env == 'default'
170
+ end
171
+ end
172
+
173
+ describe "when a Rails application is present" do
174
+ before(:each) do
175
+ set_xbar_env('default', 'test')
176
+ Rails = mock
177
+ Rails.stub(:env).and_return('staging')
178
+ end
179
+ after(:each) do
180
+ Object.class_eval {remove_const :Rails}
181
+ end
182
+
183
+ it "should sychronize rails_env and app_env environments" do
184
+ XBar.rails_env.should == 'staging'
185
+ XBar::Mapper.app_env == 'test'
186
+ XBar::Mapper.reset
187
+ XBar.rails_env.should == 'staging'
188
+ XBar::Mapper.app_env == 'staging'
189
+ end
190
+ end
191
+
192
+ describe "when loading a new XBar environment" do
193
+
194
+ context "when in the test environment" do
195
+ before(:each) do
196
+ XBar::Mapper.reset(xbar_env: "acme", app_env: "test")
197
+ end
198
+
199
+ it "should be using correct environments" do
200
+ XBar::Mapper.app_env.should == 'test'
201
+ XBar::Mapper.xbar_env.should == 'acme'
202
+ XBar.rails_env.should be_nil
203
+ end
204
+
205
+ it "should have all environments installed" do
206
+ XBar::Mapper.environments.should == ["test", "development", "production", "staging"]
207
+ end
208
+
209
+ it "should have correct shards installed" do
210
+ XBar::Mapper.shards.keys.should == ["master", "sales", "inventory", "common"]
211
+ XBar::Mapper.shards[:inventory].size.should == 1
212
+ XBar::Mapper.shards[:common].size.should == 3
213
+ end
214
+
215
+ end
216
+
217
+ context "when in the development environment" do
218
+ before(:each) do
219
+ XBar::Mapper.reset(xbar_env: "acme", app_env: "development")
220
+ end
221
+
222
+ it "should be using correct environments" do
223
+ XBar::Mapper.app_env.should == 'development'
224
+ XBar::Mapper.xbar_env.should == 'acme'
225
+ XBar.rails_env.should be_nil
226
+ end
227
+
228
+ it "should have all environments installed" do
229
+ XBar::Mapper.environments.should == ["test", "development", "production", "staging"]
230
+ end
231
+
232
+ it "should have the correct shards intalled"do
233
+ XBar::Mapper.shards.keys.should == ["master", "sales", "inventory", "common"]
234
+ XBar::Mapper.shards[:inventory].size.should == 1
235
+ XBar::Mapper.shards[:common].size.should == 3
236
+ end
237
+
238
+ end
239
+
240
+
241
+ context "when in the staging environment" do
242
+ before(:each) do
243
+ XBar::Mapper.reset(xbar_env: "acme", app_env: "staging")
244
+ end
245
+
246
+ it "should be using correct environments" do
247
+ XBar::Mapper.app_env.should == 'staging'
248
+ XBar::Mapper.xbar_env.should == 'acme'
249
+ end
250
+
251
+ it "should have all environments installed" do
252
+ XBar::Mapper.environments.should == ["test", "development", "production", "staging"]
253
+ end
254
+
255
+ it "should have correct shards installed" do
256
+ XBar::Mapper.shards.keys.should == ["master", "inventory", "common"]
257
+ XBar::Mapper.shards[:inventory].size.should == 1
258
+ XBar::Mapper.shards[:common].size.should == 3
259
+ end
260
+
261
+ end
262
+
263
+
264
+ end
265
+
266
+ end
267
+
268
+ describe "when you specify a bogus application environment" do
269
+ before(:each) do
270
+ set_xbar_env("acme", "bogus")
271
+ @proxy = Thread.current[:connection_proxy]
272
+ end
273
+
274
+ it "should initialize the list of shards" do
275
+ XBar::Mapper.shards.keys.should == ["master"]
276
+ @proxy.shard_list.keys.should == ["master"]
277
+ shard = @proxy.shard_list[:master]
278
+ shard.instance_eval do
279
+ @shard_name.should == "master"
280
+ @slaves.length.should == 0
281
+ end
282
+ end
283
+ end
@@ -0,0 +1,110 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
2
+
3
+ describe XBar::Migration do
4
+ before(:each) do
5
+ set_xbar_env('default', 'test')
6
+ end
7
+ it "should run just in the master shard" do
8
+ migrating_to_version 1 do
9
+ User.using(:master).find_by_name("Master").should_not be_nil
10
+ User.using(:canada).find_by_name("Master").should be_nil
11
+ User.using(:brazil).find_by_name("Master").should be_nil
12
+ end
13
+ end
14
+
15
+ it "should run on specific shard" do
16
+ migrating_to_version 2 do
17
+ User.using(:master).find_by_name("Sharding").should be_nil
18
+ User.using(:canada).find_by_name("Sharding").should_not be_nil
19
+ User.using(:canada_east).find_by_name("Sharding").should_not be_nil
20
+ # May need to delay for enventual consistency
21
+ User.using(:canada_central).find_by_name("Sharding").should_not be_nil
22
+ User.using(:canada_west).find_by_name("Sharding").should_not be_nil
23
+ end
24
+ end
25
+
26
+ it "should run on specified shards" do
27
+ migrating_to_version 3 do
28
+ User.using(:canada).find_by_name("Both").should_not be_nil
29
+ User.using(:brazil).find_by_name("Both").should_not be_nil
30
+ end
31
+ end
32
+
33
+ it "should run on specified group" do
34
+ pending
35
+ migrating_to_version 4 do
36
+ User.using(:canada).find_by_name("Group").should_not be_nil
37
+ User.using(:brazil).find_by_name("Group").should_not be_nil
38
+ User.using(:russia).find_by_name("Group").should_not be_nil
39
+ end
40
+ end
41
+
42
+ it "should run on multiples groups" do
43
+ pending
44
+ migrating_to_version 5 do
45
+ User.using(:canada).find(:all, {:conditions => {:name => "MultipleGroup"}}).size.should == 2
46
+ User.using(:brazil).find(:all, {:conditions => {:name => "MultipleGroup"}}).size.should == 2
47
+ User.using(:russia).find(:all, {:conditions => {:name => "MultipleGroup"}}).size.should == 2
48
+ end
49
+ end
50
+
51
+ it "should create users inside block" do
52
+ migrating_to_version 12 do
53
+ User.using(:brazil).where(:name => "UsingBlock1").size.should == 1
54
+ User.using(:brazil).find(:all, :conditions => {:name => "UsingBlock1"}).size.should == 1
55
+ User.using(:brazil).find(:all, :conditions => {:name => "UsingBlock2"}).size.should == 1
56
+ User.using(:canada).find(:all, :conditions => {:name => "UsingCanada"}).size.should == 1
57
+ User.using(:canada).find(:all, :conditions => {:name => "UsingCanada2"}).size.should == 1
58
+ end
59
+ end
60
+
61
+ it "should send the query to the correct shard" do
62
+ migrating_to_version 13 do
63
+ User.using(:brazil).find(:all, :conditions => {:name => "Brazil"}).size.should == 1
64
+ User.using(:brazil).find(:all, :conditions => {:name => "Canada"}).size.should == 0
65
+ User.using(:canada).find(:all, :conditions => {:name => "Brazil"}).size.should == 0
66
+ User.using(:canada).find(:all, :conditions => {:name => "Canada"}).size.should == 1
67
+ end
68
+ end
69
+
70
+ describe "should raise a exception when" do
71
+ it "you specify a invalid shard name" do
72
+ lambda { ActiveRecord::Migrator.run(:up, MIGRATIONS_ROOT, 6) }.should raise_error("Nonexistent Shard Name: amazing_shard")
73
+ end
74
+
75
+ it "you specify a invalid shard name, even if you have multiple shards, and one of them are right" do
76
+ lambda { ActiveRecord::Migrator.run(:up, MIGRATIONS_ROOT, 7) }.should raise_error("Nonexistent Shard Name: invalid_shard")
77
+ end
78
+
79
+ it "you specify a invalid group name" do
80
+ pending
81
+ lambda { ActiveRecord::Migrator.run(:up, MIGRATIONS_ROOT, 8) }.should raise_error("Nonexistent Group Name: invalid_group")
82
+ end
83
+
84
+ it "you specify a invalid group name, even if you have multiple groups, and one of them are right" do
85
+ pending
86
+ lambda { ActiveRecord::Migrator.run(:up, MIGRATIONS_ROOT, 9) }.should raise_error("Nonexistent Group Name: invalid_group")
87
+ end
88
+ end
89
+
90
+ describe "when using replication" do
91
+ it "should run writes on master when you use replication" do
92
+ migrating_to_version 10 do
93
+ Cat.using(:china_east).find_by_name("Replication").should_not be_nil
94
+ Cat.using(:china_west).find_by_name("Replication").should be_nil
95
+ end
96
+ end
97
+
98
+ it "should run in all shards, master or another shards" do
99
+ pending
100
+ using_environment :production_replicated do
101
+ migrating_to_version 11 do
102
+ [:slave4, :slave1, :slave2, :slave3].each do |sym|
103
+ Cat.find_by_name("Slaves").should_not be_nil
104
+ end
105
+ end
106
+ end
107
+ end
108
+ end
109
+ end
110
+