xbar 0.0.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. data/Appraisals +25 -0
  2. data/README.mkdn +215 -0
  3. data/Rakefile +337 -1
  4. data/examples/README +5 -0
  5. data/examples/config/simple.json +22 -0
  6. data/examples/example1.rb +34 -0
  7. data/examples/migrations/1_create_users.rb +10 -0
  8. data/examples/setup.rb +43 -0
  9. data/gemfiles/rails3.gemfile +8 -0
  10. data/gemfiles/rails3.gemfile.lock +74 -0
  11. data/gemfiles/rails31.gemfile +8 -0
  12. data/gemfiles/rails31.gemfile.lock +83 -0
  13. data/gemfiles/rails32.gemfile +7 -0
  14. data/gemfiles/rails32.gemfile.lock +117 -0
  15. data/gemfiles/rails4.gemfile +9 -0
  16. data/gemfiles/rails4.gemfile.lock +134 -0
  17. data/lib/migrations/1_create_usage_statistics.rb +23 -0
  18. data/lib/xbar/association.rb +49 -0
  19. data/lib/xbar/association_collection.rb +69 -0
  20. data/lib/xbar/colors.rb +32 -0
  21. data/lib/xbar/has_and_belongs_to_many_association.rb +17 -0
  22. data/lib/xbar/logger.rb +14 -0
  23. data/lib/xbar/mapper.rb +304 -0
  24. data/lib/xbar/migration.rb +76 -0
  25. data/lib/xbar/model.rb +165 -0
  26. data/lib/xbar/proxy.rb +249 -0
  27. data/lib/xbar/rails2/association.rb +133 -0
  28. data/lib/xbar/rails2/persistence.rb +39 -0
  29. data/lib/xbar/rails3/arel.rb +13 -0
  30. data/lib/xbar/rails3/association.rb +112 -0
  31. data/lib/xbar/rails3/persistence.rb +37 -0
  32. data/lib/xbar/rails3.1/singular_association.rb +34 -0
  33. data/lib/xbar/scope_proxy.rb +55 -0
  34. data/lib/xbar/shard.rb +95 -0
  35. data/lib/xbar/version.rb +2 -2
  36. data/lib/xbar.rb +121 -2
  37. data/run +27 -0
  38. data/spec/config/acme.json +53 -0
  39. data/spec/config/connection.rb +2 -0
  40. data/spec/config/default.json +160 -0
  41. data/spec/config/duplicate_shard.json +21 -0
  42. data/spec/config/missing_key.json +20 -0
  43. data/spec/config/new_shards.json +29 -0
  44. data/spec/config/no_master_shard.json +19 -0
  45. data/spec/config/not_entire_sharded.json +23 -0
  46. data/spec/config/octopus.json +27 -0
  47. data/spec/config/octopus_rails.json +25 -0
  48. data/spec/config/production_fully_replicated.json +21 -0
  49. data/spec/config/production_raise_error.json +17 -0
  50. data/spec/config/simple.json +22 -0
  51. data/spec/config/single_adapter.json +20 -0
  52. data/spec/console.rb +15 -0
  53. data/spec/migrations/10_create_users_using_replication.rb +12 -0
  54. data/spec/migrations/11_add_field_in_all_slaves.rb +11 -0
  55. data/spec/migrations/12_create_users_using_block.rb +23 -0
  56. data/spec/migrations/13_create_users_using_block_and_using.rb +15 -0
  57. data/spec/migrations/1_create_users_on_master.rb +9 -0
  58. data/spec/migrations/2_create_users_on_canada.rb +11 -0
  59. data/spec/migrations/3_create_users_on_both_shards.rb +11 -0
  60. data/spec/migrations/4_create_users_on_shards_of_a_group.rb +11 -0
  61. data/spec/migrations/5_create_users_on_multiples_groups.rb +11 -0
  62. data/spec/migrations/6_raise_exception_with_invalid_shard_name.rb +11 -0
  63. data/spec/migrations/7_raise_exception_with_invalid_multiple_shard_names.rb +11 -0
  64. data/spec/migrations/8_raise_exception_with_invalid_group_name.rb +11 -0
  65. data/spec/migrations/9_raise_exception_with_multiple_invalid_group_names.rb +11 -0
  66. data/spec/spec_helper.rb +25 -0
  67. data/spec/support/database_models.rb +78 -0
  68. data/spec/support/xbar_helper.rb +42 -0
  69. data/spec/xbar/association_spec.rb +660 -0
  70. data/spec/xbar/controller_spec.rb +40 -0
  71. data/spec/xbar/logger_spec.rb +22 -0
  72. data/spec/xbar/mapper_spec.rb +283 -0
  73. data/spec/xbar/migration_spec.rb +110 -0
  74. data/spec/xbar/model_spec.rb +434 -0
  75. data/spec/xbar/proxy_spec.rb +124 -0
  76. data/spec/xbar/replication_spec.rb +94 -0
  77. data/spec/xbar/scope_proxy_spec.rb +22 -0
  78. data/spec/xbar/shard_spec.rb +36 -0
  79. data/xbar.gemspec +13 -3
  80. metadata +231 -10
@@ -0,0 +1,304 @@
1
+ module XBar
2
+ #
3
+ # This module holds the current configuration. It is read from a JSON document
4
+ # and always represents exactly the state of that document. The configuration
5
+ # should not be 'tweaked' by changing the state of in-memory structures. The
6
+ # approved way to change the configuration is to call:
7
+ #
8
+ # XBar::Mapper.reset(:xbar_env => "<config file>",
9
+ # :app_env => "<application environment>")
10
+ #
11
+ # This loads a new configuration file from the XBar 'config' directory.
12
+ # This file may contain multiple 'environments'. Only the specified
13
+ # environment is used. Both arguments are required. Changes to the
14
+ # configuration via this API cause all Proxies and their Shards to be reset.
15
+ # This is, all Proxies are found and 'cleaned', and the Shard references held
16
+ # by each Proxy are dropped and new Shards are allocated.
17
+ #
18
+ # No thread-specific state is kept in the Mapper structure. In fact, the state
19
+ # it encapsulates is shared by all threads. In contrast, thread-specific
20
+ # state is kept in instances of the Proxy class, or instances of the Shard
21
+ # class. Thread specific state is handled as follows:
22
+ #
23
+ # Thread.current[:connection_proxy] references an instance of Proxy.
24
+ # In turn, an an instance of XBar::Proxy references a list of instances of
25
+ # XBar::Shard.
26
+ #
27
+ # Each Shard is considered to be an array of replicas, even if the
28
+ # configuration JSON specifies a single Connection (as a String literal). In
29
+ # this case, the Shard is an array of one replica. At any point in time, each
30
+ # Shard has one replica designated as the master. Thus the complete Shard
31
+ # description always an array of Connections and the first Connection in the
32
+ # array is considered to be the master.
33
+ #
34
+ # This module is included in the XBar::Proxy class. This adds instance
35
+ # methods to instances of XBar::Proxy that allow the configuration state to
36
+ # be read.
37
+ #
38
+ # The following data structures are maintained:
39
+ #
40
+ # @@connections -- a Hash, the key is the connection name and the value
41
+ # is a connection pool.
42
+ # @@shards -- an array of Hashes. For each hash the key is the shard name
43
+ # and the value is a connection pool.
44
+ #
45
+ module Mapper
46
+
47
+ def self.exports
48
+ # omit master_config
49
+ %w(connections shards adapters options app_env proxies).map(&:to_sym)
50
+ end
51
+
52
+ module ClassMethods
53
+
54
+ Mapper.exports.each do |var|
55
+ mattr_reader var
56
+ end
57
+
58
+ @@cached_config = nil
59
+ @@shards = HashWithIndifferentAccess.new
60
+ @@connections = HashWithIndifferentAccess.new
61
+ @@proxies = []
62
+ @@adapters = Set.new
63
+ @@config = nil
64
+ @@app_env = nil
65
+ @@xbar_env = nil
66
+
67
+ def config_file_name
68
+ file = "#{xbar_env}.json"
69
+ "#{XBar.directory}/config/#{file}"
70
+ end
71
+
72
+ def connection_file_name
73
+ file = "connection.rb"
74
+ "#{XBar.directory}/config/#{file}"
75
+ end
76
+
77
+ def config_from_file
78
+ file_name = config_file_name
79
+ puts "XBar::Mapper, reading configuration from file #{file_name}"
80
+ if File.exists? file_name
81
+ config = JSON.parse(ERB.new(File.read(file_name)).result)
82
+ else
83
+ config = {}
84
+ end
85
+ HashWithIndifferentAccess.new(config)
86
+ end
87
+
88
+ def config
89
+ @@cached_config ||= config_from_file
90
+ end
91
+
92
+ # Alter the configuration in-memory for the current XBar envirnoment.
93
+ def shards=(shards)
94
+ cached_config["environments"][app_env] = shards
95
+ end
96
+
97
+ # This needs to be reconciled with the 'environments' method in the
98
+ # XBar module. That method specifies the environments that XBar should
99
+ # be enabled for. The present method returns the environments that
100
+ # the current config file contains. XXX
101
+ def environments
102
+ config['environments'].keys
103
+ end
104
+
105
+ #
106
+ # When we switch the XBar env or the Rails env (either of which
107
+ # changes the set of available shards, we have to find all the
108
+ # connection proxies and reset their current shard to :master.)
109
+ #
110
+ # Q1. Are all the connection proxies pointed to by model classes
111
+ # findable through Thread.current[:connection_proxy]? We'll have
112
+ # to loop over all threads. XXX
113
+ #
114
+ # Alternatively, we can register each XBar::Proxy.new call to
115
+ # a hash in the XBar module.
116
+ #
117
+ def reset(options = {})
118
+ new_xbar_env = options[:xbar_env] || xbar_env
119
+ if (new_xbar_env != xbar_env) || (options[:clear_cache]) ||
120
+ (!@@cached_config.nil? && @@cached_config.empty?)
121
+ @@cached_config = nil
122
+ end
123
+ self.xbar_env = new_xbar_env
124
+ self.app_env = options[:app_env] if options[:app_env]
125
+
126
+ # puts "XBar::Mapper#reset, xbar_env=#{xbar_env}, app_env=#{app_env}"
127
+ initialize_shards(config)
128
+ initialize_options(config)
129
+
130
+ # If Rails or some other entity has not assigned a native connection
131
+ # for ActiveRecord, we will try to do something sensible. This is only
132
+ # needed if you have some enviroments for which XBar is not enabled.
133
+ # However, it's not likely you'll want to enable XBar for only some
134
+ # environments. (What would be a use case?) The first
135
+ # choice is that if we have a shard called 'master', we will use its
136
+ # connection specification. The second choice is to include a Ruby
137
+ # file that contains a call to 'establish connection'. In this case,
138
+ # we will create a shard called master with the same connection
139
+ # specification. Thus there will always be a 'master' shard.
140
+ #
141
+ # Also, there is the case where there is a connection, but the config
142
+ # document didn't specify a master shard.
143
+
144
+ begin
145
+ connection_pool = ActiveRecord::Base.connection_pool_without_xbar
146
+ rescue
147
+ if @@shards.keys.include? "master"
148
+ ActiveRecord::Base.establish_connection(
149
+ XBar::Mapper.shards[:master][0].spec.config)
150
+ else
151
+ # The config file didn't exist or didn't specify a master shard. Or
152
+ # app_env wasn't specified (as an argument option).
153
+ require connection_file_name
154
+ connection_pool = ActiveRecord::Base.connection_pool_without_xbar
155
+ end
156
+ end
157
+ if !@@shards.keys.include?("master") && connection_pool
158
+ @@shards[:master] = Array(connection_pool)
159
+ @@adapters << connection_pool.spec.config
160
+ end
161
+
162
+ @@proxies.each do |proxy|
163
+ proxy.reset_proxy
164
+ end
165
+ self
166
+ end
167
+
168
+ def initialize_options(aconfig)
169
+ @@options = aconfig["environments"][app_env].dup
170
+ @@options.delete("shards")
171
+ rescue
172
+ @@options = {}
173
+ ensure
174
+ @@options[:verify_connection] ||= false
175
+ end
176
+
177
+ def register(proxy)
178
+ @@proxies << proxy
179
+
180
+ # If we hang on to a reference to proxies here, the proxy will
181
+ # never be garbage collected, even when the thread that it was
182
+ # assigned to goes away. Find a way to fix this. XXX
183
+ end
184
+
185
+ def app_env
186
+ @@app_env = XBar.rails_env || @@app_env
187
+ end
188
+
189
+ def xbar_env
190
+ @env ||= 'default'
191
+ end
192
+
193
+ private
194
+
195
+ def app_env=(env)
196
+ if XBar.rails_env && XBar.rails_env != env
197
+ raise XBar::ConfigError, "Can't change app_env when you have a Rails environment."
198
+ end
199
+ @@app_env = env
200
+ end
201
+
202
+ # When XBar::Mapper is processing a reset, it will call this method. No other
203
+ # method should call this.
204
+ def xbar_env=(xbar_env)
205
+ @env = xbar_env
206
+ end
207
+
208
+ def initialize_shards(aconfig)
209
+
210
+ @@connections.clear
211
+ @@adapters.clear
212
+ @@shards.clear
213
+
214
+ if aconfig
215
+ begin
216
+ shards_config = aconfig["environments"][app_env]["shards"]
217
+ rescue
218
+ shards_config = nil
219
+ end
220
+ end
221
+ shards_config ||= []
222
+ shards_config.delete_if {|k| k == "__COMMENT"}
223
+
224
+ shards_config.each do |shard_key, connection_key|
225
+ if @@shards.include? shard_key
226
+ raise ConfigError, "You have duplicate shard names!"
227
+ end
228
+ if connection_key.kind_of? String
229
+ spec = aconfig["connections"][connection_key]
230
+ pool = install_connection(connection_key, spec)
231
+ @@shards[shard_key] = [pool]
232
+ else # an array of connection keys
233
+ @@shards[shard_key] = []
234
+ connection_key.each do |conn_key|
235
+ spec = aconfig["connections"][conn_key]
236
+ pool = install_connection(conn_key, spec)
237
+ @@shards[shard_key] << pool
238
+ end
239
+ end
240
+ end
241
+ # @@shards[:master] = [@@connections[:master]]
242
+ end
243
+
244
+ # Should return a ConnectionPool.
245
+ def install_connection(conn_key, spec)
246
+ unless spec
247
+ raise XBar::ConfigError, "No connection for key #{conn_key}"
248
+ end
249
+ if defined? ActiveRecord::Base::ConnectionSpecification::Resolver
250
+ resolver = ActiveRecord::Base::ConnectionSpecification::Resolver.new(spec, {})
251
+ spec = resolver.spec
252
+ @@adapters << spec.config[:adapter]
253
+ @@connections[conn_key.to_sym] =
254
+ ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec)
255
+ else
256
+ old_install_connection(conn_key, spec)
257
+ end
258
+ end
259
+
260
+ def old_install_connection(conn_key, spec)
261
+ unless spec
262
+ raise XBar::ConfigError, "No connection for key #{conn_key}"
263
+ end
264
+ install_adapter(spec['adapter'])
265
+ @@connections[conn_key.to_sym] =
266
+ connection_pool_for(spec, "#{spec['adapter']}_connection")
267
+ end
268
+
269
+ # Called only from 'old_install_connection'. If you get here, you should not
270
+ # be using connection URI's.
271
+ def install_adapter(adapter)
272
+ @@adapters << adapter
273
+ begin
274
+ require "active_record/connection_adapters/#{adapter}_adapter"
275
+ rescue LoadError
276
+ raise "Please install the #{adapter} adapter: " +
277
+ "`gem install activerecord-#{adapter}-adapter` (#{$!})"
278
+ end
279
+ end
280
+
281
+ def connection_pool_for(adapter, spec)
282
+ ActiveRecord::ConnectionAdapters::ConnectionPool.new(
283
+ ActiveRecord::Base::ConnectionSpecification.new(adapter, spec))
284
+ end
285
+ end
286
+
287
+ # Give module XBar::Mapper the above class methods.
288
+ self.extend(ClassMethods)
289
+
290
+ Mapper.exports.each do |meth|
291
+ define_method(meth) {Mapper.send(meth)}
292
+ end
293
+
294
+ def reset_config(options = {})
295
+ Mapper.reset(options)
296
+ end
297
+
298
+ def register
299
+ Mapper.register(self)
300
+ end
301
+
302
+ end
303
+
304
+ end
@@ -0,0 +1,76 @@
1
+ module XBar::Migration
2
+
3
+ def self.extended(base)
4
+ class << base
5
+ def announce_with_xbar(message)
6
+ announce_without_xbar("#{message} - #{get_current_shard}")
7
+ end
8
+ alias_method_chain :migrate, :xbar
9
+ alias_method_chain :announce, :xbar
10
+ attr_accessor :current_shard
11
+ end
12
+ end
13
+
14
+ def self.included(base)
15
+ base.class_eval do
16
+ def announce_with_xbar(message)
17
+ announce_without_xbar("#{message} - #{get_current_shard}")
18
+ end
19
+ alias_method_chain :migrate, :xbar
20
+ alias_method_chain :announce, :xbar
21
+ attr_accessor :current_shard
22
+ end
23
+ base.extend(ClassMethods)
24
+ end
25
+
26
+ module ClassMethods
27
+
28
+ def using(*args)
29
+ if self.connection.is_a?(XBar::Proxy)
30
+ # Doesn't it make sense to only keep the schema_migrations table on the
31
+ # master shard? If we create these other tables, they are unused anyway.
32
+ #args.each do |shard|
33
+ # self.connection.check_schema_migrations(shard)
34
+ #end
35
+ @current_shard = *args
36
+ self.connection.enter_block_scope
37
+ self.current_shard = args
38
+ self.connection.current_shard = args
39
+ end
40
+ return self
41
+ end
42
+
43
+ end
44
+
45
+ def migrate_with_xbar(direction)
46
+ conn = ActiveRecord::Base.connection
47
+ raise "XBar::Migration#mismatched connections" unless conn == self.connection
48
+ if conn.kind_of?(XBar::Proxy)
49
+ u2 = self.class.instance_variable_get(:@current_shard)
50
+ conn.current_shard = u2 if u2
51
+ conn.send_queries_to_multiple_shards(conn.current_shard) do
52
+ migrate_without_xbar(direction)
53
+ end
54
+ else
55
+ migrate_without_xbar(direction)
56
+ end
57
+ ensure
58
+ if conn.kind_of?(XBar::Proxy)
59
+ conn.clean_proxy
60
+ end
61
+ end
62
+
63
+ # Used by migration when printing out results.
64
+ def get_current_shard
65
+ if ActiveRecord::Base.connection.respond_to?(:current_shard)
66
+ "Shard: #{ActiveRecord::Base.connection.current_shard}"
67
+ end
68
+ end
69
+
70
+ end
71
+
72
+ if XBar.rails31?
73
+ ActiveRecord::Migration.send :include, XBar::Migration
74
+ else
75
+ ActiveRecord::Migration.extend(XBar::Migration)
76
+ end
data/lib/xbar/model.rb ADDED
@@ -0,0 +1,165 @@
1
+ require 'active_support/concern'
2
+
3
+ module XBar::Model
4
+
5
+ extend ActiveSupport::Concern
6
+
7
+ included do
8
+ attr_accessor :current_shard
9
+ before_save :reload_connection
10
+ if XBar.rails3?
11
+ after_initialize :set_current_shard
12
+ else
13
+ def after_initialize
14
+ set_current_shard
15
+ end
16
+ end
17
+ class << self
18
+ alias_method_chain :connection, :xbar
19
+ alias_method_chain :connection_pool, :xbar
20
+ end
21
+ end
22
+
23
+ def should_set_current_shard?
24
+ current_shard
25
+ end
26
+
27
+ def connection_proxy
28
+ self.class.connection_proxy
29
+ end
30
+
31
+ def reload_connection_safe
32
+ return yield unless should_set_current_shard?
33
+ original = connection_proxy.current_shard
34
+ connection_proxy.current_shard = current_shard
35
+ result = yield
36
+ connection_proxy.current_shard = original
37
+ result
38
+ end
39
+
40
+ def reload_connection
41
+ return unless should_set_current_shard?
42
+ connection_proxy.current_shard = current_shard
43
+ end
44
+
45
+ private
46
+
47
+ # After initialize callback.
48
+ def set_current_shard
49
+ if new_record? || connection_proxy.in_block_scope?
50
+ if XBar.debug
51
+ type = new_record? ? "New" : "Existing"
52
+ puts "#{type} model callback, current_shard=#{connection_proxy.current_shard}, " +
53
+ "block_scope=#{connection_proxy.in_block_scope?}"
54
+ end
55
+ self.current_shard = connection_proxy.current_shard
56
+ else
57
+ if XBar.debug
58
+ type = new_record? ? "New" : "Existing"
59
+ puts "#{type} model callback, current_shard=#{connection_proxy.current_shard} " +
60
+ "last_current_shard=#{connection_proxy.last_current_shard}, " +
61
+ "block_scope=#{connection_proxy.in_block_scope?}"
62
+ end
63
+ self.current_shard = connection_proxy.last_current_shard
64
+ end
65
+ end
66
+
67
+ module ClassMethods
68
+
69
+ def should_use_normal_connection?
70
+ (defined?(Rails) && XBar.config &&
71
+ !XBar.environments.include?(Rails.env.to_s)) ||
72
+ (if XBar.rails32?
73
+ _establish_connection
74
+ else
75
+ self.read_inheritable_attribute(:_establish_connection)
76
+ end
77
+ )
78
+ end
79
+
80
+ def connection_proxy
81
+ puts "Model allocating new connection proxy" unless Thread.current[:connection_proxy]
82
+ Thread.current[:connection_proxy] ||= XBar::Proxy.new
83
+ end
84
+
85
+ def connection_with_xbar
86
+ if should_use_normal_connection?
87
+ connection_without_xbar
88
+ else
89
+ #puts "Model connection with octopus" if XBar.debug
90
+ #if (connection_proxy.current_model.nil?) || (self != ActiveRecord::Base)
91
+ connection_proxy.current_model = self
92
+ #end
93
+ connection_proxy
94
+ end
95
+ end
96
+
97
+ def connection_pool_with_xbar
98
+ if should_use_normal_connection?
99
+ connection_pool_without_xbar
100
+ else
101
+ connection_proxy.connection_pool
102
+ end
103
+ end
104
+
105
+ def clean_table_name
106
+ return unless connection_proxy.should_clean_table_name?
107
+ if self != ActiveRecord::Base && self.respond_to?(:reset_table_name) &&
108
+ (if XBar.rails32?
109
+ !self._reset_table_name
110
+ else
111
+ !self.read_inheritable_attribute(:_reset_table_name)
112
+ end
113
+ )
114
+ self.reset_table_name
115
+ end
116
+
117
+ if XBar.rails3?
118
+ self.reset_column_information
119
+ self.instance_variable_set(:@quoted_table_name, nil)
120
+ end
121
+ end
122
+
123
+ def using(shard_name)
124
+ return self if defined?(::Rails) && !XBar.environments.include?(Rails.env.to_s)
125
+ clean_table_name
126
+ return XBar::ScopeProxy.new(shard_name, self)
127
+ end
128
+
129
+ def unreplicated_model
130
+ if XBar.rails32?
131
+ self._unreplicated = true
132
+ else
133
+ write_inheritable_attribute(:_unreplicated, true)
134
+ end
135
+ end
136
+
137
+ def unreplicated_model?
138
+ if XBar.rails32?
139
+ _unreplicated
140
+ else
141
+ read_inheritable_attribute(:_unreplicated)
142
+ end
143
+ end
144
+
145
+ def xbar_establish_connection(spec = nil)
146
+ if XBar.rails32?
147
+ self._establish_connection = true
148
+ else
149
+ write_inheritable_attribute(:_establish_connection, true)
150
+ end
151
+ establish_connection(spec)
152
+ end
153
+
154
+ def xbar_set_table_name(value = nil)
155
+ if XBar.rails32?
156
+ self._reset_table_name = true
157
+ self.table_name = value
158
+ else
159
+ write_inheritable_attribute(:_reset_table_name, true)
160
+ set_table_name(value)
161
+ end
162
+ end
163
+ end
164
+ end
165
+