makara 0.3.9 → 0.5.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (92) hide show
  1. checksums.yaml +5 -5
  2. data/.github/dependabot.yml +11 -0
  3. data/.github/workflows/CI.yml +88 -0
  4. data/.github/workflows/gem-publish-public.yml +36 -0
  5. data/.rspec +1 -1
  6. data/.rubocop.yml +15 -0
  7. data/.rubocop_todo.yml +670 -0
  8. data/CHANGELOG.md +88 -32
  9. data/Gemfile +1 -16
  10. data/README.md +39 -35
  11. data/Rakefile +1 -1
  12. data/gemfiles/activerecord_5.2.gemfile +8 -0
  13. data/gemfiles/activerecord_6.0.gemfile +8 -0
  14. data/gemfiles/activerecord_6.1.gemfile +8 -0
  15. data/gemfiles/activerecord_head.gemfile +6 -0
  16. data/lib/active_record/connection_adapters/jdbcmysql_makara_adapter.rb +4 -18
  17. data/lib/active_record/connection_adapters/jdbcpostgresql_makara_adapter.rb +4 -18
  18. data/lib/active_record/connection_adapters/makara_abstract_adapter.rb +111 -33
  19. data/lib/active_record/connection_adapters/makara_jdbcmysql_adapter.rb +4 -18
  20. data/lib/active_record/connection_adapters/makara_jdbcpostgresql_adapter.rb +4 -18
  21. data/lib/active_record/connection_adapters/makara_mysql2_adapter.rb +4 -20
  22. data/lib/active_record/connection_adapters/makara_postgis_adapter.rb +4 -19
  23. data/lib/active_record/connection_adapters/makara_postgresql_adapter.rb +4 -20
  24. data/lib/active_record/connection_adapters/mysql2_makara_adapter.rb +4 -20
  25. data/lib/active_record/connection_adapters/postgresql_makara_adapter.rb +4 -20
  26. data/lib/makara.rb +14 -5
  27. data/lib/makara/cache.rb +4 -42
  28. data/lib/makara/config_parser.rb +18 -16
  29. data/lib/makara/connection_wrapper.rb +43 -22
  30. data/lib/makara/context.rb +108 -37
  31. data/lib/makara/cookie.rb +53 -0
  32. data/lib/makara/error_handler.rb +2 -11
  33. data/lib/makara/errors/all_connections_blacklisted.rb +0 -2
  34. data/lib/makara/errors/blacklist_connection.rb +0 -2
  35. data/lib/makara/errors/blacklisted_while_in_transaction.rb +12 -0
  36. data/lib/makara/errors/invalid_shard.rb +14 -0
  37. data/lib/makara/errors/makara_error.rb +0 -1
  38. data/lib/makara/errors/no_connections_available.rb +0 -2
  39. data/lib/makara/logging/logger.rb +1 -5
  40. data/lib/makara/logging/subscriber.rb +0 -2
  41. data/lib/makara/middleware.rb +12 -76
  42. data/lib/makara/pool.rb +55 -47
  43. data/lib/makara/proxy.rb +76 -56
  44. data/lib/makara/railtie.rb +0 -8
  45. data/lib/makara/strategies/abstract.rb +1 -0
  46. data/lib/makara/strategies/priority_failover.rb +2 -0
  47. data/lib/makara/strategies/round_robin.rb +7 -3
  48. data/lib/makara/strategies/shard_aware.rb +45 -0
  49. data/lib/makara/version.rb +2 -4
  50. data/makara.gemspec +26 -3
  51. data/spec/active_record/connection_adapters/makara_abstract_adapter_error_handling_spec.rb +1 -6
  52. data/spec/active_record/connection_adapters/makara_abstract_adapter_spec.rb +10 -14
  53. data/spec/active_record/connection_adapters/makara_mysql2_adapter_spec.rb +18 -16
  54. data/spec/active_record/connection_adapters/makara_postgis_adapter_spec.rb +6 -9
  55. data/spec/active_record/connection_adapters/makara_postgresql_adapter_spec.rb +70 -10
  56. data/spec/cache_spec.rb +2 -53
  57. data/spec/config_parser_spec.rb +75 -57
  58. data/spec/connection_wrapper_spec.rb +6 -4
  59. data/spec/context_spec.rb +163 -100
  60. data/spec/cookie_spec.rb +72 -0
  61. data/spec/middleware_spec.rb +27 -56
  62. data/spec/pool_spec.rb +25 -14
  63. data/spec/proxy_spec.rb +50 -39
  64. data/spec/spec_helper.rb +10 -10
  65. data/spec/strategies/priority_failover_spec.rb +3 -4
  66. data/spec/strategies/round_robin_spec.rb +4 -8
  67. data/spec/strategies/shard_aware_spec.rb +218 -0
  68. data/spec/support/deep_dup.rb +1 -1
  69. data/spec/support/helpers.rb +10 -6
  70. data/spec/support/mock_objects.rb +6 -5
  71. data/spec/support/mysql2_database.yml +3 -2
  72. data/spec/support/mysql2_database_with_custom_errors.yml +6 -1
  73. data/spec/support/pool_extensions.rb +0 -3
  74. data/spec/support/postgis_database.yml +2 -0
  75. data/spec/support/postgis_schema.rb +6 -3
  76. data/spec/support/postgresql_database.yml +2 -2
  77. data/spec/support/proxy_extensions.rb +1 -3
  78. data/spec/support/schema.rb +6 -6
  79. data/spec/support/user.rb +4 -0
  80. metadata +170 -22
  81. data/.travis.yml +0 -70
  82. data/gemfiles/ar-head.gemfile +0 -15
  83. data/gemfiles/ar30.gemfile +0 -32
  84. data/gemfiles/ar31.gemfile +0 -31
  85. data/gemfiles/ar32.gemfile +0 -31
  86. data/gemfiles/ar40.gemfile +0 -17
  87. data/gemfiles/ar41.gemfile +0 -17
  88. data/gemfiles/ar42.gemfile +0 -17
  89. data/gemfiles/ar50.gemfile +0 -15
  90. data/gemfiles/ar51.gemfile +0 -15
  91. data/lib/makara/cache/memory_store.rb +0 -28
  92. data/lib/makara/cache/noop_store.rb +0 -15
@@ -1,20 +1,16 @@
1
1
  require 'active_record'
2
+ require 'makara'
2
3
 
3
4
  module ActiveRecord
4
5
  module ConnectionAdapters
5
6
  class MakaraAbstractAdapter < ::Makara::Proxy
6
-
7
-
8
7
  class ErrorHandler < ::Makara::ErrorHandler
9
-
10
-
11
8
  HARSH_ERRORS = [
12
9
  'ActiveRecord::RecordNotUnique',
13
10
  'ActiveRecord::InvalidForeignKey',
14
11
  'Makara::Errors::BlacklistConnection'
15
12
  ].map(&:freeze).freeze
16
13
 
17
-
18
14
  CONNECTION_MATCHERS = [
19
15
  /(closed|lost|no|terminating|terminated)\s?([^\s]+)?\sconnection/,
20
16
  /gone away/,
@@ -31,11 +27,8 @@ module ActiveRecord
31
27
  /the database system is (starting|shutting)/
32
28
  ].map(&:freeze).freeze
33
29
 
34
-
35
30
  def handle(connection)
36
-
37
31
  yield
38
-
39
32
  rescue Exception => e
40
33
  # do it via class name to avoid version-specific constant dependencies
41
34
  case e.class.name
@@ -48,20 +41,16 @@ module ActiveRecord
48
41
  harshly(e)
49
42
  end
50
43
  end
51
-
52
44
  end
53
45
 
54
-
55
46
  def harsh_errors
56
47
  HARSH_ERRORS
57
48
  end
58
49
 
59
-
60
50
  def connection_matchers
61
51
  CONNECTION_MATCHERS
62
52
  end
63
53
 
64
-
65
54
  def connection_message?(message)
66
55
  message = message.to_s.downcase
67
56
 
@@ -73,7 +62,6 @@ module ActiveRecord
73
62
  end
74
63
  end
75
64
 
76
-
77
65
  def custom_error_message?(connection, message)
78
66
  custom_error_matchers = connection._makara_custom_error_matchers
79
67
  return false if custom_error_matchers.empty?
@@ -81,7 +69,6 @@ module ActiveRecord
81
69
  message = message.to_s
82
70
 
83
71
  custom_error_matchers.each do |matcher|
84
-
85
72
  if matcher.is_a?(String)
86
73
 
87
74
  # accept strings that look like "/.../" as a regex
@@ -100,49 +87,43 @@ module ActiveRecord
100
87
 
101
88
  false
102
89
  end
103
-
104
-
105
90
  end
106
91
 
107
-
108
- hijack_method :execute, :select_rows, :exec_query, :transaction
92
+ hijack_method :execute, :exec_query, :exec_no_cache, :exec_cache, :transaction
109
93
  send_to_all :connect, :reconnect!, :verify!, :clear_cache!, :reset!
110
94
 
111
- SQL_MASTER_MATCHERS = [/\A\s*select.+for update\Z/i, /select.+lock in share mode\Z/i].map(&:freeze).freeze
112
- SQL_SLAVE_MATCHERS = [/\A\s*select\s/i].map(&:freeze).freeze
95
+ control_method :close, :steal!, :expire, :lease, :in_use?, :owner, :schema_cache, :pool=, :pool,
96
+ :schema_cache=, :lock, :seconds_idle, :==
97
+
98
+ SQL_MASTER_MATCHERS = [/\A\s*select.+for update\Z/i, /select.+lock in share mode\Z/i, /\A\s*select.+(nextval|currval|lastval|get_lock|release_lock|pg_advisory_lock|pg_advisory_unlock)\(/i].map(&:freeze).freeze
99
+ SQL_SLAVE_MATCHERS = [/\A\s*(select|with.+\)\s*select)\s/i].map(&:freeze).freeze
113
100
  SQL_ALL_MATCHERS = [/\A\s*set\s/i].map(&:freeze).freeze
114
101
  SQL_SKIP_STICKINESS_MATCHERS = [/\A\s*show\s([\w]+\s)?(field|table|database|schema|view|index)(es|s)?/i, /\A\s*(set|describe|explain|pragma)\s/i].map(&:freeze).freeze
115
102
 
116
-
117
103
  def sql_master_matchers
118
104
  SQL_MASTER_MATCHERS
119
105
  end
120
106
 
121
-
122
107
  def sql_slave_matchers
123
108
  SQL_SLAVE_MATCHERS
124
109
  end
125
110
 
126
-
127
111
  def sql_all_matchers
128
112
  SQL_ALL_MATCHERS
129
113
  end
130
114
 
131
-
132
115
  def sql_skip_stickiness_matchers
133
116
  SQL_SKIP_STICKINESS_MATCHERS
134
117
  end
135
118
 
136
-
137
119
  def initialize(config)
138
120
  @error_handler = ::ActiveRecord::ConnectionAdapters::MakaraAbstractAdapter::ErrorHandler.new
121
+ @control = ActiveRecordPoolControl.new(self)
139
122
  super(config)
140
123
  end
141
124
 
142
-
143
125
  protected
144
126
 
145
-
146
127
  def appropriate_connection(method_name, args, &block)
147
128
  if needed_by_all?(method_name, args)
148
129
 
@@ -163,29 +144,28 @@ module ActiveRecord
163
144
  end
164
145
  end
165
146
 
166
-
167
147
  def should_stick?(method_name, args)
168
148
  sql = coerce_query_to_sql_string(args.first)
169
149
  return false if sql_skip_stickiness_matchers.any?{|m| sql =~ m }
150
+
170
151
  super
171
152
  end
172
153
 
173
-
174
154
  def needed_by_all?(method_name, args)
175
155
  sql = coerce_query_to_sql_string(args.first)
176
156
  return true if sql_all_matchers.any?{|m| sql =~ m }
157
+
177
158
  false
178
159
  end
179
160
 
180
-
181
161
  def needs_master?(method_name, args)
182
162
  sql = coerce_query_to_sql_string(args.first)
183
163
  return true if sql_master_matchers.any?{|m| sql =~ m }
184
164
  return false if sql_slave_matchers.any?{|m| sql =~ m }
165
+
185
166
  true
186
167
  end
187
168
 
188
-
189
169
  def coerce_query_to_sql_string(sql_or_arel)
190
170
  if sql_or_arel.respond_to?(:to_sql)
191
171
  sql_or_arel.to_sql
@@ -194,18 +174,116 @@ module ActiveRecord
194
174
  end
195
175
  end
196
176
 
197
-
198
177
  def connection_for(config)
199
178
  config = Makara::ConfigParser.merge_and_resolve_default_url_config(config)
200
179
  active_record_connection_for(config)
201
180
  end
202
181
 
203
-
204
182
  def active_record_connection_for(config)
205
183
  raise NotImplementedError
206
184
  end
207
185
 
186
+ class ActiveRecordPoolControl
187
+ attr_reader :owner
188
+ alias :in_use? :owner
189
+
190
+ def initialize(proxy)
191
+ @proxy = proxy
192
+ @owner = nil
193
+ @pool = nil
194
+ @schema_cache = ActiveRecord::ConnectionAdapters::SchemaCache.new @proxy
195
+ @idle_since = Concurrent.monotonic_time
196
+ @adapter = ActiveRecord::ConnectionAdapters::AbstractAdapter.new(@proxy)
197
+ end
198
+
199
+ def close(*args)
200
+ @pool.checkin @proxy
201
+ end
202
+
203
+ # this method must only be called while holding connection pool's mutex
204
+ def lease(*args)
205
+ if in_use?
206
+ msg = +"Cannot lease connection, "
207
+ if @owner == Thread.current
208
+ msg << "it is already leased by the current thread."
209
+ else
210
+ msg << "it is already in use by a different thread: #{@owner}. " \
211
+ "Current thread: #{Thread.current}."
212
+ end
213
+ raise ActiveRecordError, msg
214
+ end
215
+ @owner = Thread.current
216
+ end
217
+
218
+ # this method must only be called while holding connection pool's mutex
219
+ def expire(*args)
220
+ if in_use?
221
+ if @owner != Thread.current
222
+ raise ActiveRecordError, "Cannot expire connection, " \
223
+ "it is owned by a different thread: #{@owner}. " \
224
+ "Current thread: #{Thread.current}."
225
+ end
226
+
227
+ @idle_since = Concurrent.monotonic_time
228
+ @owner = nil
229
+ else
230
+ raise ActiveRecordError, "Cannot expire connection, it is not currently leased."
231
+ end
232
+ end
233
+
234
+ # Seconds since this connection was returned to the pool
235
+ def seconds_idle(*args)
236
+ return 0 if in_use?
237
+
238
+ Concurrent.monotonic_time - @idle_since
239
+ end
240
+
241
+ # this method must only be called while holding connection pool's mutex (and a desire for segfaults)
242
+ def steal!(*args)
243
+ if in_use?
244
+ if @owner != Thread.current
245
+ @pool.send :remove_connection_from_thread_cache, @proxy, @owner
246
+ @owner = Thread.current
247
+ end
248
+ else
249
+ raise ActiveRecordError, "Cannot steal connection, it is not currently leased."
250
+ end
251
+ end
252
+
253
+ def schema_cache(*args)
254
+ if @pool.respond_to?(:get_schema_cache) # AR6
255
+ @pool.get_schema_cache(@proxy)
256
+ else
257
+ @schema_cache
258
+ end
259
+ end
260
+
261
+ def schema_cache=(*args)
262
+ cache = args[0]
263
+ cache.connection = @proxy
264
+ if @pool.respond_to?(:set_schema_cache) # AR6
265
+ @pool.set_schema_cache(cache)
266
+ else
267
+ @schema_cache = cache
268
+ end
269
+ end
270
+
271
+ def lock(*args)
272
+ @adapter.lock
273
+ end
274
+
275
+ def pool=(*args)
276
+ @pool = args[0]
277
+ end
208
278
 
279
+ def pool(*args)
280
+ @pool
281
+ end
282
+
283
+ def ==(*args)
284
+ @proxy.object_id == args[0].object_id
285
+ end
286
+ end
209
287
  end
210
288
  end
211
289
  end
@@ -2,24 +2,10 @@ require 'active_record/connection_adapters/makara_abstract_adapter'
2
2
  require 'active_record/connection_adapters/jdbcmysql_adapter'
3
3
  require 'active_record/connection_adapters/makara_mysql2_adapter'
4
4
 
5
- if ActiveRecord::VERSION::MAJOR >= 4
6
-
7
- module ActiveRecord
8
- module ConnectionHandling
9
- def makara_jdbcmysql_connection(config)
10
- makara_mysql2_connection(config)
11
- end
12
- end
13
- end
14
-
15
- else
16
-
17
- module ActiveRecord
18
- class Base
19
- def self.makara_jdbcmysql_connection(config)
20
- self.makara_mysql2_connection(config)
21
- end
5
+ module ActiveRecord
6
+ module ConnectionHandling
7
+ def makara_jdbcmysql_connection(config)
8
+ makara_mysql2_connection(config)
22
9
  end
23
10
  end
24
-
25
11
  end
@@ -2,24 +2,10 @@ require 'active_record/connection_adapters/makara_abstract_adapter'
2
2
  require 'active_record/connection_adapters/jdbcpostgresql_adapter'
3
3
  require 'active_record/connection_adapters/makara_postgresql_adapter'
4
4
 
5
- if ActiveRecord::VERSION::MAJOR >= 4
6
-
7
- module ActiveRecord
8
- module ConnectionHandling
9
- def makara_jdbcpostgresql_connection(config)
10
- makara_postgresql_connection(config)
11
- end
12
- end
13
- end
14
-
15
- else
16
-
17
- module ActiveRecord
18
- class Base
19
- def self.makara_jdbcpostgresql_connection(config)
20
- self.makara_postgresql_connection(config)
21
- end
5
+ module ActiveRecord
6
+ module ConnectionHandling
7
+ def makara_jdbcpostgresql_connection(config)
8
+ makara_postgresql_connection(config)
22
9
  end
23
10
  end
24
-
25
11
  end
@@ -1,32 +1,17 @@
1
1
  require 'active_record/connection_adapters/makara_abstract_adapter'
2
2
  require 'active_record/connection_adapters/mysql2_adapter'
3
3
 
4
- if ActiveRecord::VERSION::MAJOR >= 4
5
-
6
- module ActiveRecord
7
- module ConnectionHandling
8
- def makara_mysql2_connection(config)
9
- ActiveRecord::ConnectionAdapters::MakaraMysql2Adapter.new(config)
10
- end
11
- end
12
- end
13
-
14
- else
15
-
16
- module ActiveRecord
17
- class Base
18
- def self.makara_mysql2_connection(config)
19
- ActiveRecord::ConnectionAdapters::MakaraMysql2Adapter.new(config)
20
- end
4
+ module ActiveRecord
5
+ module ConnectionHandling
6
+ def makara_mysql2_connection(config)
7
+ ActiveRecord::ConnectionAdapters::MakaraMysql2Adapter.new(config)
21
8
  end
22
9
  end
23
-
24
10
  end
25
11
 
26
12
  module ActiveRecord
27
13
  module ConnectionAdapters
28
14
  class MakaraMysql2Adapter < ActiveRecord::ConnectionAdapters::MakaraAbstractAdapter
29
-
30
15
  class << self
31
16
  def visitor_for(*args)
32
17
  ActiveRecord::ConnectionAdapters::Mysql2Adapter.visitor_for(*args)
@@ -38,7 +23,6 @@ module ActiveRecord
38
23
  def active_record_connection_for(config)
39
24
  ::ActiveRecord::Base.mysql2_connection(config)
40
25
  end
41
-
42
26
  end
43
27
  end
44
28
  end
@@ -1,29 +1,14 @@
1
1
  require 'active_record/connection_adapters/makara_abstract_adapter'
2
2
  require 'active_record/connection_adapters/postgis_adapter'
3
3
 
4
- if ActiveRecord::VERSION::MAJOR >= 4
5
-
6
- module ActiveRecord
7
- module ConnectionHandling
8
- def makara_postgis_connection(config)
9
- ActiveRecord::ConnectionAdapters::MakaraPostgisAdapter.new(config)
10
- end
11
- end
12
- end
13
-
14
- else
15
-
16
- module ActiveRecord
17
- class Base
18
- def self.makara_postgis_connection(config)
19
- ActiveRecord::ConnectionAdapters::MakaraPostgisAdapter.new(config)
20
- end
4
+ module ActiveRecord
5
+ module ConnectionHandling
6
+ def makara_postgis_connection(config)
7
+ ActiveRecord::ConnectionAdapters::MakaraPostgisAdapter.new(config)
21
8
  end
22
9
  end
23
-
24
10
  end
25
11
 
26
-
27
12
  module ActiveRecord
28
13
  module ConnectionAdapters
29
14
  class MakaraPostgisAdapter < ActiveRecord::ConnectionAdapters::MakaraAbstractAdapter
@@ -1,32 +1,17 @@
1
1
  require 'active_record/connection_adapters/makara_abstract_adapter'
2
2
  require 'active_record/connection_adapters/postgresql_adapter'
3
3
 
4
- if ActiveRecord::VERSION::MAJOR >= 4
5
-
6
- module ActiveRecord
7
- module ConnectionHandling
8
- def makara_postgresql_connection(config)
9
- ActiveRecord::ConnectionAdapters::MakaraPostgreSQLAdapter.new(config)
10
- end
11
- end
12
- end
13
-
14
- else
15
-
16
- module ActiveRecord
17
- class Base
18
- def self.makara_postgresql_connection(config)
19
- ActiveRecord::ConnectionAdapters::MakaraPostgreSQLAdapter.new(config)
20
- end
4
+ module ActiveRecord
5
+ module ConnectionHandling
6
+ def makara_postgresql_connection(config)
7
+ ActiveRecord::ConnectionAdapters::MakaraPostgreSQLAdapter.new(config)
21
8
  end
22
9
  end
23
-
24
10
  end
25
11
 
26
12
  module ActiveRecord
27
13
  module ConnectionAdapters
28
14
  class MakaraPostgreSQLAdapter < ActiveRecord::ConnectionAdapters::MakaraAbstractAdapter
29
-
30
15
  class << self
31
16
  def visitor_for(*args)
32
17
  ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.visitor_for(*args)
@@ -38,7 +23,6 @@ module ActiveRecord
38
23
  def active_record_connection_for(config)
39
24
  ::ActiveRecord::Base.postgresql_connection(config)
40
25
  end
41
-
42
26
  end
43
27
  end
44
28
  end