makara 0.5.1 → 0.6.0.pre
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/CI.yml +0 -4
- data/.rubocop_todo.yml +6 -6
- data/CHANGELOG.md +3 -1
- data/README.md +51 -51
- data/lib/active_record/connection_adapters/makara_abstract_adapter.rb +32 -16
- data/lib/makara/config_parser.rb +63 -104
- data/lib/makara/connection_wrapper.rb +1 -1
- data/lib/makara/context.rb +1 -1
- data/lib/makara/logging/subscriber.rb +1 -1
- data/lib/makara/middleware.rb +1 -1
- data/lib/makara/pool.rb +2 -2
- data/lib/makara/proxy.rb +67 -52
- data/lib/makara/version.rb +5 -3
- data/spec/active_record/connection_adapters/makara_abstract_adapter_error_handling_spec.rb +2 -2
- data/spec/active_record/connection_adapters/makara_abstract_adapter_spec.rb +6 -6
- data/spec/active_record/connection_adapters/makara_mysql2_adapter_spec.rb +28 -26
- data/spec/active_record/connection_adapters/makara_postgis_adapter_spec.rb +15 -15
- data/spec/active_record/connection_adapters/makara_postgresql_adapter_spec.rb +25 -23
- data/spec/config_parser_spec.rb +26 -26
- data/spec/connection_wrapper_spec.rb +2 -2
- data/spec/context_spec.rb +1 -1
- data/spec/middleware_spec.rb +4 -4
- data/spec/pool_spec.rb +9 -9
- data/spec/proxy_spec.rb +74 -74
- data/spec/spec_helper.rb +7 -0
- data/spec/strategies/priority_failover_spec.rb +2 -2
- data/spec/strategies/shard_aware_spec.rb +16 -16
- data/spec/support/helpers.rb +6 -6
- data/spec/support/mock_objects.rb +1 -1
- data/spec/support/mysql2_database.yml +4 -4
- data/spec/support/mysql2_database_with_custom_errors.yml +4 -4
- data/spec/support/postgis_database.yml +4 -4
- data/spec/support/postgresql_database.yml +4 -4
- data/spec/support/proxy_extensions.rb +3 -3
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a421cf24f6e652ac4b137d83325de56710ea14215971d958f6dc13726ec2898f
|
4
|
+
data.tar.gz: 297a249df8d86aa04b84173fd0de9943080fb59f293df959c8772937bb53997d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c187bc3841763896882419eab2bac8cb3ba323a06353e2e7d522a25ded7de5f588a463bef270a361556b41aa53471523224d7c302a7f88fd4f4e1e91a77a6c07
|
7
|
+
data.tar.gz: ef575dec489cee7219eda6f767a4b23e9fcdb816f0b33927bba14fcdf31f17fd0b71a6eaed30845352e90e2000750886172d04cdf273d358e1df95318f0a7e82
|
data/.github/workflows/CI.yml
CHANGED
@@ -29,10 +29,6 @@ jobs:
|
|
29
29
|
ruby: '3.0'
|
30
30
|
- activerecord: '5.2'
|
31
31
|
ruby: head
|
32
|
-
- activerecord: head
|
33
|
-
ruby: '2.5'
|
34
|
-
- activerecord: head
|
35
|
-
ruby: '2.6'
|
36
32
|
continue-on-error: ${{ matrix.ruby == 'jruby' || matrix.ruby == 'head' || matrix.activerecord == 'head' }}
|
37
33
|
name: Ruby ${{ matrix.ruby }} / ActiveRecord ${{ matrix.activerecord }}
|
38
34
|
services:
|
data/.rubocop_todo.yml
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# This configuration was generated by
|
2
2
|
# `rubocop --auto-gen-config`
|
3
|
-
# on 2021-01-
|
3
|
+
# on 2021-01-20 22:15:06 UTC using RuboCop version 1.8.1.
|
4
4
|
# The point is for the user to remove these configuration records
|
5
5
|
# one by one as the offenses are removed from the code base.
|
6
6
|
# Note that changes in the inspected code, or installation of new
|
@@ -71,7 +71,7 @@ Layout/FirstHashElementIndentation:
|
|
71
71
|
- 'lib/makara/config_parser.rb'
|
72
72
|
- 'spec/strategies/shard_aware_spec.rb'
|
73
73
|
|
74
|
-
# Offense count:
|
74
|
+
# Offense count: 10
|
75
75
|
# Cop supports --auto-correct.
|
76
76
|
# Configuration parameters: AllowMultipleStyles, EnforcedHashRocketStyle, EnforcedColonStyle, EnforcedLastArgumentHashStyle.
|
77
77
|
# SupportedHashRocketStyles: key, separator, table
|
@@ -243,7 +243,7 @@ Lint/UnusedBlockArgument:
|
|
243
243
|
Exclude:
|
244
244
|
- 'lib/makara/strategies/priority_failover.rb'
|
245
245
|
|
246
|
-
# Offense count:
|
246
|
+
# Offense count: 25
|
247
247
|
# Cop supports --auto-correct.
|
248
248
|
# Configuration parameters: AllowUnusedKeywordArguments, IgnoreEmptyMethods, IgnoreNotImplementedMethods.
|
249
249
|
Lint/UnusedMethodArgument:
|
@@ -283,7 +283,7 @@ Metrics/BlockNesting:
|
|
283
283
|
# Offense count: 4
|
284
284
|
# Configuration parameters: CountComments, CountAsOne.
|
285
285
|
Metrics/ClassLength:
|
286
|
-
Max:
|
286
|
+
Max: 253
|
287
287
|
|
288
288
|
# Offense count: 3
|
289
289
|
# Configuration parameters: IgnoredMethods.
|
@@ -514,7 +514,7 @@ Style/NumericPredicate:
|
|
514
514
|
- 'lib/makara/context.rb'
|
515
515
|
- 'lib/makara/proxy.rb'
|
516
516
|
|
517
|
-
# Offense count:
|
517
|
+
# Offense count: 3
|
518
518
|
# Configuration parameters: AllowedMethods.
|
519
519
|
# AllowedMethods: respond_to_missing?
|
520
520
|
Style/OptionalBooleanParameter:
|
@@ -662,7 +662,7 @@ Style/TrivialAccessors:
|
|
662
662
|
Style/WordArray:
|
663
663
|
EnforcedStyle: brackets
|
664
664
|
|
665
|
-
# Offense count:
|
665
|
+
# Offense count: 27
|
666
666
|
# Cop supports --auto-correct.
|
667
667
|
# Configuration parameters: AutoCorrect, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
|
668
668
|
# URISchemes: http, https
|
data/CHANGELOG.md
CHANGED
@@ -3,9 +3,11 @@ All notable changes to this project will be documented in this file.
|
|
3
3
|
|
4
4
|
## Unreleased
|
5
5
|
|
6
|
-
|
6
|
+
## v0.6.0.pre - 2021-01-08
|
7
7
|
[Full Changelog](https://github.com/instacart/makara/compare/v0.5.0...v0.6.0.pre)
|
8
8
|
- Use ActiveRecord URL resolver instead of copying definition [#294](https://github.com/instacart/makara/pull/294) Matt Larraz
|
9
|
+
- Deprecated the term `master` in favor of `primary` [#290](https://github.com/instacart/makara/pull/290) Matt Larraz
|
10
|
+
- Deprecated the term `slave` in favor of `replica` [#286](https://github.com/instacart/makara/pull/286) Matt Larraz
|
9
11
|
- Fix Ruby 2.7 kwarg warning and add Ruby 3 support [#283](https://github.com/instacart/makara/pull/283) Matt Larraz
|
10
12
|
- Drop support for Ruby < 2.5 and ActiveRecord < 5.2 [#281](https://github.com/instacart/makara/pull/281) Matt Larraz
|
11
13
|
|
data/README.md
CHANGED
@@ -5,7 +5,7 @@
|
|
5
5
|
[![Code Climate](https://codeclimate.com/repos/526886a7f3ea00679b00cae6/badges/7905f7a000492a1078f7/gpa.png)](https://codeclimate.com/repos/526886a7f3ea00679b00cae6/feed)
|
6
6
|
|
7
7
|
|
8
|
-
Makara is generic
|
8
|
+
Makara is generic primary/replica proxy. It handles the heavy lifting of managing, choosing, blacklisting, and cycling through connections. It comes with an ActiveRecord database adapter implementation.
|
9
9
|
|
10
10
|
## Installation
|
11
11
|
|
@@ -37,18 +37,18 @@ Next, you need to decide which methods are proxied and which methods should be s
|
|
37
37
|
send_to_all :connect, :reconnect, :disconnect, :clear_cache
|
38
38
|
```
|
39
39
|
|
40
|
-
Assuming you don't need to split requests between a
|
40
|
+
Assuming you don't need to split requests between a primary and a replica, you're done. If you do need to, implement the `needs_primary?` method:
|
41
41
|
|
42
42
|
```ruby
|
43
43
|
# within MyAwesomeSqlProxy
|
44
|
-
def
|
44
|
+
def needs_primary?(method_name, args)
|
45
45
|
return false if args.empty?
|
46
46
|
sql = args.first
|
47
47
|
sql !~ /^select/i
|
48
48
|
end
|
49
49
|
```
|
50
50
|
|
51
|
-
This implementation will send any request not like "SELECT..." to a
|
51
|
+
This implementation will send any request not like "SELECT..." to a primary connection. There are more methods you can override and more control over blacklisting - check out the [makara database adapter](lib/active_record/connection_adapters/makara_abstract_adapter.rb) for examples of advanced usage.
|
52
52
|
|
53
53
|
### Config Parsing
|
54
54
|
|
@@ -58,13 +58,13 @@ Makara comes with a config parser which will handle providing subconfigs to the
|
|
58
58
|
|
59
59
|
Makara handles stickiness by keeping track of which proxies are stuck at any given moment. The context is basically a mapping of proxy ids to the timestamp until which they are stuck.
|
60
60
|
|
61
|
-
To handle persistence of context across requests in a Rack app, makara provides a middleware. It lays a cookie named `_mkra_stck` which contains the current context. If the next request is executed before the cookie expires, that given context will be used. If something occurs which naturally requires
|
61
|
+
To handle persistence of context across requests in a Rack app, makara provides a middleware. It lays a cookie named `_mkra_stck` which contains the current context. If the next request is executed before the cookie expires, that given context will be used. If something occurs which naturally requires the primary on the second request, the context is updated and stored again.
|
62
62
|
|
63
63
|
#### Stickiness Impact
|
64
64
|
|
65
|
-
When `sticky:true`, once a query as been sent to
|
65
|
+
When `sticky:true`, once a query as been sent to the primary, all queries for the rest of the request will also be sent to the primary. In addition, the cookie described above will be set client side with an expiration defined by time at end of original request + `primary_ttl`. As long as the cookie is valid, all requests will send queries to primary.
|
66
66
|
|
67
|
-
When `sticky:false`, only queries that need to go to
|
67
|
+
When `sticky:false`, only queries that need to go to the primary will go there. Subsequent read queries in the same request will go to replicas.
|
68
68
|
|
69
69
|
#### Releasing stuck connections (clearing context)
|
70
70
|
|
@@ -82,28 +82,28 @@ Makara::Context.release('redis')
|
|
82
82
|
...
|
83
83
|
```
|
84
84
|
|
85
|
-
A context is local to the curent thread of execution. This will allow you to stick to
|
85
|
+
A context is local to the curent thread of execution. This will allow you to stick to the primary safely in a single thread
|
86
86
|
in systems such as sidekiq, for instance.
|
87
87
|
|
88
88
|
|
89
|
-
#### Forcing
|
89
|
+
#### Forcing Primary
|
90
90
|
|
91
|
-
If you need to force
|
91
|
+
If you need to force the primary in your app then you can simply invoke stick_to_primary! on your connection:
|
92
92
|
|
93
93
|
```ruby
|
94
94
|
persist = true # or false, it's true by default
|
95
|
-
proxy.
|
95
|
+
proxy.stick_to_primary!(persist)
|
96
96
|
```
|
97
97
|
|
98
|
-
It'll keep the proxy stuck to
|
98
|
+
It'll keep the proxy stuck to the primary for the current request, and if `persist = true` (default), it'll be also stored in the context for subsequent requests, keeping the proxy stuck up to the duration of `primary_ttl` configured for the proxy.
|
99
99
|
|
100
100
|
#### Skipping the Stickiness
|
101
101
|
|
102
|
-
If you're using the `sticky: true` configuration and you find yourself in a situation where you need to write information through the proxy but you don't want the context to be stuck to
|
102
|
+
If you're using the `sticky: true` configuration and you find yourself in a situation where you need to write information through the proxy but you don't want the context to be stuck to the primary, you should use a `without_sticking` block:
|
103
103
|
|
104
104
|
```ruby
|
105
105
|
proxy.without_sticking do
|
106
|
-
# do my stuff that would normally cause the proxy to stick to
|
106
|
+
# do my stuff that would normally cause the proxy to stick to the primary
|
107
107
|
end
|
108
108
|
```
|
109
109
|
|
@@ -117,23 +117,23 @@ Makara::Logging::Logger.logger = ::Logger.new(STDOUT)
|
|
117
117
|
|
118
118
|
## ActiveRecord Database Adapter
|
119
119
|
|
120
|
-
So you've found yourself with an ActiveRecord-based project which is starting to get some traffic and you realize 95% of you DB load is from reads. Well you've come to the right spot. Makara is a great solution to break up that load not only between
|
120
|
+
So you've found yourself with an ActiveRecord-based project which is starting to get some traffic and you realize 95% of you DB load is from reads. Well you've come to the right spot. Makara is a great solution to break up that load not only between primary and replica but potentially multiple primaries and/or multiple replicas.
|
121
121
|
|
122
122
|
By creating a makara database adapter which simply acts as a proxy we avoid any major complexity surrounding specific database implementations. The makara adapter doesn't care if the underlying connection is mysql, postgresql, etc it simply cares about the sql string being executed.
|
123
123
|
|
124
124
|
### What goes where?
|
125
125
|
|
126
|
-
In general: Any `SELECT` statements will execute against your
|
126
|
+
In general: Any `SELECT` statements will execute against your replica(s), anything else will go to the primary.
|
127
127
|
|
128
128
|
There are some edge cases:
|
129
129
|
* `SET` operations will be sent to all connections
|
130
130
|
* Execution of specific methods such as `connect!`, `disconnect!`, and `clear_cache!` are invoked on all underlying connections
|
131
|
-
* Calls inside a transaction will always be sent to the
|
132
|
-
* Locking reads (e.g. `SELECT ... FOR UPDATE`) will always be sent to the
|
131
|
+
* Calls inside a transaction will always be sent to the primary (otherwise changes from within the transaction could not be read back on most transaction isolation levels)
|
132
|
+
* Locking reads (e.g. `SELECT ... FOR UPDATE`) will always be sent to the primary
|
133
133
|
|
134
134
|
### Errors / blacklisting
|
135
135
|
|
136
|
-
Whenever a node fails an operation due to a connection issue, it is blacklisted for the amount of time specified in your database.yml. After that amount of time has passed, the connection will begin receiving queries again. If all
|
136
|
+
Whenever a node fails an operation due to a connection issue, it is blacklisted for the amount of time specified in your database.yml. After that amount of time has passed, the connection will begin receiving queries again. If all replica nodes are blacklisted, the primary connection will begin receiving read queries as if it were a replica. Once all nodes are blacklisted the error is raised to the application and all nodes are whitelisted.
|
137
137
|
|
138
138
|
### Database.yml
|
139
139
|
|
@@ -152,19 +152,19 @@ production:
|
|
152
152
|
id: mysql
|
153
153
|
# the following are default values
|
154
154
|
blacklist_duration: 5
|
155
|
-
|
156
|
-
|
155
|
+
primary_ttl: 5
|
156
|
+
primary_strategy: round_robin
|
157
157
|
sticky: true
|
158
158
|
|
159
159
|
# list your connections with the override values (they're merged into the top-level config)
|
160
|
-
# be sure to provide the role if
|
160
|
+
# be sure to provide the role if primary, role is assumed to be a replica if not provided
|
161
161
|
connections:
|
162
|
-
- role:
|
163
|
-
host:
|
164
|
-
- role:
|
165
|
-
host:
|
166
|
-
- role:
|
167
|
-
host:
|
162
|
+
- role: primary
|
163
|
+
host: primary.sql.host
|
164
|
+
- role: replica
|
165
|
+
host: replica1.sql.host
|
166
|
+
- role: replica
|
167
|
+
host: replica2.sql.host
|
168
168
|
```
|
169
169
|
|
170
170
|
Let's break this down a little bit. At the top level of your config you have the standard `adapter` choice. Currently the available adapters are listed in [lib/active_record/connection_adapters/](lib/active_record/connection_adapters/). They are in the form of `#{db_type}_makara` where db_type is mysql, postgresql, etc.
|
@@ -174,31 +174,31 @@ Following the adapter choice is all the standard configurations (host, port, ret
|
|
174
174
|
The makara subconfig sets up the proxy with a few of its own options, then provides the connection list. The makara options are:
|
175
175
|
* id - an identifier for the proxy, used for sticky behaviour and context. The default is to use a MD5 hash of the configuration contents, so if you are setting `sticky` to true, it's a good idea to also set an `id`. Otherwise any stuck connections will be cleared if the configuration changes (as the default MD5 hash id would change as well)
|
176
176
|
* blacklist_duration - the number of seconds a node is blacklisted when a connection failure occurs
|
177
|
-
* disable_blacklist - do not blacklist node at any error, useful in case of one
|
177
|
+
* disable_blacklist - do not blacklist node at any error, useful in case of one primary
|
178
178
|
* sticky - if a node should be stuck to once it's used during a specific context
|
179
|
-
*
|
180
|
-
*
|
181
|
-
*
|
179
|
+
* primary_ttl - how long the primary context is persisted. generally, this needs to be longer than any replication lag
|
180
|
+
* primary_strategy - use a different strategy for picking the "current" primary node: `failover` will try to keep the same one until it is blacklisted. The default is `round_robin` which will cycle through available ones.
|
181
|
+
* replica_strategy - use a different strategy for picking the "current" replica node: `failover` will try to keep the same one until it is blacklisted. The default is `round_robin` which will cycle through available ones.
|
182
182
|
* connection_error_matchers - array of custom error matchers you want to be handled gracefully by Makara (as in, errors matching these regexes will result in blacklisting the connection as opposed to raising directly).
|
183
183
|
|
184
|
-
Connection definitions contain any extra node-specific configurations. If the node should behave as a
|
184
|
+
Connection definitions contain any extra node-specific configurations. If the node should behave as a primary you must provide `role: primary`. Any previous configurations can be overridden within a specific node's config. Nodes can also contain weights if you'd like to balance usage based on hardware specifications. Optionally, you can provide a name attribute which will be used in sql logging.
|
185
185
|
|
186
186
|
```yml
|
187
187
|
connections:
|
188
|
-
- role:
|
189
|
-
host:
|
188
|
+
- role: primary
|
189
|
+
host: myprimary.sql.host
|
190
190
|
blacklist_duration: 0
|
191
191
|
|
192
|
-
# implicit role:
|
193
|
-
- host:
|
192
|
+
# implicit role: replica
|
193
|
+
- host: mybigreplica.sql.host
|
194
194
|
weight: 8
|
195
|
-
name: Big
|
196
|
-
- host:
|
195
|
+
name: Big Replica
|
196
|
+
- host: mysmallreplica.sql.host
|
197
197
|
weight: 2
|
198
|
-
name: Small
|
198
|
+
name: Small Replica
|
199
199
|
```
|
200
200
|
|
201
|
-
In the previous config the "Big
|
201
|
+
In the previous config the "Big Replica" would receive ~80% of traffic.
|
202
202
|
|
203
203
|
#### DATABASE_URL
|
204
204
|
|
@@ -206,7 +206,7 @@ Connections may specify a `url` parameter in place of host, username, password,
|
|
206
206
|
|
207
207
|
```yml
|
208
208
|
connections:
|
209
|
-
- role:
|
209
|
+
- role: primary
|
210
210
|
blacklist_duration: 0
|
211
211
|
url: 'mysql2://db_username:db_password@localhost:3306/db_name'
|
212
212
|
```
|
@@ -215,11 +215,11 @@ We recommend, if using environmental variables, to interpolate them via ERb.
|
|
215
215
|
|
216
216
|
```yml
|
217
217
|
connections:
|
218
|
-
- role:
|
218
|
+
- role: primary
|
219
219
|
blacklist_duration: 0
|
220
|
-
url: <%= ENV['
|
221
|
-
- role:
|
222
|
-
url: <%= ENV['
|
220
|
+
url: <%= ENV['DATABASE_URL_PRIMARY'] %>
|
221
|
+
- role: replica
|
222
|
+
url: <%= ENV['DATABASE_URL_REPLICA'] %>
|
223
223
|
```
|
224
224
|
|
225
225
|
**Important**: *Do NOT use `ENV['DATABASE_URL']`*, as it inteferes with the the database configuration
|
@@ -244,7 +244,7 @@ For more information on url parsing, as used in
|
|
244
244
|
- ActiveRecord::ConnectionHandling::MergeAndResolveDefaultUrlConfig.new(url_config).resolve
|
245
245
|
- 4.2
|
246
246
|
[ActiveRecord::ConnectionAdapters::ConnectionSpecification::ConnectionUrlResolver.new(url).to_hash](https://github.com/rails/rails/blob/4-2-stable/activerecord/lib/active_record/connection_handling.rb#L60-L81)
|
247
|
-
-
|
247
|
+
- primary
|
248
248
|
[ActiveRecord::ConnectionAdapters::ConnectionSpecification::ConnectionUrlResolver.new(url).to_hash](https://github.com/rails/rails/blob/97b980b4e61aea3cee429bdee4b2eae2329905cd/activerecord/lib/active_record/connection_handling.rb#L60-L81)
|
249
249
|
|
250
250
|
|
@@ -269,15 +269,15 @@ You can provide strings or regexes. In the case of strings, if they start with
|
|
269
269
|
|
270
270
|
## Common Problems / Solutions
|
271
271
|
|
272
|
-
On occasion your app may deal with a situation where makara is not present during a write but a read should use
|
272
|
+
On occasion your app may deal with a situation where makara is not present during a write but a read should use primary. In the generic proxy details above you are encouraged to use `stick_to_primary!` to accomplish this. Here's an example:
|
273
273
|
|
274
274
|
```ruby
|
275
|
-
# some third party creates a resource in your db,
|
275
|
+
# some third party creates a resource in your db, replication may not have completed yet
|
276
276
|
# ...
|
277
277
|
# then your app is told to read the resource.
|
278
278
|
def handle_request_after_third_party_record_creation
|
279
|
-
CreatedResourceClass.connection.
|
280
|
-
CreatedResourceClass.find(params[:id]) # will go to
|
279
|
+
CreatedResourceClass.connection.stick_to_primary!
|
280
|
+
CreatedResourceClass.find(params[:id]) # will go to the primary
|
281
281
|
end
|
282
282
|
```
|
283
283
|
|
@@ -95,17 +95,32 @@ module ActiveRecord
|
|
95
95
|
control_method :close, :steal!, :expire, :lease, :in_use?, :owner, :schema_cache, :pool=, :pool,
|
96
96
|
:schema_cache=, :lock, :seconds_idle, :==
|
97
97
|
|
98
|
-
|
99
|
-
|
98
|
+
SQL_PRIMARY_MATCHERS = [/\A\s*select.+for update\Z/i, /select.+lock in share mode\Z/i, /\A\s*select.+(nextval|currval|lastval|get_lock|release_lock|pg_advisory_lock|pg_advisory_unlock)\(/i].map(&:freeze).freeze
|
99
|
+
SQL_REPLICA_MATCHERS = [/\A\s*(select|with.+\)\s*select)\s/i].map(&:freeze).freeze
|
100
100
|
SQL_ALL_MATCHERS = [/\A\s*set\s/i].map(&:freeze).freeze
|
101
101
|
SQL_SKIP_STICKINESS_MATCHERS = [/\A\s*show\s([\w]+\s)?(field|table|database|schema|view|index)(es|s)?/i, /\A\s*(set|describe|explain|pragma)\s/i].map(&:freeze).freeze
|
102
102
|
|
103
|
+
SQL_MASTER_MATCHERS = SQL_PRIMARY_MATCHERS
|
104
|
+
deprecate_constant :SQL_MASTER_MATCHERS
|
105
|
+
SQL_SLAVE_MATCHERS = SQL_REPLICA_MATCHERS
|
106
|
+
deprecate_constant :SQL_SLAVE_MATCHERS
|
107
|
+
|
108
|
+
def sql_primary_matchers
|
109
|
+
SQL_PRIMARY_MATCHERS
|
110
|
+
end
|
111
|
+
|
103
112
|
def sql_master_matchers
|
104
|
-
|
113
|
+
warn "#{self.class}#sql_master_matchers is deprecated. Use #sql_primary_matchers"
|
114
|
+
sql_primary_matchers
|
115
|
+
end
|
116
|
+
|
117
|
+
def sql_replica_matchers
|
118
|
+
SQL_REPLICA_MATCHERS
|
105
119
|
end
|
106
120
|
|
107
121
|
def sql_slave_matchers
|
108
|
-
|
122
|
+
warn "#{self.class}#sql_slave_matchers is deprecated. Use #sql_replica_matchers"
|
123
|
+
sql_replica_matchers
|
109
124
|
end
|
110
125
|
|
111
126
|
def sql_all_matchers
|
@@ -126,21 +141,17 @@ module ActiveRecord
|
|
126
141
|
|
127
142
|
def appropriate_connection(method_name, args, &block)
|
128
143
|
if needed_by_all?(method_name, args)
|
129
|
-
|
130
144
|
handling_an_all_execution(method_name) do
|
131
145
|
hijacked do
|
132
|
-
#
|
133
|
-
@
|
134
|
-
@
|
146
|
+
# replica pool must run first.
|
147
|
+
@replica_pool.send_to_all(nil, &block) # just yields to each con
|
148
|
+
@primary_pool.send_to_all(nil, &block) # just yields to each con
|
135
149
|
end
|
136
150
|
end
|
137
|
-
|
138
151
|
else
|
139
|
-
|
140
152
|
super(method_name, args) do |con|
|
141
153
|
yield con
|
142
154
|
end
|
143
|
-
|
144
155
|
end
|
145
156
|
end
|
146
157
|
|
@@ -158,12 +169,17 @@ module ActiveRecord
|
|
158
169
|
false
|
159
170
|
end
|
160
171
|
|
161
|
-
def
|
162
|
-
|
163
|
-
|
164
|
-
|
172
|
+
def needs_primary?(method_name, args)
|
173
|
+
if respond_to?(:needs_master?)
|
174
|
+
warn "#{self.class}#needs_master? is deprecated. Switch to #needs_primary?"
|
175
|
+
needs_master?(method_name, args)
|
176
|
+
else
|
177
|
+
sql = coerce_query_to_sql_string(args.first)
|
178
|
+
return true if sql_primary_matchers.any?{|m| sql =~ m }
|
179
|
+
return false if sql_replica_matchers.any?{|m| sql =~ m }
|
165
180
|
|
166
|
-
|
181
|
+
true
|
182
|
+
end
|
167
183
|
end
|
168
184
|
|
169
185
|
def coerce_query_to_sql_string(sql_or_arel)
|
data/lib/makara/config_parser.rb
CHANGED
@@ -10,114 +10,39 @@ require 'cgi'
|
|
10
10
|
# top_level: 'variable'
|
11
11
|
# another: 'top level variable'
|
12
12
|
# makara:
|
13
|
-
#
|
13
|
+
# primary_ttl: 3
|
14
14
|
# blacklist_duration: 20
|
15
15
|
# connections:
|
16
|
-
# - role: 'master'
|
17
|
-
# - role: '
|
18
|
-
# - role: 'slave'
|
19
|
-
#
|
16
|
+
# - role: 'master' # Deprecated in favor of 'primary'
|
17
|
+
# - role: 'primary'
|
18
|
+
# - role: 'slave' # Deprecated in favor of 'replica'
|
19
|
+
# - role: 'replica'
|
20
|
+
# name: 'replica2'
|
20
21
|
|
21
22
|
module Makara
|
22
23
|
class ConfigParser
|
23
24
|
DEFAULTS = {
|
24
|
-
|
25
|
+
primary_ttl: 5,
|
25
26
|
blacklist_duration: 30,
|
26
27
|
sticky: true
|
27
28
|
}
|
28
29
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
# "username" => "foo",
|
45
|
-
# "password" => "bar",
|
46
|
-
# "pool" => "5",
|
47
|
-
# "timeout" => "3000"
|
48
|
-
# }
|
49
|
-
def initialize(url)
|
50
|
-
raise "Database URL cannot be empty" if url.blank?
|
51
|
-
|
52
|
-
@uri = URI.parse(url)
|
53
|
-
@adapter = @uri.scheme.tr('-', '_')
|
54
|
-
@adapter = "postgresql" if @adapter == "postgres"
|
55
|
-
|
56
|
-
if @uri.opaque
|
57
|
-
@uri.opaque, @query = @uri.opaque.split('?', 2)
|
58
|
-
else
|
59
|
-
@query = @uri.query
|
60
|
-
end
|
61
|
-
end
|
62
|
-
|
63
|
-
# Converts the given URL to a full connection hash.
|
64
|
-
def to_hash
|
65
|
-
config = raw_config.reject { |_,value| value.blank? }
|
66
|
-
config.map { |key,value| config[key] = CGI.unescape(value) if value.is_a? String }
|
67
|
-
config
|
68
|
-
end
|
69
|
-
|
70
|
-
private
|
71
|
-
|
72
|
-
def uri
|
73
|
-
@uri
|
74
|
-
end
|
75
|
-
|
76
|
-
# Converts the query parameters of the URI into a hash.
|
77
|
-
#
|
78
|
-
# "localhost?pool=5&reaping_frequency=2"
|
79
|
-
# # => { "pool" => "5", "reaping_frequency" => "2" }
|
80
|
-
#
|
81
|
-
# returns empty hash if no query present.
|
82
|
-
#
|
83
|
-
# "localhost"
|
84
|
-
# # => {}
|
85
|
-
def query_hash
|
86
|
-
Hash[(@query || '').split("&").map { |pair| pair.split("=") }]
|
87
|
-
end
|
88
|
-
|
89
|
-
def raw_config
|
90
|
-
if uri.opaque
|
91
|
-
query_hash.merge({
|
92
|
-
"adapter" => @adapter,
|
93
|
-
"database" => uri.opaque })
|
94
|
-
else
|
95
|
-
query_hash.merge({
|
96
|
-
"adapter" => @adapter,
|
97
|
-
"username" => uri.user,
|
98
|
-
"password" => uri.password,
|
99
|
-
"port" => uri.port,
|
100
|
-
"database" => database_from_path,
|
101
|
-
"host" => uri.host })
|
102
|
-
end
|
103
|
-
end
|
104
|
-
|
105
|
-
# Returns name of the database.
|
106
|
-
def database_from_path
|
107
|
-
if @adapter == 'sqlite3'
|
108
|
-
# 'sqlite3:/foo' is absolute, because that makes sense. The
|
109
|
-
# corresponding relative version, 'sqlite3:foo', is handled
|
110
|
-
# elsewhere, as an "opaque".
|
111
|
-
|
112
|
-
uri.path
|
113
|
-
else
|
114
|
-
# Only SQLite uses a filename as the "database" name; for
|
115
|
-
# anything else, a leading slash would be silly.
|
116
|
-
|
117
|
-
uri.path.sub(%r{^/}, "")
|
118
|
-
end
|
30
|
+
DEPRECATED_KEYS = {
|
31
|
+
slave_strategy: :replica_strategy,
|
32
|
+
slave_shard_aware: :replica_shard_aware,
|
33
|
+
slave_default_shard: :replica_default_shard,
|
34
|
+
master_strategy: :primary_strategy,
|
35
|
+
master_shard_aware: :primary_shard_aware,
|
36
|
+
master_default_shard: :primary_default_shard,
|
37
|
+
master_ttl: :primary_ttl
|
38
|
+
}.freeze
|
39
|
+
|
40
|
+
ConnectionUrlResolver =
|
41
|
+
if ::ActiveRecord::VERSION::STRING >= "6.1.0"
|
42
|
+
::ActiveRecord::DatabaseConfigurations::ConnectionUrlResolver
|
43
|
+
else
|
44
|
+
::ActiveRecord::ConnectionAdapters::ConnectionSpecification::ConnectionUrlResolver
|
119
45
|
end
|
120
|
-
end
|
121
46
|
|
122
47
|
# NOTE: url format must be, e.g.
|
123
48
|
# url: mysql2://...
|
@@ -145,6 +70,9 @@ module Makara
|
|
145
70
|
@config = config.symbolize_keys
|
146
71
|
@makara_config = DEFAULTS.merge(@config[:makara] || {})
|
147
72
|
@makara_config = @makara_config.symbolize_keys
|
73
|
+
|
74
|
+
replace_deprecated_keys!
|
75
|
+
|
148
76
|
@id = sanitize_id(@makara_config[:id])
|
149
77
|
end
|
150
78
|
|
@@ -155,24 +83,45 @@ module Makara
|
|
155
83
|
end
|
156
84
|
end
|
157
85
|
|
158
|
-
def
|
86
|
+
def primary_configs
|
159
87
|
all_configs
|
160
|
-
.select { |config| config[:role] == '
|
88
|
+
.select { |config| config[:role] == 'primary' }
|
161
89
|
.map { |config| config.except(:role) }
|
162
90
|
end
|
163
91
|
|
164
|
-
def
|
92
|
+
def master_configs
|
93
|
+
warn "#{self.class}#master_configs is deprecated. Switch to #primary_configs"
|
94
|
+
primary_configs
|
95
|
+
end
|
96
|
+
|
97
|
+
def replica_configs
|
165
98
|
all_configs
|
166
|
-
.reject { |config| config[:role] == '
|
99
|
+
.reject { |config| config[:role] == 'primary' }
|
167
100
|
.map { |config| config.except(:role) }
|
168
101
|
end
|
169
102
|
|
103
|
+
def slave_configs
|
104
|
+
warn "#{self.class}#slave_configs is deprecated. Switch to #replica_configs"
|
105
|
+
replica_configs
|
106
|
+
end
|
107
|
+
|
170
108
|
protected
|
171
109
|
|
172
110
|
def all_configs
|
173
|
-
@makara_config[:connections].map do |connection|
|
174
|
-
base_config.merge(makara_config.except(:connections))
|
175
|
-
|
111
|
+
@all_configs ||= @makara_config[:connections].map do |connection|
|
112
|
+
config = base_config.merge(makara_config.except(:connections)).merge(connection.symbolize_keys)
|
113
|
+
|
114
|
+
if config[:role] == "master"
|
115
|
+
warn "Makara role 'master' is deprecated. Use 'primary' instead"
|
116
|
+
config[:role] = "primary"
|
117
|
+
end
|
118
|
+
|
119
|
+
if config[:role] == "slave"
|
120
|
+
warn "Makara role 'slave' is deprecated. Use 'replica' instead"
|
121
|
+
config[:role] = "primary"
|
122
|
+
end
|
123
|
+
|
124
|
+
config
|
176
125
|
end
|
177
126
|
end
|
178
127
|
|
@@ -199,5 +148,15 @@ module Makara
|
|
199
148
|
end
|
200
149
|
end
|
201
150
|
end
|
151
|
+
|
152
|
+
def replace_deprecated_keys!
|
153
|
+
DEPRECATED_KEYS.each do |key, replacement|
|
154
|
+
next unless @makara_config[key]
|
155
|
+
|
156
|
+
warn "Makara config key #{key.inspect} is deprecated, use #{replacement.inspect} instead"
|
157
|
+
|
158
|
+
@makara_config[replacement] = @makara_config.delete(key)
|
159
|
+
end
|
160
|
+
end
|
202
161
|
end
|
203
162
|
end
|