makara 0.5.0 → 0.6.0.pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/dependabot.yml +11 -0
- data/.github/workflows/CI.yml +84 -0
- data/.rspec +1 -1
- data/.rubocop.yml +15 -0
- data/.rubocop_todo.yml +670 -0
- data/CHANGELOG.md +16 -6
- data/Gemfile +1 -16
- data/README.md +53 -52
- data/Rakefile +1 -1
- data/gemfiles/activerecord_5.2.gemfile +8 -0
- data/gemfiles/activerecord_6.0.gemfile +8 -0
- data/gemfiles/activerecord_6.1.gemfile +8 -0
- data/gemfiles/activerecord_head.gemfile +6 -0
- data/lib/active_record/connection_adapters/jdbcmysql_makara_adapter.rb +4 -18
- data/lib/active_record/connection_adapters/jdbcpostgresql_makara_adapter.rb +4 -18
- data/lib/active_record/connection_adapters/makara_abstract_adapter.rb +32 -44
- data/lib/active_record/connection_adapters/makara_jdbcmysql_adapter.rb +4 -18
- data/lib/active_record/connection_adapters/makara_jdbcpostgresql_adapter.rb +4 -18
- data/lib/active_record/connection_adapters/makara_mysql2_adapter.rb +4 -20
- data/lib/active_record/connection_adapters/makara_postgis_adapter.rb +4 -19
- data/lib/active_record/connection_adapters/makara_postgresql_adapter.rb +4 -20
- data/lib/active_record/connection_adapters/mysql2_makara_adapter.rb +4 -20
- data/lib/active_record/connection_adapters/postgresql_makara_adapter.rb +4 -20
- data/lib/makara.rb +0 -2
- data/lib/makara/cache.rb +0 -2
- data/lib/makara/config_parser.rb +64 -114
- data/lib/makara/connection_wrapper.rb +23 -26
- data/lib/makara/context.rb +2 -1
- data/lib/makara/cookie.rb +1 -0
- data/lib/makara/error_handler.rb +0 -9
- data/lib/makara/errors/all_connections_blacklisted.rb +0 -2
- data/lib/makara/errors/blacklist_connection.rb +0 -2
- data/lib/makara/errors/blacklisted_while_in_transaction.rb +0 -2
- data/lib/makara/errors/invalid_shard.rb +1 -3
- data/lib/makara/errors/makara_error.rb +0 -1
- data/lib/makara/errors/no_connections_available.rb +0 -2
- data/lib/makara/logging/logger.rb +0 -4
- data/lib/makara/logging/subscriber.rb +1 -3
- data/lib/makara/middleware.rb +2 -3
- data/lib/makara/pool.rb +4 -9
- data/lib/makara/proxy.rb +90 -77
- data/lib/makara/railtie.rb +0 -2
- data/lib/makara/strategies/abstract.rb +1 -0
- data/lib/makara/strategies/priority_failover.rb +2 -0
- data/lib/makara/strategies/round_robin.rb +1 -3
- data/lib/makara/strategies/shard_aware.rb +0 -2
- data/lib/makara/version.rb +4 -4
- data/makara.gemspec +24 -5
- data/spec/active_record/connection_adapters/makara_abstract_adapter_error_handling_spec.rb +3 -8
- data/spec/active_record/connection_adapters/makara_abstract_adapter_spec.rb +6 -15
- data/spec/active_record/connection_adapters/makara_mysql2_adapter_spec.rb +36 -47
- data/spec/active_record/connection_adapters/makara_postgis_adapter_spec.rb +17 -25
- data/spec/active_record/connection_adapters/makara_postgresql_adapter_spec.rb +31 -42
- data/spec/cache_spec.rb +0 -1
- data/spec/config_parser_spec.rb +61 -63
- data/spec/connection_wrapper_spec.rb +2 -3
- data/spec/context_spec.rb +1 -1
- data/spec/cookie_spec.rb +1 -1
- data/spec/middleware_spec.rb +5 -5
- data/spec/pool_spec.rb +12 -25
- data/spec/proxy_spec.rb +74 -78
- data/spec/spec_helper.rb +12 -1
- data/spec/strategies/priority_failover_spec.rb +4 -5
- data/spec/strategies/round_robin_spec.rb +4 -8
- data/spec/strategies/shard_aware_spec.rb +20 -21
- data/spec/support/deep_dup.rb +1 -1
- data/spec/support/helpers.rb +9 -9
- data/spec/support/mock_objects.rb +2 -5
- data/spec/support/mysql2_database.yml +6 -6
- data/spec/support/mysql2_database_with_custom_errors.yml +6 -6
- data/spec/support/pool_extensions.rb +0 -3
- data/spec/support/postgis_database.yml +4 -4
- data/spec/support/postgis_schema.rb +1 -1
- data/spec/support/postgresql_database.yml +4 -6
- data/spec/support/proxy_extensions.rb +3 -5
- data/spec/support/schema.rb +1 -1
- data/spec/support/user.rb +1 -2
- metadata +158 -22
- data/.travis.yml +0 -131
- data/gemfiles/ar-head.gemfile +0 -24
- data/gemfiles/ar30.gemfile +0 -36
- data/gemfiles/ar31.gemfile +0 -36
- data/gemfiles/ar32.gemfile +0 -36
- data/gemfiles/ar40.gemfile +0 -24
- data/gemfiles/ar41.gemfile +0 -24
- data/gemfiles/ar42.gemfile +0 -24
- data/gemfiles/ar50.gemfile +0 -24
- data/gemfiles/ar51.gemfile +0 -24
- data/gemfiles/ar52.gemfile +0 -24
- data/gemfiles/ar60.gemfile +0 -24
data/CHANGELOG.md
CHANGED
|
@@ -1,16 +1,26 @@
|
|
|
1
1
|
# Change Log
|
|
2
2
|
All notable changes to this project will be documented in this file.
|
|
3
3
|
|
|
4
|
+
## Unreleased
|
|
5
|
+
|
|
6
|
+
## v0.6.0.pre - 2021-01-08
|
|
7
|
+
[Full Changelog](https://github.com/instacart/makara/compare/v0.5.0...v0.6.0.pre)
|
|
8
|
+
- Use ActiveRecord URL resolver instead of copying definition [#294](https://github.com/instacart/makara/pull/294) Matt Larraz
|
|
9
|
+
- Deprecated the term `master` in favor of `primary` [#290](https://github.com/instacart/makara/pull/290) Matt Larraz
|
|
10
|
+
- Deprecated the term `slave` in favor of `replica` [#286](https://github.com/instacart/makara/pull/286) Matt Larraz
|
|
11
|
+
- Fix Ruby 2.7 kwarg warning and add Ruby 3 support [#283](https://github.com/instacart/makara/pull/283) Matt Larraz
|
|
12
|
+
- Drop support for Ruby < 2.5 and ActiveRecord < 5.2 [#281](https://github.com/instacart/makara/pull/281) Matt Larraz
|
|
13
|
+
|
|
4
14
|
## v0.5.0 - 2021-01-08
|
|
5
15
|
[Full Changelog](https://github.com/instacart/makara/compare/v0.4.1...v0.4.2)
|
|
6
16
|
- Replace deprecated URI.unescape with CGI.unescape [#252](https://github.com/instacart/makara/pull/252) Kevin Robatel
|
|
7
17
|
- Override equality operator for ActiveRecord connection wrapper [#269](https://github.com/instacart/makara/pull/269) Praveen Burgu
|
|
8
|
-
- Handle blacklisted connections in master pool while in transaction [#267]
|
|
9
|
-
- Handle ActiveRecord connection pools correctly [#267]
|
|
10
|
-
- Add preliminary support for sharded databases [#267]
|
|
11
|
-
- Fix ActiveRecord connection pool exhaustion [#268]
|
|
12
|
-
- Drop support for Ruby 2.0, 2.1 and 2.2 [#267]
|
|
13
|
-
- Drop support ActiveRecord 3.x and 4.x [#267]
|
|
18
|
+
- Handle blacklisted connections in master pool while in transaction [#267](https://github.com/instacart/makara/pull/267) Praveen Burgu
|
|
19
|
+
- Handle ActiveRecord connection pools correctly [#267](https://github.com/instacart/makara/pull/267) Praveen Burgu
|
|
20
|
+
- Add preliminary support for sharded databases [#267](https://github.com/instacart/makara/pull/267) Praveen Burgu
|
|
21
|
+
- Fix ActiveRecord connection pool exhaustion [#268](https://github.com/instacart/makara/pull/268) Praveen Burgu
|
|
22
|
+
- Drop support for Ruby 2.0, 2.1 and 2.2 [#267](https://github.com/instacart/makara/pull/267) Praveen Burgu
|
|
23
|
+
- Drop support ActiveRecord 3.x and 4.x [#267](https://github.com/instacart/makara/pull/267) Praveen Burgu
|
|
14
24
|
- Set up automatic publishing to Github and Rubygems [#275](https://github.com/instacart/makara/pull/275) Matt Larraz
|
|
15
25
|
|
|
16
26
|
|
data/Gemfile
CHANGED
|
@@ -1,19 +1,4 @@
|
|
|
1
|
-
source
|
|
1
|
+
source "https://rubygems.org"
|
|
2
2
|
|
|
3
3
|
# Specify your gem's dependencies in makara.gemspec
|
|
4
4
|
gemspec
|
|
5
|
-
|
|
6
|
-
gem 'rake'
|
|
7
|
-
gem 'rspec'
|
|
8
|
-
gem 'timecop'
|
|
9
|
-
gem 'byebug', :platform => :ruby
|
|
10
|
-
gem 'ruby-debug', :platform => :jruby
|
|
11
|
-
gem 'rack', '2.2.3'
|
|
12
|
-
|
|
13
|
-
gem 'mysql2', :platform => :ruby
|
|
14
|
-
gem 'pg', '0.21.0', :platform => :ruby
|
|
15
|
-
gem 'activerecord-postgis-adapter', :platform => :ruby
|
|
16
|
-
gem 'rgeo', :platform => :ruby
|
|
17
|
-
|
|
18
|
-
gem 'activerecord-jdbcmysql-adapter', :platform => :jruby
|
|
19
|
-
gem 'activerecord-jdbcpostgresql-adapter', :platform => :jruby
|
data/README.md
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
# Makara
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+

|
|
4
|
+
[](https://badge.fury.io/rb/makara)
|
|
4
5
|
[](https://codeclimate.com/repos/526886a7f3ea00679b00cae6/feed)
|
|
5
6
|
|
|
6
7
|
|
|
7
|
-
Makara is generic
|
|
8
|
+
Makara is generic primary/replica proxy. It handles the heavy lifting of managing, choosing, blacklisting, and cycling through connections. It comes with an ActiveRecord database adapter implementation.
|
|
8
9
|
|
|
9
10
|
## Installation
|
|
10
11
|
|
|
@@ -36,18 +37,18 @@ Next, you need to decide which methods are proxied and which methods should be s
|
|
|
36
37
|
send_to_all :connect, :reconnect, :disconnect, :clear_cache
|
|
37
38
|
```
|
|
38
39
|
|
|
39
|
-
Assuming you don't need to split requests between a
|
|
40
|
+
Assuming you don't need to split requests between a primary and a replica, you're done. If you do need to, implement the `needs_primary?` method:
|
|
40
41
|
|
|
41
42
|
```ruby
|
|
42
43
|
# within MyAwesomeSqlProxy
|
|
43
|
-
def
|
|
44
|
+
def needs_primary?(method_name, args)
|
|
44
45
|
return false if args.empty?
|
|
45
46
|
sql = args.first
|
|
46
47
|
sql !~ /^select/i
|
|
47
48
|
end
|
|
48
49
|
```
|
|
49
50
|
|
|
50
|
-
This implementation will send any request not like "SELECT..." to a
|
|
51
|
+
This implementation will send any request not like "SELECT..." to a primary connection. There are more methods you can override and more control over blacklisting - check out the [makara database adapter](lib/active_record/connection_adapters/makara_abstract_adapter.rb) for examples of advanced usage.
|
|
51
52
|
|
|
52
53
|
### Config Parsing
|
|
53
54
|
|
|
@@ -57,13 +58,13 @@ Makara comes with a config parser which will handle providing subconfigs to the
|
|
|
57
58
|
|
|
58
59
|
Makara handles stickiness by keeping track of which proxies are stuck at any given moment. The context is basically a mapping of proxy ids to the timestamp until which they are stuck.
|
|
59
60
|
|
|
60
|
-
To handle persistence of context across requests in a Rack app, makara provides a middleware. It lays a cookie named `_mkra_stck` which contains the current context. If the next request is executed before the cookie expires, that given context will be used. If something occurs which naturally requires
|
|
61
|
+
To handle persistence of context across requests in a Rack app, makara provides a middleware. It lays a cookie named `_mkra_stck` which contains the current context. If the next request is executed before the cookie expires, that given context will be used. If something occurs which naturally requires the primary on the second request, the context is updated and stored again.
|
|
61
62
|
|
|
62
63
|
#### Stickiness Impact
|
|
63
64
|
|
|
64
|
-
When `sticky:true`, once a query as been sent to
|
|
65
|
+
When `sticky:true`, once a query as been sent to the primary, all queries for the rest of the request will also be sent to the primary. In addition, the cookie described above will be set client side with an expiration defined by time at end of original request + `primary_ttl`. As long as the cookie is valid, all requests will send queries to primary.
|
|
65
66
|
|
|
66
|
-
When `sticky:false`, only queries that need to go to
|
|
67
|
+
When `sticky:false`, only queries that need to go to the primary will go there. Subsequent read queries in the same request will go to replicas.
|
|
67
68
|
|
|
68
69
|
#### Releasing stuck connections (clearing context)
|
|
69
70
|
|
|
@@ -81,28 +82,28 @@ Makara::Context.release('redis')
|
|
|
81
82
|
...
|
|
82
83
|
```
|
|
83
84
|
|
|
84
|
-
A context is local to the curent thread of execution. This will allow you to stick to
|
|
85
|
+
A context is local to the curent thread of execution. This will allow you to stick to the primary safely in a single thread
|
|
85
86
|
in systems such as sidekiq, for instance.
|
|
86
87
|
|
|
87
88
|
|
|
88
|
-
#### Forcing
|
|
89
|
+
#### Forcing Primary
|
|
89
90
|
|
|
90
|
-
If you need to force
|
|
91
|
+
If you need to force the primary in your app then you can simply invoke stick_to_primary! on your connection:
|
|
91
92
|
|
|
92
93
|
```ruby
|
|
93
94
|
persist = true # or false, it's true by default
|
|
94
|
-
proxy.
|
|
95
|
+
proxy.stick_to_primary!(persist)
|
|
95
96
|
```
|
|
96
97
|
|
|
97
|
-
It'll keep the proxy stuck to
|
|
98
|
+
It'll keep the proxy stuck to the primary for the current request, and if `persist = true` (default), it'll be also stored in the context for subsequent requests, keeping the proxy stuck up to the duration of `primary_ttl` configured for the proxy.
|
|
98
99
|
|
|
99
100
|
#### Skipping the Stickiness
|
|
100
101
|
|
|
101
|
-
If you're using the `sticky: true` configuration and you find yourself in a situation where you need to write information through the proxy but you don't want the context to be stuck to
|
|
102
|
+
If you're using the `sticky: true` configuration and you find yourself in a situation where you need to write information through the proxy but you don't want the context to be stuck to the primary, you should use a `without_sticking` block:
|
|
102
103
|
|
|
103
104
|
```ruby
|
|
104
105
|
proxy.without_sticking do
|
|
105
|
-
# do my stuff that would normally cause the proxy to stick to
|
|
106
|
+
# do my stuff that would normally cause the proxy to stick to the primary
|
|
106
107
|
end
|
|
107
108
|
```
|
|
108
109
|
|
|
@@ -116,23 +117,23 @@ Makara::Logging::Logger.logger = ::Logger.new(STDOUT)
|
|
|
116
117
|
|
|
117
118
|
## ActiveRecord Database Adapter
|
|
118
119
|
|
|
119
|
-
So you've found yourself with an ActiveRecord-based project which is starting to get some traffic and you realize 95% of you DB load is from reads. Well you've come to the right spot. Makara is a great solution to break up that load not only between
|
|
120
|
+
So you've found yourself with an ActiveRecord-based project which is starting to get some traffic and you realize 95% of you DB load is from reads. Well you've come to the right spot. Makara is a great solution to break up that load not only between primary and replica but potentially multiple primaries and/or multiple replicas.
|
|
120
121
|
|
|
121
122
|
By creating a makara database adapter which simply acts as a proxy we avoid any major complexity surrounding specific database implementations. The makara adapter doesn't care if the underlying connection is mysql, postgresql, etc it simply cares about the sql string being executed.
|
|
122
123
|
|
|
123
124
|
### What goes where?
|
|
124
125
|
|
|
125
|
-
In general: Any `SELECT` statements will execute against your
|
|
126
|
+
In general: Any `SELECT` statements will execute against your replica(s), anything else will go to the primary.
|
|
126
127
|
|
|
127
128
|
There are some edge cases:
|
|
128
129
|
* `SET` operations will be sent to all connections
|
|
129
130
|
* Execution of specific methods such as `connect!`, `disconnect!`, and `clear_cache!` are invoked on all underlying connections
|
|
130
|
-
* Calls inside a transaction will always be sent to the
|
|
131
|
-
* Locking reads (e.g. `SELECT ... FOR UPDATE`) will always be sent to the
|
|
131
|
+
* Calls inside a transaction will always be sent to the primary (otherwise changes from within the transaction could not be read back on most transaction isolation levels)
|
|
132
|
+
* Locking reads (e.g. `SELECT ... FOR UPDATE`) will always be sent to the primary
|
|
132
133
|
|
|
133
134
|
### Errors / blacklisting
|
|
134
135
|
|
|
135
|
-
Whenever a node fails an operation due to a connection issue, it is blacklisted for the amount of time specified in your database.yml. After that amount of time has passed, the connection will begin receiving queries again. If all
|
|
136
|
+
Whenever a node fails an operation due to a connection issue, it is blacklisted for the amount of time specified in your database.yml. After that amount of time has passed, the connection will begin receiving queries again. If all replica nodes are blacklisted, the primary connection will begin receiving read queries as if it were a replica. Once all nodes are blacklisted the error is raised to the application and all nodes are whitelisted.
|
|
136
137
|
|
|
137
138
|
### Database.yml
|
|
138
139
|
|
|
@@ -151,19 +152,19 @@ production:
|
|
|
151
152
|
id: mysql
|
|
152
153
|
# the following are default values
|
|
153
154
|
blacklist_duration: 5
|
|
154
|
-
|
|
155
|
-
|
|
155
|
+
primary_ttl: 5
|
|
156
|
+
primary_strategy: round_robin
|
|
156
157
|
sticky: true
|
|
157
158
|
|
|
158
159
|
# list your connections with the override values (they're merged into the top-level config)
|
|
159
|
-
# be sure to provide the role if
|
|
160
|
+
# be sure to provide the role if primary, role is assumed to be a replica if not provided
|
|
160
161
|
connections:
|
|
161
|
-
- role:
|
|
162
|
-
host:
|
|
163
|
-
- role:
|
|
164
|
-
host:
|
|
165
|
-
- role:
|
|
166
|
-
host:
|
|
162
|
+
- role: primary
|
|
163
|
+
host: primary.sql.host
|
|
164
|
+
- role: replica
|
|
165
|
+
host: replica1.sql.host
|
|
166
|
+
- role: replica
|
|
167
|
+
host: replica2.sql.host
|
|
167
168
|
```
|
|
168
169
|
|
|
169
170
|
Let's break this down a little bit. At the top level of your config you have the standard `adapter` choice. Currently the available adapters are listed in [lib/active_record/connection_adapters/](lib/active_record/connection_adapters/). They are in the form of `#{db_type}_makara` where db_type is mysql, postgresql, etc.
|
|
@@ -173,31 +174,31 @@ Following the adapter choice is all the standard configurations (host, port, ret
|
|
|
173
174
|
The makara subconfig sets up the proxy with a few of its own options, then provides the connection list. The makara options are:
|
|
174
175
|
* id - an identifier for the proxy, used for sticky behaviour and context. The default is to use a MD5 hash of the configuration contents, so if you are setting `sticky` to true, it's a good idea to also set an `id`. Otherwise any stuck connections will be cleared if the configuration changes (as the default MD5 hash id would change as well)
|
|
175
176
|
* blacklist_duration - the number of seconds a node is blacklisted when a connection failure occurs
|
|
176
|
-
* disable_blacklist - do not blacklist node at any error, useful in case of one
|
|
177
|
+
* disable_blacklist - do not blacklist node at any error, useful in case of one primary
|
|
177
178
|
* sticky - if a node should be stuck to once it's used during a specific context
|
|
178
|
-
*
|
|
179
|
-
*
|
|
180
|
-
*
|
|
179
|
+
* primary_ttl - how long the primary context is persisted. generally, this needs to be longer than any replication lag
|
|
180
|
+
* primary_strategy - use a different strategy for picking the "current" primary node: `failover` will try to keep the same one until it is blacklisted. The default is `round_robin` which will cycle through available ones.
|
|
181
|
+
* replica_strategy - use a different strategy for picking the "current" replica node: `failover` will try to keep the same one until it is blacklisted. The default is `round_robin` which will cycle through available ones.
|
|
181
182
|
* connection_error_matchers - array of custom error matchers you want to be handled gracefully by Makara (as in, errors matching these regexes will result in blacklisting the connection as opposed to raising directly).
|
|
182
183
|
|
|
183
|
-
Connection definitions contain any extra node-specific configurations. If the node should behave as a
|
|
184
|
+
Connection definitions contain any extra node-specific configurations. If the node should behave as a primary you must provide `role: primary`. Any previous configurations can be overridden within a specific node's config. Nodes can also contain weights if you'd like to balance usage based on hardware specifications. Optionally, you can provide a name attribute which will be used in sql logging.
|
|
184
185
|
|
|
185
186
|
```yml
|
|
186
187
|
connections:
|
|
187
|
-
- role:
|
|
188
|
-
host:
|
|
188
|
+
- role: primary
|
|
189
|
+
host: myprimary.sql.host
|
|
189
190
|
blacklist_duration: 0
|
|
190
191
|
|
|
191
|
-
# implicit role:
|
|
192
|
-
- host:
|
|
192
|
+
# implicit role: replica
|
|
193
|
+
- host: mybigreplica.sql.host
|
|
193
194
|
weight: 8
|
|
194
|
-
name: Big
|
|
195
|
-
- host:
|
|
195
|
+
name: Big Replica
|
|
196
|
+
- host: mysmallreplica.sql.host
|
|
196
197
|
weight: 2
|
|
197
|
-
name: Small
|
|
198
|
+
name: Small Replica
|
|
198
199
|
```
|
|
199
200
|
|
|
200
|
-
In the previous config the "Big
|
|
201
|
+
In the previous config the "Big Replica" would receive ~80% of traffic.
|
|
201
202
|
|
|
202
203
|
#### DATABASE_URL
|
|
203
204
|
|
|
@@ -205,7 +206,7 @@ Connections may specify a `url` parameter in place of host, username, password,
|
|
|
205
206
|
|
|
206
207
|
```yml
|
|
207
208
|
connections:
|
|
208
|
-
- role:
|
|
209
|
+
- role: primary
|
|
209
210
|
blacklist_duration: 0
|
|
210
211
|
url: 'mysql2://db_username:db_password@localhost:3306/db_name'
|
|
211
212
|
```
|
|
@@ -214,11 +215,11 @@ We recommend, if using environmental variables, to interpolate them via ERb.
|
|
|
214
215
|
|
|
215
216
|
```yml
|
|
216
217
|
connections:
|
|
217
|
-
- role:
|
|
218
|
+
- role: primary
|
|
218
219
|
blacklist_duration: 0
|
|
219
|
-
url: <%= ENV['
|
|
220
|
-
- role:
|
|
221
|
-
url: <%= ENV['
|
|
220
|
+
url: <%= ENV['DATABASE_URL_PRIMARY'] %>
|
|
221
|
+
- role: replica
|
|
222
|
+
url: <%= ENV['DATABASE_URL_REPLICA'] %>
|
|
222
223
|
```
|
|
223
224
|
|
|
224
225
|
**Important**: *Do NOT use `ENV['DATABASE_URL']`*, as it inteferes with the the database configuration
|
|
@@ -243,7 +244,7 @@ For more information on url parsing, as used in
|
|
|
243
244
|
- ActiveRecord::ConnectionHandling::MergeAndResolveDefaultUrlConfig.new(url_config).resolve
|
|
244
245
|
- 4.2
|
|
245
246
|
[ActiveRecord::ConnectionAdapters::ConnectionSpecification::ConnectionUrlResolver.new(url).to_hash](https://github.com/rails/rails/blob/4-2-stable/activerecord/lib/active_record/connection_handling.rb#L60-L81)
|
|
246
|
-
-
|
|
247
|
+
- primary
|
|
247
248
|
[ActiveRecord::ConnectionAdapters::ConnectionSpecification::ConnectionUrlResolver.new(url).to_hash](https://github.com/rails/rails/blob/97b980b4e61aea3cee429bdee4b2eae2329905cd/activerecord/lib/active_record/connection_handling.rb#L60-L81)
|
|
248
249
|
|
|
249
250
|
|
|
@@ -268,15 +269,15 @@ You can provide strings or regexes. In the case of strings, if they start with
|
|
|
268
269
|
|
|
269
270
|
## Common Problems / Solutions
|
|
270
271
|
|
|
271
|
-
On occasion your app may deal with a situation where makara is not present during a write but a read should use
|
|
272
|
+
On occasion your app may deal with a situation where makara is not present during a write but a read should use primary. In the generic proxy details above you are encouraged to use `stick_to_primary!` to accomplish this. Here's an example:
|
|
272
273
|
|
|
273
274
|
```ruby
|
|
274
|
-
# some third party creates a resource in your db,
|
|
275
|
+
# some third party creates a resource in your db, replication may not have completed yet
|
|
275
276
|
# ...
|
|
276
277
|
# then your app is told to read the resource.
|
|
277
278
|
def handle_request_after_third_party_record_creation
|
|
278
|
-
CreatedResourceClass.connection.
|
|
279
|
-
CreatedResourceClass.find(params[:id]) # will go to
|
|
279
|
+
CreatedResourceClass.connection.stick_to_primary!
|
|
280
|
+
CreatedResourceClass.find(params[:id]) # will go to the primary
|
|
280
281
|
end
|
|
281
282
|
```
|
|
282
283
|
|
data/Rakefile
CHANGED
|
@@ -2,24 +2,10 @@ require 'active_record/connection_adapters/makara_abstract_adapter'
|
|
|
2
2
|
require 'active_record/connection_adapters/jdbcmysql_adapter'
|
|
3
3
|
require 'active_record/connection_adapters/mysql2_makara_adapter'
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
def jdbcmysql_makara_connection(config)
|
|
10
|
-
mysql2_makara_connection(config)
|
|
11
|
-
end
|
|
12
|
-
end
|
|
13
|
-
end
|
|
14
|
-
|
|
15
|
-
else
|
|
16
|
-
|
|
17
|
-
module ActiveRecord
|
|
18
|
-
class Base
|
|
19
|
-
def self.jdbcmysql_makara_connection(config)
|
|
20
|
-
self.mysql2_makara_connection(config)
|
|
21
|
-
end
|
|
5
|
+
module ActiveRecord
|
|
6
|
+
module ConnectionHandling
|
|
7
|
+
def jdbcmysql_makara_connection(config)
|
|
8
|
+
mysql2_makara_connection(config)
|
|
22
9
|
end
|
|
23
10
|
end
|
|
24
|
-
|
|
25
11
|
end
|
|
@@ -2,24 +2,10 @@ require 'active_record/connection_adapters/makara_abstract_adapter'
|
|
|
2
2
|
require 'active_record/connection_adapters/jdbcpostgresql_adapter'
|
|
3
3
|
require 'active_record/connection_adapters/postgresql_makara_adapter'
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
def jdbcpostgresql_makara_connection(config)
|
|
10
|
-
postgresql_makara_connection(config)
|
|
11
|
-
end
|
|
12
|
-
end
|
|
13
|
-
end
|
|
14
|
-
|
|
15
|
-
else
|
|
16
|
-
|
|
17
|
-
module ActiveRecord
|
|
18
|
-
class Base
|
|
19
|
-
def self.jdbcpostgresql_makara_connection(config)
|
|
20
|
-
self.postgresql_makara_connection(config)
|
|
21
|
-
end
|
|
5
|
+
module ActiveRecord
|
|
6
|
+
module ConnectionHandling
|
|
7
|
+
def jdbcpostgresql_makara_connection(config)
|
|
8
|
+
postgresql_makara_connection(config)
|
|
22
9
|
end
|
|
23
10
|
end
|
|
24
|
-
|
|
25
11
|
end
|
|
@@ -4,18 +4,13 @@ require 'makara'
|
|
|
4
4
|
module ActiveRecord
|
|
5
5
|
module ConnectionAdapters
|
|
6
6
|
class MakaraAbstractAdapter < ::Makara::Proxy
|
|
7
|
-
|
|
8
|
-
|
|
9
7
|
class ErrorHandler < ::Makara::ErrorHandler
|
|
10
|
-
|
|
11
|
-
|
|
12
8
|
HARSH_ERRORS = [
|
|
13
9
|
'ActiveRecord::RecordNotUnique',
|
|
14
10
|
'ActiveRecord::InvalidForeignKey',
|
|
15
11
|
'Makara::Errors::BlacklistConnection'
|
|
16
12
|
].map(&:freeze).freeze
|
|
17
13
|
|
|
18
|
-
|
|
19
14
|
CONNECTION_MATCHERS = [
|
|
20
15
|
/(closed|lost|no|terminating|terminated)\s?([^\s]+)?\sconnection/,
|
|
21
16
|
/gone away/,
|
|
@@ -32,11 +27,8 @@ module ActiveRecord
|
|
|
32
27
|
/the database system is (starting|shutting)/
|
|
33
28
|
].map(&:freeze).freeze
|
|
34
29
|
|
|
35
|
-
|
|
36
30
|
def handle(connection)
|
|
37
|
-
|
|
38
31
|
yield
|
|
39
|
-
|
|
40
32
|
rescue Exception => e
|
|
41
33
|
# do it via class name to avoid version-specific constant dependencies
|
|
42
34
|
case e.class.name
|
|
@@ -49,20 +41,16 @@ module ActiveRecord
|
|
|
49
41
|
harshly(e)
|
|
50
42
|
end
|
|
51
43
|
end
|
|
52
|
-
|
|
53
44
|
end
|
|
54
45
|
|
|
55
|
-
|
|
56
46
|
def harsh_errors
|
|
57
47
|
HARSH_ERRORS
|
|
58
48
|
end
|
|
59
49
|
|
|
60
|
-
|
|
61
50
|
def connection_matchers
|
|
62
51
|
CONNECTION_MATCHERS
|
|
63
52
|
end
|
|
64
53
|
|
|
65
|
-
|
|
66
54
|
def connection_message?(message)
|
|
67
55
|
message = message.to_s.downcase
|
|
68
56
|
|
|
@@ -74,7 +62,6 @@ module ActiveRecord
|
|
|
74
62
|
end
|
|
75
63
|
end
|
|
76
64
|
|
|
77
|
-
|
|
78
65
|
def custom_error_message?(connection, message)
|
|
79
66
|
custom_error_matchers = connection._makara_custom_error_matchers
|
|
80
67
|
return false if custom_error_matchers.empty?
|
|
@@ -82,7 +69,6 @@ module ActiveRecord
|
|
|
82
69
|
message = message.to_s
|
|
83
70
|
|
|
84
71
|
custom_error_matchers.each do |matcher|
|
|
85
|
-
|
|
86
72
|
if matcher.is_a?(String)
|
|
87
73
|
|
|
88
74
|
# accept strings that look like "/.../" as a regex
|
|
@@ -101,97 +87,101 @@ module ActiveRecord
|
|
|
101
87
|
|
|
102
88
|
false
|
|
103
89
|
end
|
|
104
|
-
|
|
105
|
-
|
|
106
90
|
end
|
|
107
91
|
|
|
108
|
-
|
|
109
92
|
hijack_method :execute, :exec_query, :exec_no_cache, :exec_cache, :transaction
|
|
110
93
|
send_to_all :connect, :reconnect!, :verify!, :clear_cache!, :reset!
|
|
111
94
|
|
|
112
95
|
control_method :close, :steal!, :expire, :lease, :in_use?, :owner, :schema_cache, :pool=, :pool,
|
|
113
96
|
:schema_cache=, :lock, :seconds_idle, :==
|
|
114
97
|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
SQL_SLAVE_MATCHERS = [/\A\s*(select|with.+\)\s*select)\s/i].map(&:freeze).freeze
|
|
98
|
+
SQL_PRIMARY_MATCHERS = [/\A\s*select.+for update\Z/i, /select.+lock in share mode\Z/i, /\A\s*select.+(nextval|currval|lastval|get_lock|release_lock|pg_advisory_lock|pg_advisory_unlock)\(/i].map(&:freeze).freeze
|
|
99
|
+
SQL_REPLICA_MATCHERS = [/\A\s*(select|with.+\)\s*select)\s/i].map(&:freeze).freeze
|
|
118
100
|
SQL_ALL_MATCHERS = [/\A\s*set\s/i].map(&:freeze).freeze
|
|
119
101
|
SQL_SKIP_STICKINESS_MATCHERS = [/\A\s*show\s([\w]+\s)?(field|table|database|schema|view|index)(es|s)?/i, /\A\s*(set|describe|explain|pragma)\s/i].map(&:freeze).freeze
|
|
120
102
|
|
|
103
|
+
SQL_MASTER_MATCHERS = SQL_PRIMARY_MATCHERS
|
|
104
|
+
deprecate_constant :SQL_MASTER_MATCHERS
|
|
105
|
+
SQL_SLAVE_MATCHERS = SQL_REPLICA_MATCHERS
|
|
106
|
+
deprecate_constant :SQL_SLAVE_MATCHERS
|
|
107
|
+
|
|
108
|
+
def sql_primary_matchers
|
|
109
|
+
SQL_PRIMARY_MATCHERS
|
|
110
|
+
end
|
|
121
111
|
|
|
122
112
|
def sql_master_matchers
|
|
123
|
-
|
|
113
|
+
warn "#{self.class}#sql_master_matchers is deprecated. Use #sql_primary_matchers"
|
|
114
|
+
sql_primary_matchers
|
|
124
115
|
end
|
|
125
116
|
|
|
117
|
+
def sql_replica_matchers
|
|
118
|
+
SQL_REPLICA_MATCHERS
|
|
119
|
+
end
|
|
126
120
|
|
|
127
121
|
def sql_slave_matchers
|
|
128
|
-
|
|
122
|
+
warn "#{self.class}#sql_slave_matchers is deprecated. Use #sql_replica_matchers"
|
|
123
|
+
sql_replica_matchers
|
|
129
124
|
end
|
|
130
125
|
|
|
131
|
-
|
|
132
126
|
def sql_all_matchers
|
|
133
127
|
SQL_ALL_MATCHERS
|
|
134
128
|
end
|
|
135
129
|
|
|
136
|
-
|
|
137
130
|
def sql_skip_stickiness_matchers
|
|
138
131
|
SQL_SKIP_STICKINESS_MATCHERS
|
|
139
132
|
end
|
|
140
133
|
|
|
141
|
-
|
|
142
134
|
def initialize(config)
|
|
143
135
|
@error_handler = ::ActiveRecord::ConnectionAdapters::MakaraAbstractAdapter::ErrorHandler.new
|
|
144
136
|
@control = ActiveRecordPoolControl.new(self)
|
|
145
137
|
super(config)
|
|
146
138
|
end
|
|
147
139
|
|
|
148
|
-
|
|
149
140
|
protected
|
|
150
141
|
|
|
151
|
-
|
|
152
142
|
def appropriate_connection(method_name, args, &block)
|
|
153
143
|
if needed_by_all?(method_name, args)
|
|
154
|
-
|
|
155
144
|
handling_an_all_execution(method_name) do
|
|
156
145
|
hijacked do
|
|
157
|
-
#
|
|
158
|
-
@
|
|
159
|
-
@
|
|
146
|
+
# replica pool must run first.
|
|
147
|
+
@replica_pool.send_to_all(nil, &block) # just yields to each con
|
|
148
|
+
@primary_pool.send_to_all(nil, &block) # just yields to each con
|
|
160
149
|
end
|
|
161
150
|
end
|
|
162
|
-
|
|
163
151
|
else
|
|
164
|
-
|
|
165
152
|
super(method_name, args) do |con|
|
|
166
153
|
yield con
|
|
167
154
|
end
|
|
168
|
-
|
|
169
155
|
end
|
|
170
156
|
end
|
|
171
157
|
|
|
172
|
-
|
|
173
158
|
def should_stick?(method_name, args)
|
|
174
159
|
sql = coerce_query_to_sql_string(args.first)
|
|
175
160
|
return false if sql_skip_stickiness_matchers.any?{|m| sql =~ m }
|
|
161
|
+
|
|
176
162
|
super
|
|
177
163
|
end
|
|
178
164
|
|
|
179
|
-
|
|
180
165
|
def needed_by_all?(method_name, args)
|
|
181
166
|
sql = coerce_query_to_sql_string(args.first)
|
|
182
167
|
return true if sql_all_matchers.any?{|m| sql =~ m }
|
|
168
|
+
|
|
183
169
|
false
|
|
184
170
|
end
|
|
185
171
|
|
|
172
|
+
def needs_primary?(method_name, args)
|
|
173
|
+
if respond_to?(:needs_master?)
|
|
174
|
+
warn "#{self.class}#needs_master? is deprecated. Switch to #needs_primary?"
|
|
175
|
+
needs_master?(method_name, args)
|
|
176
|
+
else
|
|
177
|
+
sql = coerce_query_to_sql_string(args.first)
|
|
178
|
+
return true if sql_primary_matchers.any?{|m| sql =~ m }
|
|
179
|
+
return false if sql_replica_matchers.any?{|m| sql =~ m }
|
|
186
180
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
return true if sql_master_matchers.any?{|m| sql =~ m }
|
|
190
|
-
return false if sql_slave_matchers.any?{|m| sql =~ m }
|
|
191
|
-
true
|
|
181
|
+
true
|
|
182
|
+
end
|
|
192
183
|
end
|
|
193
184
|
|
|
194
|
-
|
|
195
185
|
def coerce_query_to_sql_string(sql_or_arel)
|
|
196
186
|
if sql_or_arel.respond_to?(:to_sql)
|
|
197
187
|
sql_or_arel.to_sql
|
|
@@ -200,13 +190,11 @@ module ActiveRecord
|
|
|
200
190
|
end
|
|
201
191
|
end
|
|
202
192
|
|
|
203
|
-
|
|
204
193
|
def connection_for(config)
|
|
205
194
|
config = Makara::ConfigParser.merge_and_resolve_default_url_config(config)
|
|
206
195
|
active_record_connection_for(config)
|
|
207
196
|
end
|
|
208
197
|
|
|
209
|
-
|
|
210
198
|
def active_record_connection_for(config)
|
|
211
199
|
raise NotImplementedError
|
|
212
200
|
end
|