active_record_postgres_recovery 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/README.md +129 -0
- data/lib/active_record_postgres_recovery/configuration.rb +64 -0
- data/lib/active_record_postgres_recovery/handler.rb +123 -0
- data/lib/active_record_postgres_recovery/postgresql_adapter_patch.rb +111 -0
- data/lib/active_record_postgres_recovery/railtie.rb +17 -0
- data/lib/active_record_postgres_recovery/recovery_event.rb +29 -0
- data/lib/active_record_postgres_recovery/sidekiq_middleware.rb +35 -0
- data/lib/active_record_postgres_recovery/version.rb +5 -0
- data/lib/active_record_postgres_recovery.rb +39 -0
- metadata +146 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 118ac55f7530ffc31fccd078108f058f26f89bf4c4246e38a84b31fd3e48f003
|
|
4
|
+
data.tar.gz: 27891803b4db3439ee951d1422c1e60bc0ec5dc5dd75aeb40280bbe8b59e6432
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: eaf635b5582d4de246f5c121a91e18033c0714b77f94e4900641bb67c5a6b3d040fae9e6354a20f3c0f0e6bb4ed9b24114e59b02bf4288ee520b9b7f3d1ad032
|
|
7
|
+
data.tar.gz: 17126675891167aca2d5fee6f8bfd21c93dfa22573af935bc450852545cebe16538e7f3da53c8e3597fc6be32b7b08103e7ebb546ee75583921c5bf5c113c18f
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Hassan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# ActiveRecordPostgresRecovery
|
|
2
|
+
|
|
3
|
+
Safe PostgreSQL connection recovery for Rails apps using ActiveRecord.
|
|
4
|
+
|
|
5
|
+
This gem handles a narrow production failure mode: Rails still has a stale or invalid PostgreSQL connection after a deploy, AWS RDS PostgreSQL failover, restart, or network interruption. It can retry safe read queries, clear affected ActiveRecord connection pools, and emit structured recovery events to your observability stack.
|
|
6
|
+
|
|
7
|
+
It does not hide database outages and it does not retry writes.
|
|
8
|
+
|
|
9
|
+
## Installation
|
|
10
|
+
|
|
11
|
+
Add this line to your Rails app Gemfile:
|
|
12
|
+
|
|
13
|
+
```ruby
|
|
14
|
+
gem 'active_record_postgres_recovery'
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Then run:
|
|
18
|
+
|
|
19
|
+
```sh
|
|
20
|
+
bundle install
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
## Configuration
|
|
24
|
+
|
|
25
|
+
Create `config/initializers/active_record_postgres_recovery.rb`:
|
|
26
|
+
|
|
27
|
+
```ruby
|
|
28
|
+
ActiveRecordPostgresRecovery.configure do |config|
|
|
29
|
+
config.enabled = true
|
|
30
|
+
config.retry_read_queries = true
|
|
31
|
+
config.max_retries = 1
|
|
32
|
+
config.roles = %i[writing reading]
|
|
33
|
+
config.failover_clear_roles = %i[writing]
|
|
34
|
+
|
|
35
|
+
config.reporter = lambda do |event|
|
|
36
|
+
Bugsnag.notify(event.matched_error) do |report|
|
|
37
|
+
report.severity = 'warning'
|
|
38
|
+
report.add_metadata(:active_record_postgres_recovery, event.to_h)
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
The reporter is optional. Without one, recovery still runs but events are not sent anywhere. If the reporter itself raises, the gem logs a warning and continues without masking the database recovery path.
|
|
45
|
+
|
|
46
|
+
For production rollouts, prefer environment-backed switches so recovery can be disabled without a deploy:
|
|
47
|
+
|
|
48
|
+
```ruby
|
|
49
|
+
ActiveRecordPostgresRecovery.configure do |config|
|
|
50
|
+
config.enabled = ENV.fetch('ACTIVE_RECORD_POSTGRES_RECOVERY_ENABLED', true)
|
|
51
|
+
config.retry_read_queries = ENV.fetch('ACTIVE_RECORD_POSTGRES_RECOVERY_RETRY_READS', true)
|
|
52
|
+
config.max_retries = ENV.fetch('ACTIVE_RECORD_POSTGRES_RECOVERY_MAX_RETRIES', 1)
|
|
53
|
+
|
|
54
|
+
config.reporter = lambda do |event|
|
|
55
|
+
Rails.logger.warn(
|
|
56
|
+
event: 'active_record_postgres_recovery',
|
|
57
|
+
recovery: event.to_h
|
|
58
|
+
)
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
### Options
|
|
64
|
+
|
|
65
|
+
| Option | Default | Description |
|
|
66
|
+
| --- | --- | --- |
|
|
67
|
+
| `enabled` | `true` | Enables recovery handling. When false, matching database errors are re-raised without recovery logic. |
|
|
68
|
+
| `reporter` | `nil` | Callable that receives a `RecoveryEvent`. Use this to send recovery data to Bugsnag, Datadog, logs, or metrics. |
|
|
69
|
+
| `roles` | `%i[writing reading]` | ActiveRecord roles whose pools are fully cleared for normal stale connection errors. |
|
|
70
|
+
| `failover_clear_roles` | `%i[writing]` | ActiveRecord roles cleared when a read-only transaction error indicates a bad failover/write connection. |
|
|
71
|
+
| `retry_read_queries` | `true` | Enables one or more retries for safe read queries outside transactions. |
|
|
72
|
+
| `max_retries` | `1` | Maximum retry attempts for retryable read queries. Writes are still never retried. |
|
|
73
|
+
| `error_patterns` | PostgreSQL stale connection patterns | Regex list used to decide whether an exception is handled by this gem. |
|
|
74
|
+
|
|
75
|
+
You can append app-specific PostgreSQL errors if needed:
|
|
76
|
+
|
|
77
|
+
```ruby
|
|
78
|
+
ActiveRecordPostgresRecovery.configure do |config|
|
|
79
|
+
config.error_patterns += [
|
|
80
|
+
/server closed the connection unexpectedly/i
|
|
81
|
+
]
|
|
82
|
+
end
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
### Recovery Events
|
|
86
|
+
|
|
87
|
+
The reporter receives an event with these attributes:
|
|
88
|
+
|
|
89
|
+
| Attribute | Description |
|
|
90
|
+
| --- | --- |
|
|
91
|
+
| `outcome` | `:attempted` when recovery was attempted and the original error is re-raised, or `:recovered` after a retry succeeds. |
|
|
92
|
+
| `source` | Source of the recovery event, for example `ActiveRecord` or `Sidekiq`. |
|
|
93
|
+
| `context` | Query name or job context. |
|
|
94
|
+
| `error` | Original exception. |
|
|
95
|
+
| `matched_error` | Exception in the cause chain that matched `error_patterns`. |
|
|
96
|
+
| `retrying` | Whether the operation was retrying. |
|
|
97
|
+
| `clear_action` | Hash describing which connection pools were cleared before the retry or re-raise. |
|
|
98
|
+
|
|
99
|
+
Use `event.to_h` for structured metadata safe to attach to observability tools.
|
|
100
|
+
|
|
101
|
+
## Sidekiq
|
|
102
|
+
|
|
103
|
+
If you want Sidekiq jobs to clear stale ActiveRecord connections before Sidekiq retries the job:
|
|
104
|
+
|
|
105
|
+
```ruby
|
|
106
|
+
require 'active_record_postgres_recovery/sidekiq_middleware'
|
|
107
|
+
|
|
108
|
+
Sidekiq.configure_server do |config|
|
|
109
|
+
config.server_middleware do |chain|
|
|
110
|
+
chain.add ActiveRecordPostgresRecovery::SidekiqMiddleware
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## Safety Rules
|
|
116
|
+
|
|
117
|
+
The adapter patch is intentionally conservative:
|
|
118
|
+
|
|
119
|
+
- A non-transactional read query may clear the configured pools, reconnect, and retry.
|
|
120
|
+
- Write queries are not retried automatically.
|
|
121
|
+
- Queries inside an open transaction are not retried automatically.
|
|
122
|
+
- Matching write or transaction failures clear the configured pools, report the event, and re-raise.
|
|
123
|
+
- Read-only transaction errors clear the configured failover roles to force ActiveRecord away from a bad primary connection.
|
|
124
|
+
|
|
125
|
+
## Supported Scope
|
|
126
|
+
|
|
127
|
+
This gem is PostgreSQL-only and currently targets ActiveRecord 7.x.
|
|
128
|
+
|
|
129
|
+
It patches ActiveRecord's PostgreSQL adapter methods with `Module#prepend`, so test it in staging before enabling it in production.
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecordPostgresRecovery
|
|
4
|
+
class Configuration
|
|
5
|
+
BOOLEAN_FALSE_VALUES = [false, nil, 'false', 'FALSE', '0', 0, 'no', 'NO', 'off', 'OFF'].freeze
|
|
6
|
+
|
|
7
|
+
attr_accessor :reporter, :error_patterns
|
|
8
|
+
attr_reader :enabled, :roles, :failover_clear_roles, :retry_read_queries, :max_retries
|
|
9
|
+
|
|
10
|
+
def initialize
|
|
11
|
+
self.enabled = true
|
|
12
|
+
@reporter = nil
|
|
13
|
+
self.roles = %i[writing reading]
|
|
14
|
+
self.failover_clear_roles = %i[writing]
|
|
15
|
+
self.retry_read_queries = true
|
|
16
|
+
self.max_retries = 1
|
|
17
|
+
@error_patterns = [
|
|
18
|
+
/PQconsumeInput\(\).*terminating connection due to administrator command.*SSL connection has been closed unexpectedly/im,
|
|
19
|
+
/PQsocket\(\) can't get socket descriptor/i,
|
|
20
|
+
/read-only transaction/i
|
|
21
|
+
]
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def enabled=(value)
|
|
25
|
+
@enabled = boolean(value)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def enabled?
|
|
29
|
+
enabled
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def retry_read_queries=(value)
|
|
33
|
+
@retry_read_queries = boolean(value)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def retry_read_queries?
|
|
37
|
+
retry_read_queries
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def max_retries=(value)
|
|
41
|
+
@max_retries = [Integer(value), 0].max
|
|
42
|
+
rescue ArgumentError, TypeError
|
|
43
|
+
raise ArgumentError, 'max_retries must be an integer greater than or equal to 0'
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def roles=(value)
|
|
47
|
+
@roles = normalize_roles(value)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def failover_clear_roles=(value)
|
|
51
|
+
@failover_clear_roles = normalize_roles(value)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
def boolean(value)
|
|
57
|
+
!BOOLEAN_FALSE_VALUES.include?(value)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def normalize_roles(value)
|
|
61
|
+
Array(value).map(&:to_sym)
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'active_record'
|
|
4
|
+
require 'pg'
|
|
5
|
+
require_relative 'recovery_event'
|
|
6
|
+
|
|
7
|
+
module ActiveRecordPostgresRecovery
|
|
8
|
+
module Handler
|
|
9
|
+
DB_CONNECTIVITY_ERROR_CLASSES = [PG::ConnectionBad, ActiveRecord::ConnectionNotEstablished].freeze
|
|
10
|
+
READ_ONLY_TRANSACTION_MESSAGE = /read-only transaction/i
|
|
11
|
+
RECOVERY_REPORTED_IVAR = :@active_record_postgres_recovery_reported
|
|
12
|
+
|
|
13
|
+
module_function
|
|
14
|
+
|
|
15
|
+
def db_connectivity_error?(error)
|
|
16
|
+
!matching_error(error).nil?
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def clear_active_connections!
|
|
20
|
+
roles = ActiveRecordPostgresRecovery.configuration.roles
|
|
21
|
+
handler = ActiveRecord::Base.connection_handler
|
|
22
|
+
roles.each { |role| handler.clear_active_connections!(role) }
|
|
23
|
+
build_clear_action(strategy: 'active', performed: true, roles: roles)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def clear_all_connections!(roles: ActiveRecordPostgresRecovery.configuration.roles)
|
|
27
|
+
handler = ActiveRecord::Base.connection_handler
|
|
28
|
+
roles.each { |role| handler.clear_all_connections!(role) }
|
|
29
|
+
build_clear_action(strategy: 'all', performed: true, roles: roles)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def clear_failover_connections!
|
|
33
|
+
roles = ActiveRecordPostgresRecovery.configuration.failover_clear_roles
|
|
34
|
+
build_clear_action(strategy: 'failover_all', performed: true, roles: roles).tap do
|
|
35
|
+
clear_all_connections!(roles: roles)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def read_only_transaction_error?(error)
|
|
40
|
+
!find_error_in_chain(error) { |current| current.message.to_s.match?(READ_ONLY_TRANSACTION_MESSAGE) }.nil?
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def report_attempted_recovery(context:, error:, source:, retrying: false, clear_action: nil)
|
|
44
|
+
report_recovery(context: context, error: error, source: source, outcome: :attempted, retrying: retrying, clear_action: clear_action)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def report_successful_recovery(context:, error:, source:, clear_action: nil)
|
|
48
|
+
report_recovery(context: context, error: error, source: source, outcome: :recovered, retrying: true, clear_action: clear_action)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def report_recovery(context:, error:, source:, outcome:, retrying: false, clear_action: nil)
|
|
52
|
+
return if recovery_reported?(error)
|
|
53
|
+
|
|
54
|
+
matched_error = matching_error(error) || error
|
|
55
|
+
clear_action ||= build_clear_action(strategy: nil, performed: false, roles: [])
|
|
56
|
+
|
|
57
|
+
ActiveRecordPostgresRecovery.report(
|
|
58
|
+
RecoveryEvent.new(
|
|
59
|
+
outcome: outcome,
|
|
60
|
+
source: source,
|
|
61
|
+
context: context,
|
|
62
|
+
error: error,
|
|
63
|
+
matched_error: matched_error,
|
|
64
|
+
retrying: retrying,
|
|
65
|
+
clear_action: clear_action
|
|
66
|
+
)
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
mark_recovery_reported!(error)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def matching_error(error)
|
|
73
|
+
find_error_in_chain(error) { |current| connection_error?(current) }
|
|
74
|
+
end
|
|
75
|
+
private_class_method :matching_error
|
|
76
|
+
|
|
77
|
+
def connection_error?(error)
|
|
78
|
+
DB_CONNECTIVITY_ERROR_CLASSES.any? { |klass| error.is_a?(klass) } || connection_error_message?(error.message)
|
|
79
|
+
end
|
|
80
|
+
private_class_method :connection_error?
|
|
81
|
+
|
|
82
|
+
def connection_error_message?(message)
|
|
83
|
+
ActiveRecordPostgresRecovery.configuration.error_patterns.any? { |pattern| message.to_s.match?(pattern) }
|
|
84
|
+
end
|
|
85
|
+
private_class_method :connection_error_message?
|
|
86
|
+
|
|
87
|
+
def find_error_in_chain(error)
|
|
88
|
+
current = error
|
|
89
|
+
|
|
90
|
+
while current
|
|
91
|
+
return current if yield(current)
|
|
92
|
+
|
|
93
|
+
current = current.cause
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
nil
|
|
97
|
+
end
|
|
98
|
+
private_class_method :find_error_in_chain
|
|
99
|
+
|
|
100
|
+
def build_clear_action(strategy:, performed:, roles:, skipped_reason: nil)
|
|
101
|
+
{
|
|
102
|
+
strategy: strategy,
|
|
103
|
+
performed: performed,
|
|
104
|
+
roles: roles,
|
|
105
|
+
skipped_reason: skipped_reason
|
|
106
|
+
}
|
|
107
|
+
end
|
|
108
|
+
private_class_method :build_clear_action
|
|
109
|
+
|
|
110
|
+
def recovery_reported?(error)
|
|
111
|
+
error.instance_variable_defined?(RECOVERY_REPORTED_IVAR) &&
|
|
112
|
+
error.instance_variable_get(RECOVERY_REPORTED_IVAR)
|
|
113
|
+
end
|
|
114
|
+
private_class_method :recovery_reported?
|
|
115
|
+
|
|
116
|
+
def mark_recovery_reported!(error)
|
|
117
|
+
error.instance_variable_set(RECOVERY_REPORTED_IVAR, true)
|
|
118
|
+
rescue FrozenError
|
|
119
|
+
nil
|
|
120
|
+
end
|
|
121
|
+
private_class_method :mark_recovery_reported!
|
|
122
|
+
end
|
|
123
|
+
end
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'active_record/connection_adapters/postgresql_adapter'
|
|
4
|
+
require_relative 'handler'
|
|
5
|
+
|
|
6
|
+
module ActiveRecordPostgresRecovery
|
|
7
|
+
module PostgresqlAdapterPatch
|
|
8
|
+
SOURCE = 'ActiveRecord'
|
|
9
|
+
QUERY_EXCEPTIONS = [ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished, PG::ConnectionBad].freeze
|
|
10
|
+
RECONNECT_EXCEPTIONS = [ActiveRecord::ConnectionNotEstablished, PG::ConnectionBad].freeze
|
|
11
|
+
|
|
12
|
+
def execute_and_clear(sql, name, binds, prepare: false, async: false, &block)
|
|
13
|
+
active_record_postgres_recovery_with_retry(sql, active_record_postgres_recovery_context(name)) do
|
|
14
|
+
super
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def query(sql, name = nil)
|
|
19
|
+
active_record_postgres_recovery_with_retry(sql, active_record_postgres_recovery_context(name)) do
|
|
20
|
+
super
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def execute(sql, name = nil)
|
|
25
|
+
active_record_postgres_recovery_with_retry(sql, active_record_postgres_recovery_context(name)) do
|
|
26
|
+
super
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def active_record_postgres_recovery_with_retry(sql, context)
|
|
33
|
+
retry_count = 0
|
|
34
|
+
recovery_error = nil
|
|
35
|
+
recovery_clear_action = nil
|
|
36
|
+
|
|
37
|
+
begin
|
|
38
|
+
result = yield
|
|
39
|
+
rescue *QUERY_EXCEPTIONS => e
|
|
40
|
+
raise unless ActiveRecordPostgresRecovery.configuration.enabled?
|
|
41
|
+
raise unless Handler.db_connectivity_error?(e)
|
|
42
|
+
|
|
43
|
+
if active_record_postgres_recovery_retryable_query?(sql, retry_count)
|
|
44
|
+
retry_count += 1
|
|
45
|
+
recovery_error = e
|
|
46
|
+
recovery_clear_action ||= active_record_postgres_recovery_clear_connections!(e)
|
|
47
|
+
active_record_postgres_recovery_reconnect!(context, clear_action: recovery_clear_action)
|
|
48
|
+
retry
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
active_record_postgres_recovery_report_attempted!(
|
|
52
|
+
context,
|
|
53
|
+
recovery_error || e,
|
|
54
|
+
retrying: retry_count.positive?,
|
|
55
|
+
clear_action: recovery_clear_action
|
|
56
|
+
)
|
|
57
|
+
raise
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
active_record_postgres_recovery_report_successful(context, recovery_error, recovery_clear_action) if recovery_error
|
|
61
|
+
|
|
62
|
+
result
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def active_record_postgres_recovery_reconnect!(context, clear_action: nil)
|
|
66
|
+
reconnect!
|
|
67
|
+
rescue *RECONNECT_EXCEPTIONS => e
|
|
68
|
+
raise unless Handler.db_connectivity_error?(e)
|
|
69
|
+
|
|
70
|
+
active_record_postgres_recovery_report_attempted!(context, e, retrying: true, clear_action: clear_action)
|
|
71
|
+
raise
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def active_record_postgres_recovery_report_attempted!(context, error, retrying:, clear_action: nil)
|
|
75
|
+
clear_action ||= active_record_postgres_recovery_clear_connections!(error)
|
|
76
|
+
|
|
77
|
+
Handler.report_attempted_recovery(
|
|
78
|
+
context: context,
|
|
79
|
+
error: error,
|
|
80
|
+
retrying: retrying,
|
|
81
|
+
source: SOURCE,
|
|
82
|
+
clear_action: clear_action
|
|
83
|
+
)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def active_record_postgres_recovery_report_successful(context, error, clear_action)
|
|
87
|
+
Handler.report_successful_recovery(context: context, error: error, source: SOURCE, clear_action: clear_action)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def active_record_postgres_recovery_retryable_query?(sql, retry_count)
|
|
91
|
+
ActiveRecordPostgresRecovery.configuration.retry_read_queries? &&
|
|
92
|
+
retry_count < ActiveRecordPostgresRecovery.configuration.max_retries &&
|
|
93
|
+
!transaction_open? &&
|
|
94
|
+
!write_query?(sql)
|
|
95
|
+
rescue StandardError
|
|
96
|
+
false
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def active_record_postgres_recovery_context(name)
|
|
100
|
+
"SQL #{name.respond_to?(:presence) ? name.presence : name || 'SQL'}"
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def active_record_postgres_recovery_clear_connections!(error)
|
|
104
|
+
if Handler.read_only_transaction_error?(error)
|
|
105
|
+
Handler.clear_failover_connections!
|
|
106
|
+
else
|
|
107
|
+
Handler.clear_all_connections!
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'rails/railtie'
|
|
4
|
+
require_relative 'postgresql_adapter_patch'
|
|
5
|
+
|
|
6
|
+
module ActiveRecordPostgresRecovery
|
|
7
|
+
class Railtie < Rails::Railtie
|
|
8
|
+
initializer 'active_record_postgres_recovery.patch_postgresql_adapter' do
|
|
9
|
+
ActiveSupport.on_load(:active_record) do
|
|
10
|
+
adapter = ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
|
|
11
|
+
next if adapter.ancestors.include?(ActiveRecordPostgresRecovery::PostgresqlAdapterPatch)
|
|
12
|
+
|
|
13
|
+
adapter.prepend(ActiveRecordPostgresRecovery::PostgresqlAdapterPatch)
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveRecordPostgresRecovery
|
|
4
|
+
RecoveryEvent = Struct.new(
|
|
5
|
+
:outcome,
|
|
6
|
+
:source,
|
|
7
|
+
:context,
|
|
8
|
+
:error,
|
|
9
|
+
:matched_error,
|
|
10
|
+
:retrying,
|
|
11
|
+
:clear_action,
|
|
12
|
+
keyword_init: true
|
|
13
|
+
) do
|
|
14
|
+
def to_h
|
|
15
|
+
{
|
|
16
|
+
outcome: outcome,
|
|
17
|
+
source: source,
|
|
18
|
+
context: context,
|
|
19
|
+
retrying: retrying,
|
|
20
|
+
clear_strategy: clear_action[:strategy],
|
|
21
|
+
clear_performed: clear_action[:performed],
|
|
22
|
+
cleared_roles: clear_action[:roles],
|
|
23
|
+
clear_skipped_reason: clear_action[:skipped_reason],
|
|
24
|
+
matched_error_class: matched_error.class.name,
|
|
25
|
+
matched_error_message: matched_error.message.to_s.lines.first.to_s.strip
|
|
26
|
+
}
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'handler'
|
|
4
|
+
|
|
5
|
+
module ActiveRecordPostgresRecovery
|
|
6
|
+
class SidekiqMiddleware
|
|
7
|
+
SOURCE = 'Sidekiq'
|
|
8
|
+
|
|
9
|
+
def call(worker, job, queue)
|
|
10
|
+
yield
|
|
11
|
+
rescue StandardError => e
|
|
12
|
+
raise unless ActiveRecordPostgresRecovery.configuration.enabled?
|
|
13
|
+
raise unless Handler.db_connectivity_error?(e)
|
|
14
|
+
|
|
15
|
+
clear_action = if Handler.read_only_transaction_error?(e)
|
|
16
|
+
Handler.clear_failover_connections!
|
|
17
|
+
else
|
|
18
|
+
Handler.clear_all_connections!
|
|
19
|
+
end
|
|
20
|
+
Handler.report_attempted_recovery(
|
|
21
|
+
context: sidekiq_context(worker, job, queue),
|
|
22
|
+
error: e,
|
|
23
|
+
source: SOURCE,
|
|
24
|
+
clear_action: clear_action
|
|
25
|
+
)
|
|
26
|
+
raise
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
def sidekiq_context(worker, job, queue)
|
|
32
|
+
"Sidekiq #{worker.class.name} jid=#{job['jid']} queue=#{queue}"
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'active_record_postgres_recovery/version'
|
|
4
|
+
require_relative 'active_record_postgres_recovery/configuration'
|
|
5
|
+
|
|
6
|
+
module ActiveRecordPostgresRecovery
|
|
7
|
+
class << self
|
|
8
|
+
attr_writer :configuration
|
|
9
|
+
|
|
10
|
+
def configuration
|
|
11
|
+
@configuration ||= Configuration.new
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def configure
|
|
15
|
+
yield(configuration)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def report(event)
|
|
19
|
+
configuration.reporter&.call(event)
|
|
20
|
+
rescue StandardError => e
|
|
21
|
+
report_reporter_failure(e, event)
|
|
22
|
+
nil
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
private
|
|
26
|
+
|
|
27
|
+
def report_reporter_failure(error, event)
|
|
28
|
+
message = "[active_record_postgres_recovery] reporter failed with #{error.class}: #{error.message} while reporting #{event.outcome} from #{event.source}"
|
|
29
|
+
|
|
30
|
+
if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
|
|
31
|
+
Rails.logger.warn(message)
|
|
32
|
+
else
|
|
33
|
+
warn(message)
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
require_relative 'active_record_postgres_recovery/railtie' if defined?(Rails::Railtie)
|
metadata
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: active_record_postgres_recovery
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Hassan
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: bin
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2026-06-24 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: activerecord
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - ">="
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '7.0'
|
|
20
|
+
- - "<"
|
|
21
|
+
- !ruby/object:Gem::Version
|
|
22
|
+
version: '8.0'
|
|
23
|
+
type: :runtime
|
|
24
|
+
prerelease: false
|
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
26
|
+
requirements:
|
|
27
|
+
- - ">="
|
|
28
|
+
- !ruby/object:Gem::Version
|
|
29
|
+
version: '7.0'
|
|
30
|
+
- - "<"
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: '8.0'
|
|
33
|
+
- !ruby/object:Gem::Dependency
|
|
34
|
+
name: activesupport
|
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - ">="
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: '7.0'
|
|
40
|
+
- - "<"
|
|
41
|
+
- !ruby/object:Gem::Version
|
|
42
|
+
version: '8.0'
|
|
43
|
+
type: :runtime
|
|
44
|
+
prerelease: false
|
|
45
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
46
|
+
requirements:
|
|
47
|
+
- - ">="
|
|
48
|
+
- !ruby/object:Gem::Version
|
|
49
|
+
version: '7.0'
|
|
50
|
+
- - "<"
|
|
51
|
+
- !ruby/object:Gem::Version
|
|
52
|
+
version: '8.0'
|
|
53
|
+
- !ruby/object:Gem::Dependency
|
|
54
|
+
name: pg
|
|
55
|
+
requirement: !ruby/object:Gem::Requirement
|
|
56
|
+
requirements:
|
|
57
|
+
- - ">="
|
|
58
|
+
- !ruby/object:Gem::Version
|
|
59
|
+
version: '1.5'
|
|
60
|
+
- - "<"
|
|
61
|
+
- !ruby/object:Gem::Version
|
|
62
|
+
version: '2.0'
|
|
63
|
+
type: :runtime
|
|
64
|
+
prerelease: false
|
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
66
|
+
requirements:
|
|
67
|
+
- - ">="
|
|
68
|
+
- !ruby/object:Gem::Version
|
|
69
|
+
version: '1.5'
|
|
70
|
+
- - "<"
|
|
71
|
+
- !ruby/object:Gem::Version
|
|
72
|
+
version: '2.0'
|
|
73
|
+
- !ruby/object:Gem::Dependency
|
|
74
|
+
name: rake
|
|
75
|
+
requirement: !ruby/object:Gem::Requirement
|
|
76
|
+
requirements:
|
|
77
|
+
- - "~>"
|
|
78
|
+
- !ruby/object:Gem::Version
|
|
79
|
+
version: '13.0'
|
|
80
|
+
type: :development
|
|
81
|
+
prerelease: false
|
|
82
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
83
|
+
requirements:
|
|
84
|
+
- - "~>"
|
|
85
|
+
- !ruby/object:Gem::Version
|
|
86
|
+
version: '13.0'
|
|
87
|
+
- !ruby/object:Gem::Dependency
|
|
88
|
+
name: rspec
|
|
89
|
+
requirement: !ruby/object:Gem::Requirement
|
|
90
|
+
requirements:
|
|
91
|
+
- - "~>"
|
|
92
|
+
- !ruby/object:Gem::Version
|
|
93
|
+
version: '3.13'
|
|
94
|
+
type: :development
|
|
95
|
+
prerelease: false
|
|
96
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
97
|
+
requirements:
|
|
98
|
+
- - "~>"
|
|
99
|
+
- !ruby/object:Gem::Version
|
|
100
|
+
version: '3.13'
|
|
101
|
+
description: Retries safe read queries after stale PostgreSQL connection failures,
|
|
102
|
+
clears ActiveRecord connection pools, and exposes recovery events for observability.
|
|
103
|
+
email:
|
|
104
|
+
- m.hassanror@gmail.com
|
|
105
|
+
executables: []
|
|
106
|
+
extensions: []
|
|
107
|
+
extra_rdoc_files: []
|
|
108
|
+
files:
|
|
109
|
+
- LICENSE.txt
|
|
110
|
+
- README.md
|
|
111
|
+
- lib/active_record_postgres_recovery.rb
|
|
112
|
+
- lib/active_record_postgres_recovery/configuration.rb
|
|
113
|
+
- lib/active_record_postgres_recovery/handler.rb
|
|
114
|
+
- lib/active_record_postgres_recovery/postgresql_adapter_patch.rb
|
|
115
|
+
- lib/active_record_postgres_recovery/railtie.rb
|
|
116
|
+
- lib/active_record_postgres_recovery/recovery_event.rb
|
|
117
|
+
- lib/active_record_postgres_recovery/sidekiq_middleware.rb
|
|
118
|
+
- lib/active_record_postgres_recovery/version.rb
|
|
119
|
+
homepage: https://github.com/your-github/active_record_postgres_recovery
|
|
120
|
+
licenses:
|
|
121
|
+
- MIT
|
|
122
|
+
metadata:
|
|
123
|
+
homepage_uri: https://github.com/your-github/active_record_postgres_recovery
|
|
124
|
+
source_code_uri: https://github.com/your-github/active_record_postgres_recovery
|
|
125
|
+
changelog_uri: https://github.com/your-github/active_record_postgres_recovery/releases
|
|
126
|
+
rubygems_mfa_required: 'true'
|
|
127
|
+
post_install_message:
|
|
128
|
+
rdoc_options: []
|
|
129
|
+
require_paths:
|
|
130
|
+
- lib
|
|
131
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
132
|
+
requirements:
|
|
133
|
+
- - ">="
|
|
134
|
+
- !ruby/object:Gem::Version
|
|
135
|
+
version: '3.1'
|
|
136
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
137
|
+
requirements:
|
|
138
|
+
- - ">="
|
|
139
|
+
- !ruby/object:Gem::Version
|
|
140
|
+
version: '0'
|
|
141
|
+
requirements: []
|
|
142
|
+
rubygems_version: 3.3.26
|
|
143
|
+
signing_key:
|
|
144
|
+
specification_version: 4
|
|
145
|
+
summary: Safe PostgreSQL connection recovery for Rails ActiveRecord apps.
|
|
146
|
+
test_files: []
|