mongo_ha 1.12.0 → 1.12.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/mongo_ha.rb +5 -7
- data/lib/mongo_ha/mongo_client.rb +25 -39
- data/lib/mongo_ha/version.rb +1 -1
- data/test/readme.md +48 -0
- metadata +8 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 24001e509a2bf4c267e910c0086d4600092aeeff
|
4
|
+
data.tar.gz: 4af9185eaf2cdcad47e53786c4c27776702734c7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4eb4feae513d86c118d01b00d0e274a9668d551f83e456ea9abf946d05080800d0fddb1c20faa62dde4aa7d8b012dd845268c5d2aa8c60670aba31a442f6efbb
|
7
|
+
data.tar.gz: 0cd992c13c58cd09b7f32168c21189ef86456a4bba5546ebbae1530a0879d4954b6a8e9c9315f294b3dd742e59defe346b2bbb80c596f3e65b4b357a4bee1f07
|
data/README.md
CHANGED
@@ -153,7 +153,7 @@ Using the above default values, will result in retry connects at the following i
|
|
153
153
|
|
154
154
|
There is really only one place to test something like `mongo_ha` and that is in
|
155
155
|
a high volume mission critical production environment.
|
156
|
-
|
156
|
+
This gem was created and tested with MongoDB running in an
|
157
157
|
enterprise production environment with hundreds of connections to Mongo servers
|
158
158
|
in remote data centers across a WAN. It adds high availability to standalone
|
159
159
|
MongoDB servers, replica-sets, and sharded clusters.
|
data/lib/mongo_ha.rb
CHANGED
@@ -7,14 +7,12 @@ Mongo::MongoClient.send(:include, MongoHA::MongoClient::InstanceMethods)
|
|
7
7
|
|
8
8
|
# Wrap critical Mongo methods with retry_on_connection_failure
|
9
9
|
{
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
:options, :stats, :map_reduce
|
14
|
-
],
|
15
|
-
Mongo::CollectionOperationWriter => [:send_write_operation, :batch_message_send],
|
16
|
-
Mongo::CollectionCommandWriter => [:send_write_command, :batch_message_send]
|
10
|
+
# Most calls use a cursor under the covers to return the result
|
11
|
+
# If the primary is lost and it connects to a different server an expired cursor exception is raised
|
12
|
+
Mongo::Cursor => [:refresh],
|
17
13
|
|
14
|
+
# These methods do not use a Cursor
|
15
|
+
Mongo::Collection => [:insert, :remove, :update]
|
18
16
|
}.each_pair do |klass, methods|
|
19
17
|
methods.each do |method|
|
20
18
|
original_method = "#{method}_original".to_sym
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'mongo'
|
2
2
|
module MongoHA
|
3
3
|
module MongoClient
|
4
|
-
CONNECTION_RETRY_OPTS
|
4
|
+
CONNECTION_RETRY_OPTS = [:reconnect_attempts, :reconnect_retry_seconds, :reconnect_retry_multiplier, :reconnect_max_retry_seconds]
|
5
5
|
|
6
6
|
# The following errors occur when mongos cannot connect to the shard
|
7
7
|
# They require a retry to resolve them
|
@@ -50,8 +50,6 @@ module MongoHA
|
|
50
50
|
end
|
51
51
|
end
|
52
52
|
|
53
|
-
alias_method :receive_message_original, :receive_message
|
54
|
-
alias_method :connect_original, :connect
|
55
53
|
alias_method :valid_opts_original, :valid_opts
|
56
54
|
alias_method :setup_original, :setup
|
57
55
|
|
@@ -60,22 +58,6 @@ module MongoHA
|
|
60
58
|
# Prevent multiple threads from trying to reconnect at the same time during
|
61
59
|
# connection failures
|
62
60
|
@@failover_mutex = Mutex.new
|
63
|
-
# Wrap internal networking calls with retry logic
|
64
|
-
|
65
|
-
# Do not stub out :send_message_with_gle or :send_message
|
66
|
-
# It modifies the message, see CollectionWriter#send_write_operation
|
67
|
-
|
68
|
-
def receive_message(*args)
|
69
|
-
retry_on_connection_failure do
|
70
|
-
receive_message_original *args
|
71
|
-
end
|
72
|
-
end
|
73
|
-
|
74
|
-
def connect(*args)
|
75
|
-
retry_on_connection_failure do
|
76
|
-
connect_original *args
|
77
|
-
end
|
78
|
-
end
|
79
61
|
|
80
62
|
protected
|
81
63
|
|
@@ -103,19 +85,21 @@ module MongoHA
|
|
103
85
|
# Example:
|
104
86
|
# connection.retry_on_connection_failure { |retried| connection.ping }
|
105
87
|
def retry_on_connection_failure(&block)
|
106
|
-
raise
|
107
|
-
|
88
|
+
raise 'Missing mandatory block parameter on call to Mongo::Connection#retry_on_connection_failure' unless block
|
89
|
+
# No need to double retry calls
|
90
|
+
return block.call(false) if Thread.current[:mongo_ha_active?]
|
91
|
+
retried = false
|
108
92
|
mongos_retries = 0
|
109
93
|
begin
|
110
|
-
|
111
|
-
|
94
|
+
Thread.current[:mongo_ha_active?] = true
|
95
|
+
result = block.call(retried)
|
96
|
+
retried = false
|
112
97
|
result
|
113
98
|
rescue Mongo::ConnectionFailure => exc
|
114
99
|
# Retry if reconnected, but only once to prevent an infinite loop
|
115
100
|
logger.warn "Connection Failure: '#{exc.message}' [#{exc.error_code}]"
|
116
101
|
if !retried && _reconnect
|
117
102
|
retried = true
|
118
|
-
# TODO There has to be a way to flush the connection pool of all inactive connections
|
119
103
|
retry
|
120
104
|
end
|
121
105
|
raise exc
|
@@ -125,7 +109,7 @@ module MongoHA
|
|
125
109
|
# it sometimes gets: "not master and slaveok=false"
|
126
110
|
if exc.result
|
127
111
|
error = exc.result['err'] || exc.result['errmsg']
|
128
|
-
close if error && error.include?(
|
112
|
+
close if error && error.include?('not master')
|
129
113
|
end
|
130
114
|
|
131
115
|
# These get returned when connected to a local mongos router when it in turn
|
@@ -140,13 +124,16 @@ module MongoHA
|
|
140
124
|
retried = true
|
141
125
|
Kernel.sleep(0.5)
|
142
126
|
logger.warn "[#{primary.inspect}] Router Connection Failure. Retry ##{mongos_retries}. Exc: '#{exc.message}' [#{exc.error_code}]"
|
143
|
-
# TODO Is there a way to flush the connection pool of all inactive connections
|
144
127
|
retry
|
145
128
|
end
|
146
129
|
raise exc
|
130
|
+
ensure
|
131
|
+
Thread.current[:mongo_ha_active?] = false
|
147
132
|
end
|
148
133
|
end
|
149
134
|
|
135
|
+
protected
|
136
|
+
|
150
137
|
# Call this method whenever a Mongo::ConnectionFailure Exception
|
151
138
|
# has been raised to re-establish the connection
|
152
139
|
#
|
@@ -155,35 +142,35 @@ module MongoHA
|
|
155
142
|
#
|
156
143
|
# Returns whether the connection is connected again
|
157
144
|
def _reconnect
|
158
|
-
logger.debug
|
145
|
+
logger.debug 'Going to reconnect'
|
159
146
|
|
160
147
|
# Prevent other threads from invoking reconnect logic at the same time
|
161
148
|
@@failover_mutex.synchronize do
|
162
149
|
# Another thread may have already failed over the connection by the
|
163
150
|
# time this threads gets in
|
151
|
+
begin
|
152
|
+
ping
|
153
|
+
rescue Mongo::ConnectionFailure
|
154
|
+
# Connection still not available, run code below
|
155
|
+
end
|
156
|
+
|
164
157
|
if active?
|
165
158
|
logger.info "Connected to: #{primary.inspect}"
|
166
159
|
return true
|
167
160
|
end
|
168
161
|
|
169
|
-
# Close all sockets that are not checked out so that other threads not
|
170
|
-
# currently waiting on Mongo, don't get bad connections and have to
|
171
|
-
# retry each one in turn
|
172
|
-
@primary_pool.close if @primary_pool
|
173
|
-
|
174
162
|
if reconnect_attempts > 0
|
175
163
|
# Wait for other threads to finish working on their sockets
|
176
|
-
retries
|
164
|
+
retries = 1
|
177
165
|
retry_seconds = reconnect_retry_seconds
|
178
166
|
begin
|
179
167
|
logger.warn "Connection unavailable. Waiting: #{retry_seconds} seconds before retrying"
|
180
168
|
sleep retry_seconds
|
181
|
-
|
182
|
-
connect_original
|
169
|
+
ping
|
183
170
|
rescue Mongo::ConnectionFailure => exc
|
184
171
|
if retries < reconnect_attempts
|
185
|
-
retries
|
186
|
-
retry_seconds *=
|
172
|
+
retries += 1
|
173
|
+
retry_seconds *= reconnect_retry_multiplier
|
187
174
|
retry_seconds = reconnect_max_retry_seconds if retry_seconds > reconnect_max_retry_seconds
|
188
175
|
retry
|
189
176
|
end
|
@@ -195,9 +182,8 @@ module MongoHA
|
|
195
182
|
end
|
196
183
|
connected?
|
197
184
|
end
|
198
|
-
|
199
185
|
end
|
200
186
|
|
201
187
|
end
|
202
188
|
end
|
203
|
-
end
|
189
|
+
end
|
data/lib/mongo_ha/version.rb
CHANGED
data/test/readme.md
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
## Testing
|
2
|
+
|
3
|
+
Unfortunately the only way to properly test mongo_ha is to startup a console with connections
|
4
|
+
active and to stop / restart the mongo servers in the replicaset as follows:
|
5
|
+
|
6
|
+
|
7
|
+
#### Run the following code in a console
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
collection = Cache::Identity.database['test']
|
11
|
+
collection.drop
|
12
|
+
threads = 5.times.collect do |i|
|
13
|
+
Thread.new do
|
14
|
+
100.times do |j|
|
15
|
+
1_000.times do |k|
|
16
|
+
collection.insert(_id: "#{i}-#{j}-#{k}")
|
17
|
+
collection.find_one(_id: "#{i}-#{j}-#{k}")
|
18
|
+
puts("#{i}-#{j}-#{k}") if k % 1000 == 0
|
19
|
+
end
|
20
|
+
puts "#{i}-#{j} pausing"
|
21
|
+
sleep 5
|
22
|
+
end
|
23
|
+
puts "#{i} Complete"
|
24
|
+
end
|
25
|
+
end
|
26
|
+
```
|
27
|
+
|
28
|
+
#### Steps
|
29
|
+
|
30
|
+
While running the above code in the console
|
31
|
+
|
32
|
+
* Stop 1 slave server
|
33
|
+
|
34
|
+
Nothing should appear in the logs and everything should process fine
|
35
|
+
|
36
|
+
* Stop another slave
|
37
|
+
|
38
|
+
The logs should show retries
|
39
|
+
|
40
|
+
* Start up one of the 2 slaves that were stopped
|
41
|
+
|
42
|
+
The processing should resume successfully
|
43
|
+
|
44
|
+
#### To stop the test
|
45
|
+
|
46
|
+
```ruby
|
47
|
+
threads.each(&:kill)
|
48
|
+
```
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mongo_ha
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.12.
|
4
|
+
version: 1.12.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Reid Morrison
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-09-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mongo
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '1.
|
19
|
+
version: '1.10'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '1.
|
26
|
+
version: '1.10'
|
27
27
|
description: Automatic reconnects and recovery when replica-set changes, or connections
|
28
28
|
are lost, with transparent recovery
|
29
29
|
email:
|
@@ -37,6 +37,7 @@ files:
|
|
37
37
|
- lib/mongo_ha.rb
|
38
38
|
- lib/mongo_ha/mongo_client.rb
|
39
39
|
- lib/mongo_ha/version.rb
|
40
|
+
- test/readme.md
|
40
41
|
homepage: https://github.com/reidmorrison/mongo_ha
|
41
42
|
licenses:
|
42
43
|
- Apache License V2.0
|
@@ -57,8 +58,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
57
58
|
version: '0'
|
58
59
|
requirements: []
|
59
60
|
rubyforge_project:
|
60
|
-
rubygems_version: 2.4.5
|
61
|
+
rubygems_version: 2.4.5.1
|
61
62
|
signing_key:
|
62
63
|
specification_version: 4
|
63
64
|
summary: High availability for the mongo ruby driver
|
64
|
-
test_files:
|
65
|
+
test_files:
|
66
|
+
- test/readme.md
|