sp-nat-monitor 1.0.9

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 5ded3db6cc435026f25c6276fbe1d6a0fdff521a
4
+ data.tar.gz: d57e17569e4f95473c312fa544fdff62fa9953fb
5
+ SHA512:
6
+ metadata.gz: c823c3af135b0cd57f622bf3a3c784d701f8947cb254f32410c3ad93d11a78cd6553932c3cfb33067add7f05dea3cadf5008ea2106d275ee11800bd69046c9e3
7
+ data.tar.gz: 33cbd2c344b42165bb391bbfcab2da8a943e177102c33c84367980250532ff0b51ff621220545b1a92baa4d2795c2f5996acdcedb8bd58c410fb080b05ab3d18
data/.gitignore ADDED
@@ -0,0 +1,14 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
10
+ *.bundle
11
+ *.so
12
+ *.o
13
+ *.a
14
+ mkmf.log
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in nat-monitor.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2015 Eric Herot
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,55 @@
1
+ # nat-monitor
2
+
3
+ Monitors a quorum of NAT servers for an outage and reassigns the specified EC2 route table to point to a working server.
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ ```ruby
10
+ gem 'nat-monitor'
11
+ ```
12
+
13
+ And then execute:
14
+
15
+ $ bundle
16
+
17
+ Or install it yourself as:
18
+
19
+ $ gem install nat-monitor
20
+
21
+ ## Usage
22
+
23
+ Run it as a service after creating the configuration YAML file.
24
+
25
+ E.g.:
26
+
27
+ $ nat-monitor [OPTIONAL CONF_FILE]
28
+
29
+ By default it will check `/etc/nat_monitor.yml` for its configuration.
30
+
31
+ ## Example Configuration
32
+
33
+ ```yaml
34
+ ---
35
+ route_table_id: rtb-00000001
36
+ nodes:
37
+ i-00000001: 10.0.0.1
38
+ i-00000002: 10.0.1.1
39
+ i-00000003: 10.0.2.1
40
+ ```
41
+
42
+ Optional properties include:
43
+ ```yaml
44
+ pings: 3
45
+ ping_timeout: 1
46
+ heartbeat_interval: 10
47
+ ```
48
+
49
+ ## Contributing
50
+
51
+ 1. Fork it ( https://github.com/[my-github-username]/nat-monitor/fork )
52
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
53
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
54
+ 4. Push to the branch (`git push origin my-new-feature`)
55
+ 5. Create a new Pull Request
data/Rakefile ADDED
@@ -0,0 +1 @@
1
+ require 'bundler/gem_tasks'
data/bin/nat-monitor ADDED
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'nat-monitor'
4
+
5
+ EtTools::NatMonitor.new(ARGV[0]).run
@@ -0,0 +1,5 @@
1
+ module EtTools
2
+ class NatMonitor
3
+ VERSION = '1.0.9'
4
+ end
5
+ end
@@ -0,0 +1,160 @@
1
+ module EtTools
2
+ class NatMonitor
3
+ require 'net/http'
4
+ require 'net/ping'
5
+ require 'fog'
6
+ require 'yaml'
7
+ require 'syslog'
8
+
9
+ def initialize(conf_file = nil)
10
+ @conf = defaults.merge load_conf(conf_file)
11
+ end
12
+
13
+ def load_conf(conf_file = nil)
14
+ YAML.load_file(conf_file || '/etc/nat_monitor.yml')
15
+ end
16
+
17
+ def run
18
+ validate!
19
+ output 'Starting NAT Monitor'
20
+ main_loop
21
+ end
22
+
23
+ def validate!
24
+ case
25
+ when !@conf['route_table_id']
26
+ output 'route_table_id not specified'
27
+ exit 1
28
+ when !route_exists?(@conf['route_table_id'])
29
+ output "Route #{@conf['route_table_id']} not found"
30
+ exit 2
31
+ when @conf['nodes'].count < 2
32
+ output '2 or more nodes are required to create a quorum'
33
+ exit 3
34
+ end
35
+ end
36
+
37
+ def defaults
38
+ { 'pings' => 3,
39
+ 'ping_timeout' => 1,
40
+ 'heartbeat_interval' => 10 }
41
+ end
42
+
43
+ def main_loop
44
+ loop do
45
+ begin
46
+ heartbeat
47
+ rescue => e
48
+ output "Caught #{e.class} exception: #{e.message}"
49
+ output e.backtrace
50
+ end
51
+ sleep @conf['heartbeat_interval']
52
+ end
53
+ end
54
+
55
+ def heartbeat
56
+ if am_i_master?
57
+ output "Looks like I'm the master"
58
+ return
59
+ end
60
+ un = unreachable_nodes
61
+ return if un.empty?
62
+ if un.count == other_nodes.keys.count # return if I'm unreachable...
63
+ output "No nodes are reachable. Seems I'm the unreachable one."
64
+ return
65
+ end
66
+ cm = current_master
67
+ unless un.include?(cm) # ...unless master is unreachable
68
+ output "Unreachable nodes: #{un.inspect}"
69
+ output "Current master (#{cm}) is still reachable"
70
+ return
71
+ end
72
+ steal_route
73
+ end
74
+
75
+ def steal_route
76
+ output 'Stealing route 0.0.0.0/0 on route table ' \
77
+ "#{@conf['route_table_id']}"
78
+ return if @conf['mocking']
79
+ connection.replace_route(
80
+ @conf['route_table_id'],
81
+ '0.0.0.0/0',
82
+ 'InstanceId' => my_instance_id
83
+ )
84
+ end
85
+
86
+ def unreachable_nodes
87
+ other_nodes.select { |_node, ip| !pingable?(ip) }
88
+ end
89
+
90
+ def other_nodes
91
+ @other_nodes ||= begin
92
+ nodes = @conf['nodes'].dup
93
+ nodes.delete my_instance_id
94
+ nodes
95
+ end
96
+ end
97
+
98
+ def pingable?(ip)
99
+ p = Net::Ping::External.new(ip)
100
+ p.timeout = @conf['ping_timeout']
101
+ p.ping?
102
+ end
103
+
104
+ def route_exists?(route_id)
105
+ connection.route_tables.map(&:id).include? route_id
106
+ end
107
+
108
+ def connection
109
+ @connection ||= begin
110
+ if @conf['aws_access_key_id']
111
+ options = { aws_access_key_id: @conf['aws_access_key_id'],
112
+ aws_secret_access_key: @conf['aws_secret_access_key'] }
113
+ else
114
+ options = { use_iam_profile: true }
115
+ end
116
+
117
+ options[:endpoint] = @conf['aws_url'] if @conf['aws_url']
118
+ Fog::Compute::AWS.new(options)
119
+ end
120
+ end
121
+
122
+ def current_master
123
+ default_r =
124
+ connection.route_tables.get(@conf['route_table_id']).routes.find do |r|
125
+ r['destinationCidrBlock'] == '0.0.0.0/0'
126
+ end
127
+ default_r['instanceId']
128
+ end
129
+
130
+ def my_instance_id
131
+ @my_instance_id ||= begin
132
+ Net::HTTP.get(
133
+ '169.254.169.254',
134
+ '/latest/meta-data/instance-id'
135
+ )
136
+ end
137
+ end
138
+
139
+ def am_i_master?
140
+ master_node? my_instance_id
141
+ end
142
+
143
+ def master_node?(node_id)
144
+ current_master == node_id
145
+ end
146
+
147
+ private
148
+
149
+ def output(message)
150
+ puts message
151
+ log message
152
+ end
153
+
154
+ def log(message, level = 'info')
155
+ Syslog.open('nat-monitor', Syslog::LOG_PID | Syslog::LOG_CONS) do |s|
156
+ s.send(level, message)
157
+ end
158
+ end
159
+ end
160
+ end
@@ -0,0 +1,29 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'nat-monitor/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'sp-nat-monitor'
8
+ spec.version = EtTools::NatMonitor::VERSION
9
+ spec.authors = ['Eric Herot']
10
+ spec.email = ['eric.github@herot.com']
11
+ spec.summary = 'A service for providing an HA NAT in EC2'
12
+ spec.description = spec.summary
13
+ spec.homepage = ''
14
+ spec.license = 'Apache 2.0'
15
+
16
+ spec.files = `git ls-files -z`.split("\x0")
17
+ spec.executables = spec.files.grep(/^bin\//) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(/^(test|spec|features)\//)
19
+ spec.require_paths = ['lib']
20
+
21
+ spec.add_development_dependency 'rspec'
22
+ spec.add_development_dependency 'simplecov'
23
+ spec.add_development_dependency 'byebug'
24
+ spec.add_development_dependency 'bundler', '~> 1.7'
25
+ spec.add_development_dependency 'rake', '~> 10.0'
26
+
27
+ spec.add_runtime_dependency 'net-ping'
28
+ spec.add_runtime_dependency 'fog', '~> 1.23'
29
+ end
@@ -0,0 +1,197 @@
1
+ require 'spec_helper'
2
+ require 'byebug'
3
+ require 'nat-monitor'
4
+
5
+ describe EtTools::NatMonitor do
6
+ before(:each) do
7
+ @route_table_id = 'rtb-00000000'
8
+ @my_instance_id = 'i-00000001'
9
+
10
+ @other_nodes = { 'i-00000002' => '1.1.1.2',
11
+ 'i-00000003' => '1.1.1.3' }
12
+
13
+ @yaml_conf = { 'route_table_id' => @route_table_id,
14
+ 'aws_access_key_id' => 'AWS_ACCESS_KEY_ID',
15
+ 'aws_secret_access_key' => 'AWS_SECRET_ACCESS_KEY',
16
+ 'nodes' => (
17
+ { @my_instance_id => '1.1.1.1' }
18
+ ).merge(@other_nodes) }
19
+
20
+ filepath = 'bogus_filename.yml'
21
+
22
+ allow_any_instance_of(EtTools::NatMonitor).to receive(:load_conf)
23
+ .with(filepath).and_return(@yaml_conf)
24
+
25
+ @nat_monitor = EtTools::NatMonitor.new(filepath)
26
+
27
+ @defaults = { 'pings' => 3,
28
+ 'ping_timeout' => 1,
29
+ 'heartbeat_interval' => 10 }
30
+
31
+ allow(@nat_monitor).to receive(:my_instance_id).and_return(@my_instance_id)
32
+ end
33
+
34
+ context 'fewer than 3 nodes are specified' do
35
+ before do
36
+ allow(YAML).to receive(:load_file).with(any_args)
37
+ @nat_monitor.instance_variable_set(
38
+ :@conf,
39
+ ({ 'route_table_id' => @route_table_id,
40
+ 'aws_access_key_id' => 'AWS_ACCESS_KEY_ID',
41
+ 'aws_secret_access_key' => 'AWS_SECRET_ACCESS_KEY',
42
+ 'nodes' => @other_nodes })
43
+ )
44
+ allow(@nat_monitor).to receive(:route_exists?).with(any_args)
45
+ .and_return true
46
+ end
47
+
48
+ it 'exits with status 3' do
49
+ expect(@nat_monitor).to receive(:exit).with(3)
50
+ @nat_monitor.validate!
51
+ end
52
+ end
53
+
54
+ context 'invalid route is specified' do
55
+ before do
56
+ expect(@nat_monitor.connection).to receive(:route_tables)
57
+ .and_return [
58
+ double('route_tables',
59
+ id: 'rtb-99999999')
60
+ ]
61
+ end
62
+
63
+ it 'exits with status 2' do
64
+ expect(@nat_monitor).to receive(:exit).with(2)
65
+ @nat_monitor.validate!
66
+ end
67
+ end
68
+
69
+ context 'no route is specified' do
70
+ before do
71
+ @nat_monitor.instance_variable_set(
72
+ :@conf,
73
+ 'nodes' => ({ @my_instance_id => '1.1.1.1' }).merge(@other_nodes)
74
+ )
75
+ end
76
+
77
+ it 'exits with status 1' do
78
+ expect(@nat_monitor).to receive(:exit).with(1)
79
+ @nat_monitor.validate!
80
+ end
81
+ end
82
+
83
+ context 'local node is the master' do
84
+ before do
85
+ allow(@nat_monitor).to(
86
+ receive(:current_master).and_return(@my_instance_id)
87
+ )
88
+ end
89
+
90
+ it 'sets @conf correctly' do
91
+ expect(@nat_monitor.instance_variable_get(:@conf)).to eq(
92
+ @yaml_conf.merge(@defaults)
93
+ )
94
+ end
95
+
96
+ it 'and knows that it is master' do
97
+ expect(@nat_monitor).to receive(:am_i_master?).and_return(true)
98
+ @nat_monitor.heartbeat
99
+ end
100
+
101
+ it 'and does not check for unreachable nodes' do
102
+ expect(@nat_monitor).to_not receive(:unreachable_nodes)
103
+ @nat_monitor.heartbeat
104
+ end
105
+ end
106
+
107
+ context 'local node is not master' do
108
+ before do
109
+ allow(@nat_monitor).to(
110
+ receive(:current_master).and_return('i-00000002')
111
+ )
112
+ end
113
+
114
+ context 'and can ping everything' do
115
+ before do
116
+ allow(@nat_monitor).to receive(:pingable?).with(any_args)
117
+ .and_return true
118
+ end
119
+
120
+ it 'does not try to steal the route' do
121
+ expect(@nat_monitor).to_not receive(:steal_route)
122
+ @nat_monitor.heartbeat
123
+ end
124
+ end
125
+
126
+ context 'and can\'t ping anything' do
127
+ before do
128
+ allow(@nat_monitor).to receive(:pingable?).with(any_args)
129
+ .and_return false
130
+ end
131
+
132
+ it 'counts unreachable nodes correctly' do
133
+ expect(@nat_monitor).to receive(:unreachable_nodes)
134
+ .and_return(@other_nodes)
135
+ @nat_monitor.heartbeat
136
+ end
137
+
138
+ it 'does not try to steal the route' do
139
+ expect(@nat_monitor).to receive(:unreachable_nodes)
140
+ .and_return(@other_nodes)
141
+ expect(@nat_monitor).to_not receive(:steal_route)
142
+ @nat_monitor.heartbeat
143
+ end
144
+ end
145
+
146
+ context 'and only can\'t ping the master' do
147
+ before do
148
+ allow(@nat_monitor).to receive(:pingable?).with('1.1.1.2')
149
+ .and_return false
150
+ allow(@nat_monitor).to receive(:pingable?).with('1.1.1.3')
151
+ .and_return true
152
+ end
153
+
154
+ it 'computes the list of other nodes correctly' do
155
+ allow(@nat_monitor).to receive(:steal_route).and_return true
156
+ expect(@nat_monitor).to receive(:other_nodes)
157
+ .exactly(2).times.and_return(@other_nodes)
158
+ @nat_monitor.heartbeat
159
+ end
160
+
161
+ it 'finds i-00000002 unreachable' do
162
+ allow(@nat_monitor).to receive(:steal_route).and_return true
163
+ expect(@nat_monitor).to receive(:unreachable_nodes)
164
+ .and_return('i-00000002' => '1.1.1.2')
165
+ @nat_monitor.heartbeat
166
+ end
167
+
168
+ it 'tries to steal the route' do
169
+ expect(@nat_monitor).to receive(:steal_route)
170
+ @nat_monitor.heartbeat
171
+ end
172
+
173
+ it 'sends correct replace route command' do
174
+ expect(@nat_monitor.connection).to receive(:replace_route)
175
+ .with(@route_table_id,
176
+ '0.0.0.0/0',
177
+ 'InstanceId' => @my_instance_id)
178
+ .and_return true
179
+ @nat_monitor.heartbeat
180
+ end
181
+ end
182
+
183
+ context 'mocking and can\'t ping the master' do
184
+ before do
185
+ @nat_monitor.instance_variable_set(
186
+ :@conf,
187
+ @yaml_conf.merge('mocking' => true).merge(@defaults)
188
+ )
189
+ end
190
+
191
+ it 'does not steal the route when mocking is enabled' do
192
+ expect(@nat_monitor.connection).to_not receive(:replace_route)
193
+ @nat_monitor.heartbeat
194
+ end
195
+ end
196
+ end
197
+ end
@@ -0,0 +1,4 @@
1
+ require 'rspec'
2
+ require 'simplecov'
3
+
4
+ SimpleCov.start
metadata ADDED
@@ -0,0 +1,157 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: sp-nat-monitor
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.9
5
+ platform: ruby
6
+ authors:
7
+ - Eric Herot
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2015-05-27 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: rspec
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: simplecov
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: byebug
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: bundler
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '1.7'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '1.7'
69
+ - !ruby/object:Gem::Dependency
70
+ name: rake
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '10.0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '10.0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: net-ping
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :runtime
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: fog
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - "~>"
102
+ - !ruby/object:Gem::Version
103
+ version: '1.23'
104
+ type: :runtime
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: '1.23'
111
+ description: A service for providing an HA NAT in EC2
112
+ email:
113
+ - eric.github@herot.com
114
+ executables:
115
+ - nat-monitor
116
+ extensions: []
117
+ extra_rdoc_files: []
118
+ files:
119
+ - ".gitignore"
120
+ - Gemfile
121
+ - LICENSE.txt
122
+ - README.md
123
+ - Rakefile
124
+ - bin/nat-monitor
125
+ - lib/nat-monitor.rb
126
+ - lib/nat-monitor/version.rb
127
+ - nat-monitor.gemspec
128
+ - spec/lib/nat-monitor_spec.rb
129
+ - spec/spec_helper.rb
130
+ homepage: ''
131
+ licenses:
132
+ - Apache 2.0
133
+ metadata: {}
134
+ post_install_message:
135
+ rdoc_options: []
136
+ require_paths:
137
+ - lib
138
+ required_ruby_version: !ruby/object:Gem::Requirement
139
+ requirements:
140
+ - - ">="
141
+ - !ruby/object:Gem::Version
142
+ version: '0'
143
+ required_rubygems_version: !ruby/object:Gem::Requirement
144
+ requirements:
145
+ - - ">="
146
+ - !ruby/object:Gem::Version
147
+ version: '0'
148
+ requirements: []
149
+ rubyforge_project:
150
+ rubygems_version: 2.2.2
151
+ signing_key:
152
+ specification_version: 4
153
+ summary: A service for providing an HA NAT in EC2
154
+ test_files:
155
+ - spec/lib/nat-monitor_spec.rb
156
+ - spec/spec_helper.rb
157
+ has_rdoc: