flapjack 0.6.53 → 0.6.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/flapjack +103 -19
- data/bin/flapjack-nagios-receiver +166 -52
- data/bin/flapper +107 -18
- data/etc/flapjack_config.yaml.example +16 -0
- data/features/events.feature +63 -0
- data/features/steps/events_steps.rb +5 -5
- data/features/steps/notifications_steps.rb +8 -6
- data/features/steps/time_travel_steps.rb +4 -4
- data/features/support/env.rb +1 -2
- data/flapjack.gemspec +1 -1
- data/lib/flapjack/configuration.rb +11 -13
- data/lib/flapjack/coordinator.rb +100 -220
- data/lib/flapjack/data/entity_check.rb +2 -2
- data/lib/flapjack/data/event.rb +3 -3
- data/lib/flapjack/executive.rb +30 -40
- data/lib/flapjack/filters/delays.rb +1 -1
- data/lib/flapjack/gateways/api.rb +6 -23
- data/lib/flapjack/gateways/email.rb +4 -10
- data/lib/flapjack/gateways/email/alert.html.haml +0 -5
- data/lib/flapjack/gateways/email/alert.text.erb +0 -1
- data/lib/flapjack/gateways/jabber.rb +80 -67
- data/lib/flapjack/gateways/oobetet.rb +29 -25
- data/lib/flapjack/gateways/pagerduty.rb +26 -45
- data/lib/flapjack/gateways/sms_messagenet.rb +10 -17
- data/lib/flapjack/gateways/web.rb +7 -21
- data/lib/flapjack/gateways/web/views/_css.haml +3 -0
- data/lib/flapjack/gateways/web/views/check.haml +1 -1
- data/lib/flapjack/logger.rb +57 -0
- data/lib/flapjack/patches.rb +0 -10
- data/lib/flapjack/pikelet.rb +214 -30
- data/lib/flapjack/redis_pool.rb +2 -17
- data/lib/flapjack/version.rb +1 -1
- data/spec/lib/flapjack/coordinator_spec.rb +116 -136
- data/spec/lib/flapjack/data/entity_check_spec.rb +3 -3
- data/spec/lib/flapjack/executive_spec.rb +33 -34
- data/spec/lib/flapjack/gateways/api_spec.rb +4 -2
- data/spec/lib/flapjack/gateways/jabber_spec.rb +39 -36
- data/spec/lib/flapjack/gateways/oobetet_spec.rb +14 -24
- data/spec/lib/flapjack/gateways/pagerduty_spec.rb +43 -45
- data/spec/lib/flapjack/gateways/web_spec.rb +42 -35
- data/spec/lib/flapjack/logger_spec.rb +32 -0
- data/spec/lib/flapjack/pikelet_spec.rb +124 -15
- data/spec/lib/flapjack/redis_pool_spec.rb +1 -3
- data/spec/spec_helper.rb +34 -1
- data/tasks/events.rake +1 -0
- data/tmp/create_event_ok.rb +31 -0
- data/tmp/create_event_unknown.rb +31 -0
- data/tmp/create_events_ok.rb +1 -1
- metadata +10 -11
- data/bin/flapjack-nagios-receiver-control +0 -15
- data/bin/flapper-control +0 -15
- data/lib/flapjack/daemonizing.rb +0 -186
- data/lib/flapjack/gateways/base.rb +0 -38
@@ -35,10 +35,14 @@ development:
|
|
35
35
|
sms_queue: sms_notifications
|
36
36
|
jabber_queue: jabber_notifications
|
37
37
|
notification_log_file: log/flapjack-notification.log
|
38
|
+
logger:
|
39
|
+
level: INFO
|
38
40
|
gateways:
|
39
41
|
email:
|
40
42
|
enabled: yes
|
41
43
|
queue: email_notifications
|
44
|
+
logger:
|
45
|
+
level: INFO
|
42
46
|
smtp_config:
|
43
47
|
port: 2525
|
44
48
|
# address: "localhost"
|
@@ -53,6 +57,8 @@ development:
|
|
53
57
|
queue: sms_notifications
|
54
58
|
username: "ermahgerd"
|
55
59
|
password: "xxxx"
|
60
|
+
logger:
|
61
|
+
level: INFO
|
56
62
|
jabber:
|
57
63
|
enabled: yes
|
58
64
|
queue: jabber_notifications
|
@@ -64,6 +70,8 @@ development:
|
|
64
70
|
rooms:
|
65
71
|
- "gimp@conference.jabber.domain.tld"
|
66
72
|
- "log@conference.jabber.domain.tld"
|
73
|
+
logger:
|
74
|
+
level: INFO
|
67
75
|
oobetet:
|
68
76
|
enabled: yes
|
69
77
|
server: "jabber.domain.tld"
|
@@ -79,17 +87,25 @@ development:
|
|
79
87
|
- "flapjacktest@conference.jabber.domain.tld"
|
80
88
|
- "gimp@conference.jabber.domain.tld"
|
81
89
|
- "log@conference.jabber.domain.tld"
|
90
|
+
logger:
|
91
|
+
level: INFO
|
82
92
|
pagerduty:
|
83
93
|
enabled: yes
|
84
94
|
queue: pagerduty_notifications
|
95
|
+
logger:
|
96
|
+
level: INFO
|
85
97
|
web:
|
86
98
|
enabled: yes
|
87
99
|
port: 5080
|
88
100
|
access_log: "log/web_access.log"
|
101
|
+
logger:
|
102
|
+
level: INFO
|
89
103
|
api:
|
90
104
|
enabled: yes
|
91
105
|
port: 5081
|
92
106
|
access_log: "log/api_access.log"
|
107
|
+
logger:
|
108
|
+
level: INFO
|
93
109
|
|
94
110
|
test:
|
95
111
|
redis:
|
data/features/events.feature
CHANGED
@@ -160,3 +160,66 @@ Feature: events
|
|
160
160
|
And an ok event is received for check 'abc' on entity 'def'
|
161
161
|
Then a notification should not be generated for check 'abc' on entity 'def'
|
162
162
|
|
163
|
+
Scenario: Flapper (down for one minute, up for one minute, repeat)
|
164
|
+
Given check 'abc' for entity 'def' is in an ok state
|
165
|
+
When a failure event is received for check 'abc' on entity 'def'
|
166
|
+
Then a notification should not be generated for check 'abc' on entity 'def'
|
167
|
+
When 10 seconds passes
|
168
|
+
And a failure event is received for check 'abc' on entity 'def'
|
169
|
+
Then a notification should not be generated for check 'abc' on entity 'def'
|
170
|
+
When 10 seconds passes
|
171
|
+
And a failure event is received for check 'abc' on entity 'def'
|
172
|
+
Then a notification should not be generated for check 'abc' on entity 'def'
|
173
|
+
When 10 seconds passes
|
174
|
+
# 30 seconds
|
175
|
+
And a failure event is received for check 'abc' on entity 'def'
|
176
|
+
Then a notification should be generated for check 'abc' on entity 'def'
|
177
|
+
When 10 seconds passes
|
178
|
+
And a failure event is received for check 'abc' on entity 'def'
|
179
|
+
Then a notification should not be generated for check 'abc' on entity 'def'
|
180
|
+
When 10 seconds passes
|
181
|
+
And a failure event is received for check 'abc' on entity 'def'
|
182
|
+
Then a notification should not be generated for check 'abc' on entity 'def'
|
183
|
+
When 10 seconds passes
|
184
|
+
# 60 seconds
|
185
|
+
And an ok event is received for check 'abc' on entity 'def'
|
186
|
+
Then a notification should be generated for check 'abc' on entity 'def'
|
187
|
+
When 10 seconds passes
|
188
|
+
And an ok event is received for check 'abc' on entity 'def'
|
189
|
+
Then a notification should not be generated for check 'abc' on entity 'def'
|
190
|
+
When 10 seconds passes
|
191
|
+
And an ok event is received for check 'abc' on entity 'def'
|
192
|
+
Then a notification should not be generated for check 'abc' on entity 'def'
|
193
|
+
When 10 seconds passes
|
194
|
+
And an ok event is received for check 'abc' on entity 'def'
|
195
|
+
Then a notification should not be generated for check 'abc' on entity 'def'
|
196
|
+
When 10 seconds passes
|
197
|
+
And an ok event is received for check 'abc' on entity 'def'
|
198
|
+
Then a notification should not be generated for check 'abc' on entity 'def'
|
199
|
+
When 10 seconds passes
|
200
|
+
And an ok event is received for check 'abc' on entity 'def'
|
201
|
+
Then a notification should not be generated for check 'abc' on entity 'def'
|
202
|
+
When 10 seconds passes
|
203
|
+
# 120 seconds
|
204
|
+
And a failure event is received for check 'abc' on entity 'def'
|
205
|
+
Then a notification should not be generated for check 'abc' on entity 'def'
|
206
|
+
When 10 seconds passes
|
207
|
+
And a failure event is received for check 'abc' on entity 'def'
|
208
|
+
Then a notification should not be generated for check 'abc' on entity 'def'
|
209
|
+
When 10 seconds passes
|
210
|
+
And a failure event is received for check 'abc' on entity 'def'
|
211
|
+
Then a notification should not be generated for check 'abc' on entity 'def'
|
212
|
+
When 10 seconds passes
|
213
|
+
# 150 seconds
|
214
|
+
And a failure event is received for check 'abc' on entity 'def'
|
215
|
+
Then a notification should be generated for check 'abc' on entity 'def'
|
216
|
+
When 10 seconds passes
|
217
|
+
And a failure event is received for check 'abc' on entity 'def'
|
218
|
+
Then a notification should not be generated for check 'abc' on entity 'def'
|
219
|
+
When 10 seconds passes
|
220
|
+
And a failure event is received for check 'abc' on entity 'def'
|
221
|
+
Then a notification should not be generated for check 'abc' on entity 'def'
|
222
|
+
When 10 seconds passes
|
223
|
+
# 180 seconds
|
224
|
+
And an ok event is received for check 'abc' on entity 'def'
|
225
|
+
Then a notification should be generated for check 'abc' on entity 'def'
|
@@ -147,17 +147,17 @@ end
|
|
147
147
|
|
148
148
|
# TODO logging is a side-effect, should test for notification generation itself
|
149
149
|
Then /^a notification should not be generated for check '([\w\.\-]+)' on entity '([\w\.\-]+)'$/ do |check, entity|
|
150
|
-
message = @
|
151
|
-
message ? happy = message.match(/Not
|
150
|
+
message = @logger.messages.find_all {|m| m =~ /enerating notifications for event #{entity}:#{check}/ }.last
|
151
|
+
message ? happy = message.match(/Not generating notifications/) : happy = false
|
152
152
|
happy.should be_true
|
153
153
|
end
|
154
154
|
|
155
155
|
Then /^a notification should be generated for check '([\w\.\-]+)' on entity '([\w\.\-]+)'$/ do |check, entity|
|
156
|
-
message = @
|
157
|
-
message ? happy = message.match(/
|
156
|
+
message = @logger.messages.find_all {|m| m =~ /enerating notifications for event #{entity}:#{check}/ }.last
|
157
|
+
message ? happy = message.match(/Generating notifications/) : happy = false
|
158
158
|
happy.should be_true
|
159
159
|
end
|
160
160
|
|
161
161
|
Then /^show me the notifications?$/ do
|
162
|
-
puts @
|
162
|
+
puts @logger.messages.join("\n")
|
163
163
|
end
|
@@ -127,7 +127,8 @@ When /^the SMS notification handler runs successfully$/ do
|
|
127
127
|
|
128
128
|
Flapjack::Gateways::SmsMessagenet.instance_variable_set('@config', {'username' => 'abcd', 'password' => 'efgh'})
|
129
129
|
Flapjack::Gateways::SmsMessagenet.instance_variable_set('@logger', @logger)
|
130
|
-
Flapjack::Gateways::SmsMessagenet.
|
130
|
+
Flapjack::Gateways::SmsMessagenet.start
|
131
|
+
|
131
132
|
Flapjack::Gateways::SmsMessagenet.perform(@sms_notification)
|
132
133
|
end
|
133
134
|
|
@@ -135,15 +136,16 @@ When /^the SMS notification handler fails to send an SMS$/ do
|
|
135
136
|
@request = stub_request(:get, /^#{Regexp.escape(Flapjack::Gateways::SmsMessagenet::MESSAGENET_URL)}/).to_return(:status => [500, "Internal Server Error"])
|
136
137
|
Flapjack::Gateways::SmsMessagenet.instance_variable_set('@config', {'username' => 'abcd', 'password' => 'efgh'})
|
137
138
|
Flapjack::Gateways::SmsMessagenet.instance_variable_set('@logger', @logger)
|
138
|
-
Flapjack::Gateways::SmsMessagenet.
|
139
|
+
Flapjack::Gateways::SmsMessagenet.start
|
140
|
+
|
139
141
|
Flapjack::Gateways::SmsMessagenet.perform(@sms_notification)
|
140
142
|
end
|
141
143
|
|
142
144
|
When /^the email notification handler runs successfully$/ do
|
143
145
|
Resque.redis = @redis
|
144
|
-
Flapjack::Gateways::Email.
|
146
|
+
Flapjack::Gateways::Email.instance_variable_set('@config', {'smtp_config' => {'host' => '127.0.0.1', 'port' => 2525}})
|
145
147
|
Flapjack::Gateways::Email.instance_variable_set('@logger', @logger)
|
146
|
-
Flapjack::Gateways::Email.
|
148
|
+
Flapjack::Gateways::Email.start
|
147
149
|
|
148
150
|
# poor man's stubbing
|
149
151
|
EM::P::SmtpClient.class_eval {
|
@@ -159,9 +161,9 @@ end
|
|
159
161
|
|
160
162
|
When /^the email notification handler fails to send an email$/ do
|
161
163
|
Resque.redis = @redis
|
162
|
-
Flapjack::Gateways::Email.
|
164
|
+
Flapjack::Gateways::Email.instance_variable_set('@config', {'smtp_config' => {'host' => '127.0.0.1', 'port' => 2525}})
|
163
165
|
Flapjack::Gateways::Email.instance_variable_set('@logger', @logger)
|
164
|
-
Flapjack::Gateways::Email.
|
166
|
+
Flapjack::Gateways::Email.start
|
165
167
|
|
166
168
|
# poor man's stubbing
|
167
169
|
EM::P::SmtpClient.class_eval {
|
@@ -6,24 +6,24 @@ require 'chronic'
|
|
6
6
|
When /^(.+) passes$/ do |time|
|
7
7
|
period = Chronic.parse("#{time} from now")
|
8
8
|
Delorean.time_travel_to(period)
|
9
|
-
puts "Time Travelled to #{Time.now.to_s}"
|
9
|
+
# puts "Time Travelled to #{Time.now.to_s}"
|
10
10
|
end
|
11
11
|
|
12
12
|
Given /^I time travel to (.+)$/ do |period|
|
13
13
|
Delorean.time_travel_to(period)
|
14
|
-
puts "Time Travelled to #{Time.now.to_s}"
|
14
|
+
# puts "Time Travelled to #{Time.now.to_s}"
|
15
15
|
end
|
16
16
|
|
17
17
|
Given /^I come back to the present$/ do
|
18
18
|
Delorean.back_to_the_present
|
19
|
-
puts "Time Travelled to the present, #{Time.now.to_s}"
|
19
|
+
# puts "Time Travelled to the present, #{Time.now.to_s}"
|
20
20
|
end
|
21
21
|
|
22
22
|
Given /^I time travel in (.+) to (.+)$/ do |zone_name, timestamp|
|
23
23
|
zone = ::Time.find_zone!(zone_name)
|
24
24
|
time = zone.parse timestamp
|
25
25
|
Delorean.time_travel_to time
|
26
|
-
puts "Time Travelled to #{Time.now.to_s}"
|
26
|
+
# puts "Time Travelled to #{Time.now.to_s}"
|
27
27
|
end
|
28
28
|
|
29
29
|
Then /^the time in UTC should be about (.+)$/ do |timestamp|
|
data/features/support/env.rb
CHANGED
@@ -61,8 +61,7 @@ end
|
|
61
61
|
Before do
|
62
62
|
@logger = MockLogger.new
|
63
63
|
# Use a separate database whilst testing
|
64
|
-
@app = Flapjack::Executive.new
|
65
|
-
@app.bootstrap(:logger => @logger,
|
64
|
+
@app = Flapjack::Executive.new(:logger => @logger,
|
66
65
|
:config => {'email_queue' => 'email_notifications',
|
67
66
|
'sms_queue' => 'sms_notifications'},
|
68
67
|
:redis_config => redis_opts)
|
data/flapjack.gemspec
CHANGED
@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
|
|
17
17
|
gem.require_paths = ["lib"]
|
18
18
|
gem.version = Flapjack::VERSION
|
19
19
|
|
20
|
-
gem.add_dependency '
|
20
|
+
gem.add_dependency 'dante'
|
21
21
|
gem.add_dependency 'log4r'
|
22
22
|
gem.add_dependency 'yajl-ruby'
|
23
23
|
gem.add_dependency 'eventmachine', '~> 1.0.0'
|
@@ -7,19 +7,12 @@ module Flapjack
|
|
7
7
|
|
8
8
|
class Configuration
|
9
9
|
|
10
|
+
attr_reader :filename
|
11
|
+
|
10
12
|
def initialize(opts = {})
|
11
13
|
@logger = opts[:logger]
|
12
|
-
unless @logger
|
13
|
-
@logger = Logger.new(STDOUT)
|
14
|
-
@logger.level = Logger::ERROR
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
def logger
|
19
|
-
@logger
|
20
14
|
end
|
21
15
|
|
22
|
-
# TODO reduce/remove the use of this, just access parts
|
23
16
|
def all
|
24
17
|
@config_env
|
25
18
|
end
|
@@ -53,29 +46,34 @@ module Flapjack
|
|
53
46
|
end
|
54
47
|
|
55
48
|
def load(filename)
|
49
|
+
@filename = nil
|
50
|
+
@config_env = nil
|
51
|
+
|
56
52
|
unless File.file?(filename)
|
57
|
-
logger.error "Could not find file '#{filename}'"
|
53
|
+
@logger.error "Could not find file '#{filename}'" if @logger
|
58
54
|
return
|
59
55
|
end
|
60
56
|
|
61
57
|
unless defined?(FLAPJACK_ENV)
|
62
|
-
logger.error "Environment variable 'FLAPJACK_ENV' is not set"
|
58
|
+
@logger.error "Environment variable 'FLAPJACK_ENV' is not set" if @logger
|
63
59
|
return
|
64
60
|
end
|
65
61
|
|
66
62
|
config = YAML::load_file(filename)
|
67
63
|
|
68
64
|
if config.nil?
|
69
|
-
logger.error "Could not load config file '#{filename}'"
|
65
|
+
@logger.error "Could not load config file '#{filename}'" if @logger
|
70
66
|
return
|
71
67
|
end
|
72
68
|
|
73
69
|
@config_env = config[FLAPJACK_ENV]
|
74
70
|
|
75
71
|
if @config_env.nil?
|
76
|
-
logger.error "No config data for environment '#{FLAPJACK_ENV}' found in '#{filename}'"
|
72
|
+
@logger.error "No config data for environment '#{FLAPJACK_ENV}' found in '#{filename}'" if @logger
|
77
73
|
return
|
78
74
|
end
|
75
|
+
|
76
|
+
@filename = filename
|
79
77
|
end
|
80
78
|
|
81
79
|
end
|
data/lib/flapjack/coordinator.rb
CHANGED
@@ -1,297 +1,177 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
3
|
require 'eventmachine'
|
4
|
-
# the redis/synchrony gems need to be required in this particular order, see
|
5
|
-
# the redis-rb README for details
|
6
|
-
require 'hiredis'
|
7
4
|
require 'em-synchrony'
|
8
|
-
require 'redis/connection/synchrony'
|
9
|
-
require 'redis'
|
10
|
-
require 'em-resque'
|
11
|
-
require 'em-resque/worker'
|
12
|
-
require 'thin'
|
13
5
|
|
14
6
|
require 'flapjack/configuration'
|
15
7
|
require 'flapjack/patches'
|
16
|
-
require 'flapjack/daemonizing'
|
17
8
|
require 'flapjack/executive'
|
18
9
|
require 'flapjack/redis_pool'
|
19
10
|
|
20
11
|
require 'flapjack/pikelet'
|
21
|
-
require 'flapjack/gateways/base'
|
22
|
-
|
23
12
|
require 'flapjack/executive'
|
24
13
|
|
25
|
-
require 'flapjack/gateways/api'
|
26
|
-
require 'flapjack/gateways/jabber'
|
27
|
-
require 'flapjack/gateways/oobetet'
|
28
|
-
require 'flapjack/gateways/pagerduty'
|
29
|
-
require 'flapjack/gateways/email'
|
30
|
-
require 'flapjack/gateways/sms_messagenet'
|
31
|
-
require 'flapjack/gateways/web'
|
32
|
-
|
33
14
|
module Flapjack
|
34
15
|
|
35
16
|
class Coordinator
|
36
17
|
|
37
|
-
include Flapjack::Daemonizable
|
38
|
-
|
39
18
|
def initialize(config)
|
40
19
|
@config = config
|
41
20
|
@redis_options = config.for_redis
|
42
21
|
@pikelets = []
|
43
22
|
|
44
|
-
|
45
|
-
|
46
|
-
@logger
|
47
|
-
end
|
23
|
+
# TODO convert this to use flapjack-logger
|
24
|
+
logger_name = "flapjack-coordinator"
|
25
|
+
@logger = Log4r::Logger.new(logger_name)
|
48
26
|
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
27
|
+
formatter = Log4r::PatternFormatter.new(:pattern => "%d [%l] :: #{logger_name} :: %m",
|
28
|
+
:date_pattern => "%Y-%m-%dT%H:%M:%S%z")
|
29
|
+
|
30
|
+
[Log4r::StdoutOutputter, Log4r::SyslogOutputter].each do |outp_klass|
|
31
|
+
outp = outp_klass.new(logger_name)
|
32
|
+
outp.formatter = formatter
|
33
|
+
@logger.add(outp)
|
56
34
|
end
|
57
35
|
end
|
58
36
|
|
59
|
-
def
|
60
|
-
|
37
|
+
def start(options = {})
|
38
|
+
EM.synchrony do
|
39
|
+
add_pikelets(pikelets(@config.all))
|
40
|
+
setup_signals if options[:signals]
|
41
|
+
end
|
61
42
|
end
|
62
43
|
|
63
44
|
def stop
|
64
45
|
return if @stopping
|
65
46
|
@stopping = true
|
66
|
-
shutdown
|
47
|
+
remove_pikelets(@pikelets, :shutdown => true)
|
67
48
|
end
|
68
49
|
|
69
|
-
|
50
|
+
# NB: global config options (e.g. daemonize, pidfile,
|
51
|
+
# logfile, redis options) won't be checked on reload.
|
52
|
+
# should we do a full restart if some of these change?
|
53
|
+
def reload
|
54
|
+
prev_pikelet_cfg = pikelets(@config.all)
|
70
55
|
|
71
|
-
|
56
|
+
removed = []
|
57
|
+
added = []
|
58
|
+
ask_running = []
|
72
59
|
|
73
|
-
|
60
|
+
cfg_filename = @config.filename
|
61
|
+
@config = Flapjack::Configuration.new
|
62
|
+
@config.load(cfg_filename)
|
74
63
|
|
75
|
-
|
64
|
+
enabled_pikelet_cfg = pikelets(@config.all)
|
76
65
|
|
77
|
-
|
78
|
-
next unless pikelet_cfg['enabled']
|
79
|
-
build_pikelet(pikelet_class, pikelet_cfg)
|
80
|
-
end
|
66
|
+
(prev_pikelet_cfg.keys + enabled_pikelet_cfg.keys).each do |type|
|
81
67
|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
68
|
+
if prev_pikelet_cfg.keys.include?(type)
|
69
|
+
if enabled_pikelet_cfg.keys.include?(type)
|
70
|
+
ask_running << type
|
71
|
+
else
|
72
|
+
removed << type
|
73
|
+
end
|
74
|
+
elsif enabled_pikelet_cfg.keys.include?(type)
|
75
|
+
added << type
|
86
76
|
end
|
87
77
|
|
88
|
-
setup_signals if @signals
|
89
78
|
end
|
90
79
|
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
80
|
+
@pikelets.select {|pik| ask_running.include?(pik.type) }.each do |pik|
|
81
|
+
# for sections previously there and still there, ask them
|
82
|
+
# to make the config change; they will if they can, or will signal
|
83
|
+
# restart is needed if not
|
84
|
+
|
85
|
+
# reload() returns trinary value here; true means the change was made, false
|
86
|
+
# means the pikelet needs to be restarted, nil means no change
|
87
|
+
# was required
|
88
|
+
next unless pik.reload(enabled_pikelet_cfg[pik.type]).is_a?(FalseClass)
|
89
|
+
removed << pik.type
|
90
|
+
added << pik.type
|
102
91
|
end
|
103
|
-
end
|
104
92
|
|
105
|
-
|
106
|
-
|
93
|
+
# puts "removed"
|
94
|
+
# p removed
|
107
95
|
|
108
|
-
|
109
|
-
|
96
|
+
# puts "added"
|
97
|
+
# p added
|
110
98
|
|
111
|
-
|
112
|
-
fiber = nil
|
99
|
+
removed_pikelets = @pikelets.select {|pik| removed.include?(pik.type) }
|
113
100
|
|
114
|
-
|
115
|
-
|
116
|
-
pikelet.bootstrap(:config => pikelet_cfg, :redis_config => @redis_options)
|
117
|
-
else
|
118
|
-
pikelet_class.bootstrap(:config => pikelet_cfg, :redis_config => @redis_options)
|
101
|
+
# puts "removed pikelets"
|
102
|
+
# p removed_pikelets
|
119
103
|
|
120
|
-
|
121
|
-
|
122
|
-
unless @thin_silenced
|
123
|
-
Thin::Logging.silent = true
|
124
|
-
@thin_silenced = true
|
125
|
-
end
|
104
|
+
remove_pikelets(removed_pikelets)
|
126
105
|
|
127
|
-
|
128
|
-
|
129
|
-
pikelet_class, :signals => false)
|
106
|
+
# is there a nicer way to only keep the parts of the hash with matching keys?
|
107
|
+
added_pikelets = enabled_pikelet_cfg.select {|k, v| added.include?(k) }
|
130
108
|
|
131
|
-
|
109
|
+
# puts "added pikelet configs"
|
110
|
+
# p added_pikelets
|
132
111
|
|
133
|
-
|
134
|
-
|
135
|
-
@resque_pool = Flapjack::RedisPool.new(:config => @redis_options)
|
136
|
-
::Resque.redis = @resque_pool
|
137
|
-
## NB: can override the default 'resque' namespace like this
|
138
|
-
#::Resque.redis.namespace = 'flapjack'
|
139
|
-
end
|
140
|
-
|
141
|
-
# TODO error if pikelet_cfg['queue'].nil?
|
142
|
-
pikelet = EM::Resque::Worker.new(pikelet_cfg['queue'])
|
143
|
-
# # Use these to debug the resque workers
|
144
|
-
# pikelet.verbose = true
|
145
|
-
# pikelet.very_verbose = true
|
146
|
-
end
|
147
|
-
|
148
|
-
end
|
112
|
+
add_pikelets(added_pikelets)
|
113
|
+
end
|
149
114
|
|
150
|
-
|
151
|
-
|
152
|
-
if inc_mod.include?(Flapjack::GenericPikelet) ||
|
153
|
-
ext_mod.include?(Flapjack::Gateways::Resque)
|
154
|
-
|
155
|
-
fiber = Fiber.new {
|
156
|
-
begin
|
157
|
-
# Can't use local inc_mod/ext_mod variables in the new fiber
|
158
|
-
if pikelet.is_a?(Flapjack::GenericPikelet)
|
159
|
-
pikelet.main
|
160
|
-
elsif extended_modules(pikelet_class).include?(Flapjack::Gateways::Resque)
|
161
|
-
pikelet.work(0.1)
|
162
|
-
end
|
163
|
-
rescue Exception => e
|
164
|
-
trace = e.backtrace.join("\n")
|
165
|
-
@logger.fatal "#{e.message}\n#{trace}"
|
166
|
-
stop
|
167
|
-
end
|
168
|
-
}
|
115
|
+
private
|
169
116
|
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
117
|
+
# the global nature of this seems at odds with it calling stop
|
118
|
+
# within a single coordinator instance. Coordinator is essentially
|
119
|
+
# a singleton anyway...
|
120
|
+
def setup_signals
|
121
|
+
Kernel.trap('INT') { stop }
|
122
|
+
Kernel.trap('TERM') { stop }
|
123
|
+
unless RbConfig::CONFIG['host_os'] =~ /mswin|windows|cygwin/i
|
124
|
+
Kernel.trap('QUIT') { stop }
|
125
|
+
Kernel.trap('HUP') { reload }
|
176
126
|
end
|
177
|
-
|
178
|
-
@pikelets << pikelet_info
|
179
127
|
end
|
180
128
|
|
181
|
-
#
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
pik[:fiber].alive? ? 'running' : 'stopped'
|
189
|
-
end
|
190
|
-
next if pik.has_key?(:status) && pik[:status].eql?(status)
|
191
|
-
@logger.info "#{pik[:class].name}: #{status}"
|
192
|
-
pik[:status] = status
|
129
|
+
# passed a hash with {PIKELET_TYPE => PIKELET_CFG, ...}
|
130
|
+
def add_pikelets(pikelets_data = {})
|
131
|
+
pikelets_data.each_pair do |type, cfg|
|
132
|
+
next unless pikelet = Flapjack::Pikelet.create(type,
|
133
|
+
:config => cfg, :redis_config => @redis_options)
|
134
|
+
@pikelets << pikelet
|
135
|
+
pikelet.start
|
193
136
|
end
|
194
137
|
end
|
195
138
|
|
196
|
-
def
|
197
|
-
@pikelets.each do |pik|
|
198
|
-
|
199
|
-
pik_inst = pik[:instance]
|
200
|
-
ext_mod = extended_modules(pik[:class])
|
201
|
-
|
202
|
-
# would be neater if we could use something similar for the class << self
|
203
|
-
# included pikelets as well
|
204
|
-
if pik_inst.is_a?(Flapjack::GenericPikelet)
|
205
|
-
if pik[:fiber] && pik[:fiber].alive?
|
206
|
-
pik_inst.stop
|
207
|
-
Fiber.new {
|
208
|
-
# this needs to use a separate Redis connection from the pikelet's
|
209
|
-
# one, as that's in the middle of its blpop
|
210
|
-
r = Redis.new(@redis_options.merge(:driver => 'synchrony'))
|
211
|
-
pik_inst.add_shutdown_event(:redis => r)
|
212
|
-
r.quit
|
213
|
-
}.resume
|
214
|
-
end
|
215
|
-
elsif ext_mod.include?(Flapjack::Gateways::Resque)
|
216
|
-
# resque is polling, so we don't need a shutdown object
|
217
|
-
pik_inst.shutdown if pik[:fiber] && pik[:fiber].alive?
|
218
|
-
elsif ext_mod.include?(Flapjack::Gateways::Thin)
|
219
|
-
# drop from this side, as HTTP keepalive etc. means browsers
|
220
|
-
# keep connections alive for ages, and we'd be hanging around
|
221
|
-
# waiting for them to drop
|
222
|
-
pik_inst.stop!
|
223
|
-
end
|
224
|
-
end
|
225
|
-
|
139
|
+
def remove_pikelets(piks, opts = {})
|
226
140
|
Fiber.new {
|
141
|
+
piks.map(&:stop)
|
227
142
|
|
228
143
|
loop do
|
229
|
-
|
144
|
+
# only prints state changes, otherwise pikelets not closing promptly can
|
145
|
+
# cause everything else to be spammy
|
146
|
+
piks.each do |pik|
|
147
|
+
old_status = pik.status
|
148
|
+
pik.update_status
|
149
|
+
status = pik.status
|
150
|
+
next if old_status.eql?(status)
|
151
|
+
@logger.info "#{pik.type}: #{old_status} -> #{status}"
|
152
|
+
end
|
230
153
|
|
231
|
-
if
|
154
|
+
if piks.any? {|p| p.status == 'stopping' }
|
232
155
|
EM::Synchrony.sleep 0.25
|
233
156
|
else
|
234
|
-
|
235
|
-
|
236
|
-
@pikelets.each do |pik|
|
237
|
-
|
238
|
-
pik_inst = pik[:instance]
|
239
|
-
ext_mod = extended_modules(pik[:class])
|
240
|
-
|
241
|
-
if pik_inst.is_a?(Flapjack::GenericPikelet)
|
242
|
-
|
243
|
-
pik_inst.cleanup
|
244
|
-
|
245
|
-
elsif [Flapjack::Gateways::Resque, Flapjack::Gateways::Thin].any?{|fp|
|
246
|
-
ext_mod.include?(fp)
|
247
|
-
}
|
248
|
-
|
249
|
-
pik[:class].cleanup
|
250
|
-
|
251
|
-
end
|
252
|
-
end
|
253
|
-
|
254
|
-
EM.stop
|
157
|
+
EM.stop if opts[:shutdown]
|
158
|
+
@pikelets -= piks
|
255
159
|
break
|
256
160
|
end
|
257
161
|
end
|
258
162
|
}.resume
|
259
163
|
end
|
260
164
|
|
261
|
-
def extended_modules(klass)
|
262
|
-
(class << klass; self; end).included_modules
|
263
|
-
end
|
264
|
-
|
265
|
-
PIKELET_TYPES = {'executive' => Flapjack::Executive}
|
266
|
-
|
267
|
-
GATEWAY_TYPES = {'web' => Flapjack::Gateways::Web,
|
268
|
-
'api' => Flapjack::Gateways::API,
|
269
|
-
'jabber' => Flapjack::Gateways::Jabber,
|
270
|
-
'pagerduty' => Flapjack::Gateways::Pagerduty,
|
271
|
-
'oobetet' => Flapjack::Gateways::Oobetet,
|
272
|
-
'email' => Flapjack::Gateways::Email,
|
273
|
-
'sms' => Flapjack::Gateways::SmsMessagenet}
|
274
|
-
|
275
|
-
|
276
165
|
def pikelets(config_env)
|
277
166
|
return {} unless config_env
|
278
|
-
config_env.
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
memo
|
283
|
-
}
|
284
|
-
end
|
285
|
-
|
286
|
-
def gateways(config_env)
|
287
|
-
return {} unless config_env && config_env['gateways'] &&
|
167
|
+
exec_cfg = config_env.has_key?('executive') && config_env['executive']['enabled'] ?
|
168
|
+
{'executive' => config_env['executive']} :
|
169
|
+
{}
|
170
|
+
return exec_cfg unless config_env && config_env['gateways'] &&
|
288
171
|
!config_env['gateways'].nil?
|
289
|
-
config_env['gateways'].
|
290
|
-
|
291
|
-
|
292
|
-
end
|
293
|
-
memo
|
294
|
-
}
|
172
|
+
exec_cfg.merge( config_env['gateways'].select {|k, v|
|
173
|
+
Flapjack::Pikelet.is_pikelet?(k) && v['enabled']
|
174
|
+
} )
|
295
175
|
end
|
296
176
|
|
297
177
|
end
|