flapjack 0.7.20 → 0.7.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +1 -0
- data/.travis.yml +3 -1
- data/CHANGELOG.md +10 -0
- data/Gemfile +1 -0
- data/bin/flapjack +11 -0
- data/bin/simulate-failed-check +5 -5
- data/features/notification_rules.feature +77 -19
- data/features/steps/events_steps.rb +15 -3
- data/lib/flapjack/coordinator.rb +3 -3
- data/lib/flapjack/data/contact.rb +1 -1
- data/lib/flapjack/data/entity.rb +12 -1
- data/lib/flapjack/data/entity_check.rb +9 -2
- data/lib/flapjack/data/event.rb +4 -4
- data/lib/flapjack/data/notification.rb +27 -20
- data/lib/flapjack/data/notification_rule.rb +26 -24
- data/lib/flapjack/data/tag.rb +5 -0
- data/lib/flapjack/gateways/api.rb +1 -1
- data/lib/flapjack/gateways/api/contact_methods.rb +3 -3
- data/lib/flapjack/gateways/email.rb +73 -46
- data/lib/flapjack/gateways/email/alert.html.erb +13 -4
- data/lib/flapjack/gateways/email/alert.text.erb +2 -2
- data/lib/flapjack/gateways/jabber.rb +22 -16
- data/lib/flapjack/gateways/pagerduty.rb +7 -3
- data/lib/flapjack/gateways/web.rb +1 -1
- data/lib/flapjack/gateways/web/views/check.html.erb +2 -2
- data/lib/flapjack/gateways/web/views/contact.html.erb +3 -3
- data/lib/flapjack/logger.rb +67 -35
- data/lib/flapjack/notifier.rb +9 -3
- data/lib/flapjack/pikelet.rb +3 -1
- data/lib/flapjack/processor.rb +34 -10
- data/lib/flapjack/version.rb +1 -1
- data/spec/lib/flapjack/coordinator_spec.rb +17 -13
- data/spec/lib/flapjack/data/contact_spec.rb +4 -3
- data/spec/lib/flapjack/data/entity_check_spec.rb +10 -0
- data/spec/lib/flapjack/data/entity_spec.rb +60 -5
- data/spec/lib/flapjack/data/event_spec.rb +4 -4
- data/spec/lib/flapjack/data/notification_rule_spec.rb +9 -2
- data/spec/lib/flapjack/data/tag_spec.rb +0 -1
- data/spec/lib/flapjack/gateways/api/contact_methods_spec.rb +1 -1
- data/spec/lib/flapjack/gateways/email_spec.rb +2 -1
- data/spec/lib/flapjack/gateways/jabber_spec.rb +5 -3
- data/spec/lib/flapjack/gateways/pagerduty_spec.rb +3 -1
- data/spec/lib/flapjack/logger_spec.rb +5 -5
- data/spec/lib/flapjack/pikelet_spec.rb +4 -2
- data/spec/lib/flapjack/processor_spec.rb +16 -7
- data/tasks/benchmarks.rake +228 -0
- data/tasks/events.rake +11 -10
- data/tasks/support/flapjack_config_benchmark.yaml +58 -0
- metadata +6 -4
@@ -6,11 +6,12 @@ require 'ice_cube'
|
|
6
6
|
require 'flapjack/data/contact'
|
7
7
|
require 'flapjack/data/entity_check'
|
8
8
|
require 'flapjack/data/notification_rule'
|
9
|
+
require 'flapjack/data/tag_set'
|
9
10
|
|
10
11
|
describe Flapjack::Data::Contact, :redis => true do
|
11
12
|
|
12
13
|
let(:notification_rule_data) {
|
13
|
-
{:
|
14
|
+
{:tags => ["database","physical"],
|
14
15
|
:entities => ["foo-app-01.example.com"],
|
15
16
|
:time_restrictions => [],
|
16
17
|
:warning_media => ["email"],
|
@@ -22,7 +23,7 @@ describe Flapjack::Data::Contact, :redis => true do
|
|
22
23
|
|
23
24
|
let(:general_notification_rule_data) {
|
24
25
|
{:entities => [],
|
25
|
-
:
|
26
|
+
:tags => Flapjack::Data::TagSet.new([]),
|
26
27
|
:time_restrictions => [],
|
27
28
|
:warning_media => ['email', 'sms', 'jabber', 'pagerduty'],
|
28
29
|
:critical_media => ['email', 'sms', 'jabber', 'pagerduty'],
|
@@ -156,7 +157,7 @@ describe Flapjack::Data::Contact, :redis => true do
|
|
156
157
|
rules = contact.notification_rules
|
157
158
|
rules.should have(1).rule
|
158
159
|
rule = rules.first
|
159
|
-
[:entities, :
|
160
|
+
[:entities, :tags, :time_restrictions,
|
160
161
|
:warning_media, :critical_media,
|
161
162
|
:warning_blackhole, :critical_blackhole].each do |k|
|
162
163
|
rule.send(k).should == general_notification_rule_data[k]
|
@@ -2,6 +2,7 @@ require 'spec_helper'
|
|
2
2
|
|
3
3
|
require 'flapjack/data/entity'
|
4
4
|
require 'flapjack/data/entity_check'
|
5
|
+
require 'flapjack/data/tag_set'
|
5
6
|
|
6
7
|
describe Flapjack::Data::EntityCheck, :redis => true do
|
7
8
|
|
@@ -574,4 +575,13 @@ describe Flapjack::Data::EntityCheck, :redis => true do
|
|
574
575
|
contacts.should have(1).contact
|
575
576
|
contacts.first.name.should == 'John Johnson'
|
576
577
|
end
|
578
|
+
|
579
|
+
it "generates ephemeral tags for itself" do
|
580
|
+
ec = Flapjack::Data::EntityCheck.for_entity_name('foo-app-01.example.com', 'Disk / Utilisation', :redis => @redis)
|
581
|
+
tags = ec.tags
|
582
|
+
tags.should_not be_nil
|
583
|
+
tags.should be_a(Flapjack::Data::TagSet)
|
584
|
+
['foo-app-01', 'example.com', 'disk', '/', 'utilisation'].to_set.subset?(tags).should be_true
|
585
|
+
end
|
586
|
+
|
577
587
|
end
|
@@ -176,16 +176,71 @@ describe Flapjack::Data::Entity, :redis => true do
|
|
176
176
|
end
|
177
177
|
|
178
178
|
it "finds entities by tag" do
|
179
|
-
|
179
|
+
entity0 = Flapjack::Data::Entity.add({'id' => '5000',
|
180
|
+
'name' => 'abc-123',
|
181
|
+
'contacts' => []},
|
182
|
+
:redis => @redis)
|
183
|
+
|
184
|
+
entity1 = Flapjack::Data::Entity.add({'id' => '5001',
|
185
|
+
'name' => 'def-456',
|
186
|
+
'contacts' => []},
|
187
|
+
:redis => @redis)
|
188
|
+
|
189
|
+
entity0.add_tags('source:foobar', 'abc')
|
190
|
+
entity1.add_tags('source:foobar', 'def')
|
191
|
+
|
192
|
+
entity0.should_not be_nil
|
193
|
+
entity0.should be_an(Flapjack::Data::Entity)
|
194
|
+
entity0.tags.should include("source:foobar")
|
195
|
+
entity0.tags.should include("abc")
|
196
|
+
entity0.tags.should_not include("def")
|
197
|
+
entity1.should_not be_nil
|
198
|
+
entity1.should be_an(Flapjack::Data::Entity)
|
199
|
+
entity1.tags.should include("source:foobar")
|
200
|
+
entity1.tags.should include("def")
|
201
|
+
entity1.tags.should_not include("abc")
|
202
|
+
|
203
|
+
entities = Flapjack::Data::Entity.find_all_with_tags(['abc'], :redis => @redis)
|
204
|
+
entities.should be_an(Array)
|
205
|
+
entities.should have(1).entity
|
206
|
+
entities.first.should == 'abc-123'
|
207
|
+
|
208
|
+
entities = Flapjack::Data::Entity.find_all_with_tags(['donkey'], :redis => @redis)
|
209
|
+
entities.should be_an(Array)
|
210
|
+
entities.should have(0).entities
|
211
|
+
end
|
212
|
+
|
213
|
+
it "finds entities with several tags" do
|
214
|
+
entity0 = Flapjack::Data::Entity.add({'id' => '5000',
|
180
215
|
'name' => 'abc-123',
|
181
216
|
'contacts' => []},
|
182
217
|
:redis => @redis)
|
183
218
|
|
184
|
-
|
219
|
+
entity1 = Flapjack::Data::Entity.add({'id' => '5001',
|
220
|
+
'name' => 'def-456',
|
221
|
+
'contacts' => []},
|
222
|
+
:redis => @redis)
|
185
223
|
|
186
|
-
|
187
|
-
|
188
|
-
|
224
|
+
entity0.add_tags('source:foobar', 'abc')
|
225
|
+
entity1.add_tags('source:foobar', 'def')
|
226
|
+
|
227
|
+
entity0.should_not be_nil
|
228
|
+
entity0.should be_an(Flapjack::Data::Entity)
|
229
|
+
entity0.tags.should include("source:foobar")
|
230
|
+
entity0.tags.should include("abc")
|
231
|
+
entity1.should_not be_nil
|
232
|
+
entity1.should be_an(Flapjack::Data::Entity)
|
233
|
+
entity1.tags.should include("source:foobar")
|
234
|
+
entity1.tags.should include("def")
|
235
|
+
|
236
|
+
entities = Flapjack::Data::Entity.find_all_with_tags(['source:foobar'], :redis => @redis)
|
237
|
+
entities.should be_an(Array)
|
238
|
+
entities.should have(2).entity
|
239
|
+
|
240
|
+
entities = Flapjack::Data::Entity.find_all_with_tags(['source:foobar', 'def'], :redis => @redis)
|
241
|
+
entities.should be_an(Array)
|
242
|
+
entities.should have(1).entity
|
243
|
+
entities.first.should == 'def-456'
|
189
244
|
end
|
190
245
|
|
191
246
|
end
|
@@ -58,13 +58,13 @@ describe Flapjack::Data::Event do
|
|
58
58
|
events_len = 23
|
59
59
|
mock_redis.should_receive(:llen).with('events').and_return(events_len)
|
60
60
|
|
61
|
-
pc = Flapjack::Data::Event.pending_count(:redis => mock_redis)
|
61
|
+
pc = Flapjack::Data::Event.pending_count('events', :redis => mock_redis)
|
62
62
|
pc.should == events_len
|
63
63
|
end
|
64
64
|
|
65
65
|
it "creates a notification testing event" do
|
66
66
|
Time.should_receive(:now).and_return(time)
|
67
|
-
mock_redis.should_receive(:
|
67
|
+
mock_redis.should_receive(:lpush).with('events', /"testing"/ )
|
68
68
|
|
69
69
|
Flapjack::Data::Event.test_notifications(entity_name, check,
|
70
70
|
:summary => 'test', :details => 'testing', :redis => mock_redis)
|
@@ -72,7 +72,7 @@ describe Flapjack::Data::Event do
|
|
72
72
|
|
73
73
|
it "creates an acknowledgement event" do
|
74
74
|
Time.should_receive(:now).and_return(time)
|
75
|
-
mock_redis.should_receive(:
|
75
|
+
mock_redis.should_receive(:lpush).with('events', /"acking"/ )
|
76
76
|
|
77
77
|
Flapjack::Data::Event.create_acknowledgement(entity_name, check,
|
78
78
|
:summary => 'acking', :time => time.to_i, :redis => mock_redis)
|
@@ -98,4 +98,4 @@ describe Flapjack::Data::Event do
|
|
98
98
|
it { should be_a_failure }
|
99
99
|
end
|
100
100
|
|
101
|
-
end
|
101
|
+
end
|
@@ -14,7 +14,7 @@ describe Flapjack::Data::NotificationRule, :redis => true do
|
|
14
14
|
|
15
15
|
let(:rule_data) {
|
16
16
|
{:contact_id => '23',
|
17
|
-
:
|
17
|
+
:tags => ["database","physical"],
|
18
18
|
:entities => ["foo-app-01.example.com"],
|
19
19
|
:time_restrictions => [ weekdays_8_18 ],
|
20
20
|
:warning_media => ["email"],
|
@@ -76,7 +76,14 @@ describe Flapjack::Data::NotificationRule, :redis => true do
|
|
76
76
|
rule.match_entity?('foo-app-02.example.com').should be_false
|
77
77
|
end
|
78
78
|
|
79
|
-
|
79
|
+
it "checks whether entity tags match" do
|
80
|
+
rule = existing_rule
|
81
|
+
|
82
|
+
rule.match_tags?(['database', 'physical'].to_set).should be_true
|
83
|
+
rule.match_tags?(['database', 'physical', 'beetroot'].to_set).should be_true
|
84
|
+
rule.match_tags?(['database'].to_set).should be_false
|
85
|
+
rule.match_tags?(['virtual'].to_set).should be_false
|
86
|
+
end
|
80
87
|
|
81
88
|
it "checks if blackhole settings for a rule match a severity level" do
|
82
89
|
rule_data[:warning_blackhole] = true
|
@@ -37,7 +37,7 @@ describe 'Flapjack::Gateways::API::ContactMethods', :sinatra => true, :logger =>
|
|
37
37
|
|
38
38
|
let(:notification_rule_data) {
|
39
39
|
{"contact_id" => "21",
|
40
|
-
"
|
40
|
+
"tags" => ["database","physical"],
|
41
41
|
"entities" => ["foo-app-01.example.com"],
|
42
42
|
"time_restrictions" => nil,
|
43
43
|
"warning_media" => ["email"],
|
@@ -15,6 +15,7 @@ describe Flapjack::Gateways::Email, :logger => true do
|
|
15
15
|
entity_check = mock(Flapjack::Data::EntityCheck)
|
16
16
|
entity_check.should_receive(:in_scheduled_maintenance?).and_return(false)
|
17
17
|
entity_check.should_receive(:in_unscheduled_maintenance?).and_return(false)
|
18
|
+
entity_check.should_receive(:last_change).and_return(Time.now.to_i)
|
18
19
|
|
19
20
|
redis = mock('redis')
|
20
21
|
::Resque.should_receive(:redis).and_return(redis)
|
@@ -46,4 +47,4 @@ describe Flapjack::Gateways::Email, :logger => true do
|
|
46
47
|
Flapjack::Gateways::Email.perform(notification)
|
47
48
|
end
|
48
49
|
|
49
|
-
end
|
50
|
+
end
|
@@ -151,9 +151,11 @@ describe Flapjack::Gateways::Jabber, :logger => true do
|
|
151
151
|
end
|
152
152
|
|
153
153
|
it "prompts the blocking redis connection to quit" do
|
154
|
-
|
155
|
-
|
154
|
+
shutdown_redis = mock('shutdown_redis')
|
155
|
+
shutdown_redis.should_receive(:rpush).with('jabber_notifications', %q{{"notification_type":"shutdown"}})
|
156
|
+
EM::Hiredis.should_receive(:connect).and_return(shutdown_redis)
|
156
157
|
|
158
|
+
redis = mock('redis')
|
157
159
|
Flapjack::RedisPool.should_receive(:new).and_return(redis)
|
158
160
|
fj = Flapjack::Gateways::Jabber.new(:config => config, :logger => @logger)
|
159
161
|
|
@@ -163,7 +165,7 @@ describe Flapjack::Gateways::Jabber, :logger => true do
|
|
163
165
|
it "runs a blocking loop listening for notifications" do
|
164
166
|
timer = mock('timer')
|
165
167
|
timer.should_receive(:cancel)
|
166
|
-
EM::Synchrony.should_receive(:add_periodic_timer).with(
|
168
|
+
EM::Synchrony.should_receive(:add_periodic_timer).with(1).and_return(timer)
|
167
169
|
|
168
170
|
redis = mock('redis')
|
169
171
|
|
@@ -11,7 +11,9 @@ describe Flapjack::Gateways::Pagerduty, :logger => true do
|
|
11
11
|
let(:redis) { mock('redis') }
|
12
12
|
|
13
13
|
it "prompts the blocking redis connection to quit" do
|
14
|
-
|
14
|
+
shutdown_redis = mock('shutdown_redis')
|
15
|
+
shutdown_redis.should_receive(:rpush).with(config['queue'], %q{{"notification_type":"shutdown"}})
|
16
|
+
EM::Hiredis.should_receive(:connect).and_return(shutdown_redis)
|
15
17
|
|
16
18
|
Flapjack::RedisPool.should_receive(:new).and_return(redis)
|
17
19
|
fp = Flapjack::Gateways::Pagerduty.new(:config => config, :logger => @logger)
|
@@ -10,21 +10,21 @@ describe Flapjack::Logger do
|
|
10
10
|
it "creates a logger logging to STDOUT and syslog" do
|
11
11
|
logger.should_receive(:formatter=).with(an_instance_of(Proc))
|
12
12
|
logger.should_receive(:level=).and_return(Logger::DEBUG)
|
13
|
-
logger.should_receive(:
|
13
|
+
logger.should_receive(:add).with(2, nil, "Yowza!")
|
14
14
|
::Logger.should_receive(:new).with(STDOUT).and_return(logger)
|
15
15
|
|
16
16
|
if Syslog.const_defined?('Logger', false)
|
17
17
|
sys_logger.should_receive(:formatter=).with(an_instance_of(Proc))
|
18
18
|
sys_logger.should_receive(:level=).with(Logger::DEBUG)
|
19
|
-
sys_logger.should_receive(:
|
19
|
+
sys_logger.should_receive(:add).with(Logger::WARN, 'Yowza!', 'flapjack')
|
20
20
|
Syslog.const_get('Logger', false).should_receive(:new).with('flapjack').and_return(sys_logger)
|
21
21
|
else
|
22
|
-
syslog.should_receive(:log).with(Syslog::Constants::LOG_WARNING, /\[WARN\] :: spec :: %s/, "Yowza!")
|
23
|
-
Syslog.should_receive(:"opened?").and_return(false)
|
24
22
|
Syslog.should_receive(:open).with('flapjack',
|
25
23
|
(Syslog::Constants::LOG_PID | Syslog::Constants::LOG_CONS),
|
26
24
|
Syslog::Constants::LOG_USER).and_return(syslog)
|
27
|
-
Syslog.should_receive(:mask=).with(Syslog::LOG_UPTO(Syslog::Constants::
|
25
|
+
Syslog.should_receive(:mask=).with(Syslog::LOG_UPTO(Syslog::Constants::LOG_WARNING))
|
26
|
+
Syslog.should_receive(:log).with(Syslog::Constants::LOG_WARNING, /\[WARN\] :: spec :: %s/, "Yowza!")
|
27
|
+
Syslog.should_receive(:close)
|
28
28
|
end
|
29
29
|
|
30
30
|
flogger = Flapjack::Logger.new('spec', 'level' => 'debug')
|
@@ -21,17 +21,19 @@ describe Flapjack::Pikelet do
|
|
21
21
|
|
22
22
|
config.should_receive(:[]).with('logger').and_return(nil)
|
23
23
|
|
24
|
+
fc = mock('coordinator')
|
25
|
+
|
24
26
|
processor = mock('processor')
|
25
27
|
processor.should_receive(:start)
|
26
28
|
Flapjack::Processor.should_receive(:new).with(:config => config,
|
27
|
-
:redis_config => redis_config, :boot_time => time, :logger => logger).
|
29
|
+
:redis_config => redis_config, :boot_time => time, :logger => logger, :coordinator => fc).
|
28
30
|
and_return(processor)
|
29
31
|
|
30
32
|
fiber.should_receive(:resume)
|
31
33
|
Fiber.should_receive(:new).and_yield.and_return(fiber)
|
32
34
|
|
33
35
|
pik = Flapjack::Pikelet.create('processor', :config => config,
|
34
|
-
:redis_config => redis_config, :boot_time => time)
|
36
|
+
:redis_config => redis_config, :boot_time => time, :coordinator => fc)
|
35
37
|
pik.should be_a(Flapjack::Pikelet::Generic)
|
36
38
|
pik.start
|
37
39
|
end
|
@@ -1,11 +1,14 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
require 'flapjack/processor'
|
3
|
+
require 'flapjack/coordinator'
|
3
4
|
|
4
5
|
describe Flapjack::Processor, :logger => true do
|
5
6
|
|
6
7
|
# NB: this is only testing the public API of the Processor class, which is pretty limited.
|
7
8
|
# (initialize, main, stop). Most test coverage for this class comes from the cucumber features.
|
8
9
|
|
10
|
+
let(:config) { mock(Flapjack::Configuration) }
|
11
|
+
|
9
12
|
# TODO this does too much -- split it up
|
10
13
|
it "starts up, runs and shuts down" do
|
11
14
|
t = Time.now.to_i
|
@@ -37,20 +40,26 @@ describe Flapjack::Processor, :logger => true do
|
|
37
40
|
# redis.should_receive(:hincrby).with('event_counters', 'all', 1)
|
38
41
|
# redis.should_receive(:hincrby).with(/^event_counters:/, 'all', 1)
|
39
42
|
|
40
|
-
Flapjack::Data::Event.should_receive(:pending_count).with(:redis => redis).and_return(0)
|
43
|
+
Flapjack::Data::Event.should_receive(:pending_count).with('events', :redis => redis).and_return(0)
|
41
44
|
|
42
45
|
Flapjack::RedisPool.should_receive(:new).and_return(redis)
|
43
|
-
executive = Flapjack::Processor.new(:config => {}, :logger => @logger)
|
44
46
|
|
45
|
-
|
46
|
-
|
47
|
-
|
47
|
+
fc = mock('coordinator')
|
48
|
+
|
49
|
+
executive = Flapjack::Processor.new(:config => {}, :logger => @logger, :coordinator => fc)
|
50
|
+
|
51
|
+
noop_evt = mock(Flapjack::Data::Event)
|
52
|
+
noop_evt.should_receive(:inspect)
|
53
|
+
noop_evt.should_receive(:type).and_return('noop')
|
48
54
|
Flapjack::Data::Event.should_receive(:next) {
|
49
55
|
executive.instance_variable_set('@should_quit', true)
|
50
|
-
|
56
|
+
noop_evt
|
51
57
|
}
|
52
58
|
|
53
|
-
|
59
|
+
begin
|
60
|
+
executive.start
|
61
|
+
rescue SystemExit
|
62
|
+
end
|
54
63
|
end
|
55
64
|
|
56
65
|
end
|
@@ -0,0 +1,228 @@
|
|
1
|
+
require 'redis'
|
2
|
+
require 'oj'
|
3
|
+
require 'time'
|
4
|
+
|
5
|
+
namespace :benchmarks do
|
6
|
+
|
7
|
+
# add lib to the default include path
|
8
|
+
unless $:.include?(File.dirname(__FILE__) + '/../lib/')
|
9
|
+
$: << File.dirname(__FILE__) + '/../lib'
|
10
|
+
end
|
11
|
+
|
12
|
+
require 'flapjack/configuration'
|
13
|
+
require 'flapjack/data/event'
|
14
|
+
require 'flapjack/data/entity_check'
|
15
|
+
require 'flapjack/version'
|
16
|
+
|
17
|
+
FLAPJACK_ENV = ENV['FLAPJACK_ENV'] || 'test'
|
18
|
+
config_file = File.join('tasks', 'support', 'flapjack_config_benchmark.yaml')
|
19
|
+
|
20
|
+
config = Flapjack::Configuration.new
|
21
|
+
config.load( config_file )
|
22
|
+
|
23
|
+
@config_env = config.all
|
24
|
+
@redis_config = config.for_redis
|
25
|
+
|
26
|
+
if @config_env.nil? || @config_env.empty?
|
27
|
+
puts "No config data for environment '#{FLAPJACK_ENV}' found in '#{config_file}'"
|
28
|
+
exit(false)
|
29
|
+
end
|
30
|
+
|
31
|
+
redis = Redis.new(@redis_config)
|
32
|
+
|
33
|
+
desc "nukes the redis db, generates the events, runs and shuts down flapjack, generates perftools reports"
|
34
|
+
task :run => [:reset_redis, :benchmark, :run_flapjack, :reports] do
|
35
|
+
puts Oj.dump(@benchmark_data, :indent => 2)
|
36
|
+
end
|
37
|
+
|
38
|
+
desc "reset the redis database"
|
39
|
+
task :reset_redis do
|
40
|
+
raise "I'm not going to let you reset your production redis db, sorry about that." if FLAPJACK_ENV.downcase == "production"
|
41
|
+
puts "db size before: #{redis.dbsize}"
|
42
|
+
redis.flushdb
|
43
|
+
puts "db size after: #{redis.dbsize}"
|
44
|
+
end
|
45
|
+
|
46
|
+
desc "starts flapjack"
|
47
|
+
task :run_flapjack do
|
48
|
+
puts "Discovering path to perftools"
|
49
|
+
perftools = `gem which perftools | tail -1`
|
50
|
+
if system("if [ ! -d 'artifacts' ] ; then mkdir artifacts ; fi")
|
51
|
+
puts "we now have an artifacts dir"
|
52
|
+
else
|
53
|
+
raise "Problem creating artifacts: #{$?}"
|
54
|
+
end
|
55
|
+
time_flapjack_start = Time.now.to_f
|
56
|
+
puts "Starting flapjack..."
|
57
|
+
if system({"FLAPJACK_ENV" => FLAPJACK_ENV,
|
58
|
+
"CPUPROFILE" => "artifacts/flapjack-perftools-cpuprofile",
|
59
|
+
"RUBYOPT" => "-r#{perftools}"},
|
60
|
+
"bin/flapjack start --no-daemonize --config tasks/support/flapjack_config_benchmark.yaml")
|
61
|
+
puts "Flapjack run completed successfully"
|
62
|
+
else
|
63
|
+
raise "Problem starting flapjack: #{$?}"
|
64
|
+
end
|
65
|
+
@timer_flapjack = Time.now.to_f - time_flapjack_start
|
66
|
+
end
|
67
|
+
|
68
|
+
desc "generates perftools reports"
|
69
|
+
task :reports do
|
70
|
+
@benchmark_data = { 'events_created' => @events_created,
|
71
|
+
'flapjack_runtime' => @timer_flapjack,
|
72
|
+
'processing_rate' => @events_created.to_f / @timer_flapjack }.merge(@benchmark_parameters)
|
73
|
+
bytes_written = IO.write('artifacts/benchmark_data.json', Oj.dump(@benchmark_data, :indent => 2))
|
74
|
+
puts "benchmark data written to artifacts/benchmark_data.json (#{bytes_written} bytes)"
|
75
|
+
|
76
|
+
if system("pprof.rb --text artifacts/flapjack-perftools-cpuprofile > artifacts/flapjack-perftools-cpuprofile.txt")
|
77
|
+
puts "Generated perftools.rb text report at artifacts/flapjack-perftools-cpuprofile.txt"
|
78
|
+
system("head -24 artifacts/flapjack-perftools-cpuprofile.txt")
|
79
|
+
else
|
80
|
+
raise "Problem generating perftools.rb text report: #{$?}"
|
81
|
+
end
|
82
|
+
if system("pprof.rb --pdf artifacts/flapjack-perftools-cpuprofile > artifacts/flapjack-perftools-cpuprofile.pdf")
|
83
|
+
puts "Generated perftools.rb pdf report at artifacts/flapjack-perftools-cpuprofile.pdf"
|
84
|
+
else
|
85
|
+
raise "Problem generating perftools.rb pdf report: #{$?}"
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
|
90
|
+
desc "run benchmark - simulate a stream of events from the check execution system"
|
91
|
+
# Assumptions:
|
92
|
+
# - time to failure varies evenly between 1 hour and 1 month
|
93
|
+
# - time to recovery varies evenly between 10 seconds and 1 week
|
94
|
+
task :benchmark do
|
95
|
+
|
96
|
+
num_checks_per_entity = (ENV['CHECKS_PER_ENTITY'] || 5).to_i
|
97
|
+
num_entities = (ENV['ENTITIES'] || 100).to_i
|
98
|
+
interval = (ENV['INTERVAL'] || 60).to_i
|
99
|
+
hours = (ENV['HOURS'] || 1).to_f
|
100
|
+
seed = (ENV['SEED'] || 42).to_i
|
101
|
+
|
102
|
+
puts "Behaviour can be modified by setting any combination of the following environment variables: "
|
103
|
+
puts "CHECKS_PER_ENTITY - #{num_checks_per_entity}"
|
104
|
+
puts "ENTITIES - #{num_entities}"
|
105
|
+
puts "INTERVAL - #{interval}"
|
106
|
+
puts "HOURS - #{hours}"
|
107
|
+
puts "SEED - #{seed}"
|
108
|
+
puts "FLAPJACK_ENV - #{FLAPJACK_ENV}"
|
109
|
+
|
110
|
+
raise "INTERVAL must be less than (or equal to) 3600 seconds (1 hour)" unless interval <= 3600
|
111
|
+
|
112
|
+
cycles_per_hour = (60.0 * 60) / interval
|
113
|
+
cycles_per_day = (60.0 * 60 * 24) / interval
|
114
|
+
cycles_per_week = (60.0 * 60 * 24 * 7) / interval
|
115
|
+
cycles_per_month = (60.0 * 60 * 24 * 7 * 30) / interval
|
116
|
+
cycles = (hours * cycles_per_hour).to_i
|
117
|
+
failure_prob_min = 1.0 / cycles_per_month
|
118
|
+
failure_prob_max = 1.0 / cycles_per_hour
|
119
|
+
recovery_prob_min = 1.0 / cycles_per_week
|
120
|
+
recovery_prob_max = 1.0
|
121
|
+
initial_ok_prob = 1
|
122
|
+
num_checks = num_checks_per_entity * num_entities
|
123
|
+
|
124
|
+
prng = Random.new(seed)
|
125
|
+
|
126
|
+
ok = 0
|
127
|
+
critical = 0
|
128
|
+
check_id = 1
|
129
|
+
entities = (1..num_entities).to_a.inject({}) {|memo, id|
|
130
|
+
checks = (1..num_checks_per_entity).to_a.inject({}) {|memo_check, id_check|
|
131
|
+
memo_check[check_id] = {:name => "Check Type #{id_check}",
|
132
|
+
:state => ( prng.rand < initial_ok_prob ? 'OK' : 'CRITICAL' ),
|
133
|
+
:p_failure => prng.rand(failure_prob_min..failure_prob_max),
|
134
|
+
:p_recovery => prng.rand(recovery_prob_min..recovery_prob_max)}
|
135
|
+
ok += 1 if memo_check[check_id][:state] == 'OK'
|
136
|
+
critical += 1 if memo_check[check_id][:state] == 'CRITICAL'
|
137
|
+
check_id += 1
|
138
|
+
memo_check
|
139
|
+
}
|
140
|
+
memo[id] = checks
|
141
|
+
memo
|
142
|
+
}
|
143
|
+
#puts "ok: #{ok * 100.0 / num_checks}% (#{ok}), critical: #{100.0 * critical / num_checks}% (#{critical})"
|
144
|
+
|
145
|
+
events_created = 0
|
146
|
+
ok_to_critical = 0
|
147
|
+
critical_to_ok = 0
|
148
|
+
ok_events = 0
|
149
|
+
critical_events = 0
|
150
|
+
state_changes = 0
|
151
|
+
(0..cycles).to_a.each {|i|
|
152
|
+
changes = 0
|
153
|
+
ok = 0
|
154
|
+
critical = 0
|
155
|
+
summary = "You tell me summer's here \nand the time is wrong \n"
|
156
|
+
summary << "You tell me winter's here \nAnd your days are getting long"
|
157
|
+
entities.each_pair {|entity_id, checks|
|
158
|
+
checks.each_pair {|check_id, check|
|
159
|
+
changed = false
|
160
|
+
previous_state = check[:state]
|
161
|
+
case previous_state
|
162
|
+
when "OK"
|
163
|
+
if prng.rand < check[:p_failure]
|
164
|
+
check[:state] = "CRITICAL"
|
165
|
+
changed = true
|
166
|
+
changes += 1
|
167
|
+
ok_to_critical += 1
|
168
|
+
end
|
169
|
+
when "CRITICAL"
|
170
|
+
if prng.rand < check[:p_recovery]
|
171
|
+
check[:state] = "OK"
|
172
|
+
changed = true
|
173
|
+
changes += 1
|
174
|
+
critical_to_ok += 1
|
175
|
+
end
|
176
|
+
end
|
177
|
+
ok += 1 if check[:state] == 'OK'
|
178
|
+
critical += 1 if check[:state] == 'CRITICAL'
|
179
|
+
|
180
|
+
Flapjack::Data::Event.add({'entity' => "entity_#{entity_id}.example.com",
|
181
|
+
'check' => check[:name],
|
182
|
+
'type' => 'service',
|
183
|
+
'state' => check[:state],
|
184
|
+
'summary' => summary }, :redis => redis)
|
185
|
+
events_created += 1
|
186
|
+
}
|
187
|
+
}
|
188
|
+
ok_events += ok
|
189
|
+
critical_events += critical
|
190
|
+
state_changes += changes
|
191
|
+
|
192
|
+
#puts "ok: #{100.0 * ok / num_checks}% (#{ok}), critical: #{100.0 * critical / num_checks}% (#{critical}), changed: #{100.0 * changes / num_checks}% (#{changes})"
|
193
|
+
|
194
|
+
}
|
195
|
+
puts "created #{events_created} events:"
|
196
|
+
puts " OK: #{ok_events} (#{ (100.0 * ok_events / events_created).round(1)}%)"
|
197
|
+
puts " CRITICAL: #{critical_events} (#{ (100.0 * critical_events / events_created).round(1)}%)"
|
198
|
+
puts "containing #{state_changes} state changes (#{ (100.0 * state_changes / events_created).round(1)}%):"
|
199
|
+
puts " OK -> CRITICAL: #{ok_to_critical} (#{ (100.0 * ok_to_critical / events_created).round(1)}%)"
|
200
|
+
puts " CRITICAL -> OK: #{critical_to_ok} (#{ (100.0 * critical_to_ok / events_created).round(1)}%)"
|
201
|
+
|
202
|
+
@events_created = events_created
|
203
|
+
@benchmark_parameters = { 'events_created' => events_created,
|
204
|
+
'ok_to_critical' => ok_to_critical,
|
205
|
+
'critical_to_ok' => critical_to_ok,
|
206
|
+
'checks_per_entity' => num_checks_per_entity,
|
207
|
+
'entities' => num_entities,
|
208
|
+
'interval' => interval,
|
209
|
+
'hours' => hours,
|
210
|
+
'cycles' => cycles,
|
211
|
+
'failure_prob_min' => failure_prob_min,
|
212
|
+
'failure_prob_max' => failure_prob_max,
|
213
|
+
'recovery_prob_min' => recovery_prob_min,
|
214
|
+
'recovery_prob_max' => recovery_prob_max,
|
215
|
+
'initial_ok_prob' => initial_ok_prob,
|
216
|
+
'seed' => seed,
|
217
|
+
'flapjack_env' => FLAPJACK_ENV,
|
218
|
+
'version' => Flapjack::VERSION,
|
219
|
+
'git_last_commit' => `git rev-parse HEAD`.chomp,
|
220
|
+
'git_version' => `git describe --long --dirty --abbrev=10 --tags`.chomp,
|
221
|
+
'git_branch' => `git status --porcelain -b | head -1 | cut -d ' ' -f 2`.chomp,
|
222
|
+
'ruby_build' => `ruby --version`.chomp,
|
223
|
+
'time' => Time.new.iso8601,
|
224
|
+
'hostname' => `hostname -f`.chomp,
|
225
|
+
'uname' => `uname -a`.chomp }
|
226
|
+
end
|
227
|
+
|
228
|
+
end
|