flapjack 0.7.20 → 0.7.21

Sign up to get free protection for your applications and to get access to all the features.
Files changed (49) hide show
  1. data/.gitignore +1 -0
  2. data/.travis.yml +3 -1
  3. data/CHANGELOG.md +10 -0
  4. data/Gemfile +1 -0
  5. data/bin/flapjack +11 -0
  6. data/bin/simulate-failed-check +5 -5
  7. data/features/notification_rules.feature +77 -19
  8. data/features/steps/events_steps.rb +15 -3
  9. data/lib/flapjack/coordinator.rb +3 -3
  10. data/lib/flapjack/data/contact.rb +1 -1
  11. data/lib/flapjack/data/entity.rb +12 -1
  12. data/lib/flapjack/data/entity_check.rb +9 -2
  13. data/lib/flapjack/data/event.rb +4 -4
  14. data/lib/flapjack/data/notification.rb +27 -20
  15. data/lib/flapjack/data/notification_rule.rb +26 -24
  16. data/lib/flapjack/data/tag.rb +5 -0
  17. data/lib/flapjack/gateways/api.rb +1 -1
  18. data/lib/flapjack/gateways/api/contact_methods.rb +3 -3
  19. data/lib/flapjack/gateways/email.rb +73 -46
  20. data/lib/flapjack/gateways/email/alert.html.erb +13 -4
  21. data/lib/flapjack/gateways/email/alert.text.erb +2 -2
  22. data/lib/flapjack/gateways/jabber.rb +22 -16
  23. data/lib/flapjack/gateways/pagerduty.rb +7 -3
  24. data/lib/flapjack/gateways/web.rb +1 -1
  25. data/lib/flapjack/gateways/web/views/check.html.erb +2 -2
  26. data/lib/flapjack/gateways/web/views/contact.html.erb +3 -3
  27. data/lib/flapjack/logger.rb +67 -35
  28. data/lib/flapjack/notifier.rb +9 -3
  29. data/lib/flapjack/pikelet.rb +3 -1
  30. data/lib/flapjack/processor.rb +34 -10
  31. data/lib/flapjack/version.rb +1 -1
  32. data/spec/lib/flapjack/coordinator_spec.rb +17 -13
  33. data/spec/lib/flapjack/data/contact_spec.rb +4 -3
  34. data/spec/lib/flapjack/data/entity_check_spec.rb +10 -0
  35. data/spec/lib/flapjack/data/entity_spec.rb +60 -5
  36. data/spec/lib/flapjack/data/event_spec.rb +4 -4
  37. data/spec/lib/flapjack/data/notification_rule_spec.rb +9 -2
  38. data/spec/lib/flapjack/data/tag_spec.rb +0 -1
  39. data/spec/lib/flapjack/gateways/api/contact_methods_spec.rb +1 -1
  40. data/spec/lib/flapjack/gateways/email_spec.rb +2 -1
  41. data/spec/lib/flapjack/gateways/jabber_spec.rb +5 -3
  42. data/spec/lib/flapjack/gateways/pagerduty_spec.rb +3 -1
  43. data/spec/lib/flapjack/logger_spec.rb +5 -5
  44. data/spec/lib/flapjack/pikelet_spec.rb +4 -2
  45. data/spec/lib/flapjack/processor_spec.rb +16 -7
  46. data/tasks/benchmarks.rake +228 -0
  47. data/tasks/events.rake +11 -10
  48. data/tasks/support/flapjack_config_benchmark.yaml +58 -0
  49. metadata +6 -4
@@ -6,11 +6,12 @@ require 'ice_cube'
6
6
  require 'flapjack/data/contact'
7
7
  require 'flapjack/data/entity_check'
8
8
  require 'flapjack/data/notification_rule'
9
+ require 'flapjack/data/tag_set'
9
10
 
10
11
  describe Flapjack::Data::Contact, :redis => true do
11
12
 
12
13
  let(:notification_rule_data) {
13
- {:entity_tags => ["database","physical"],
14
+ {:tags => ["database","physical"],
14
15
  :entities => ["foo-app-01.example.com"],
15
16
  :time_restrictions => [],
16
17
  :warning_media => ["email"],
@@ -22,7 +23,7 @@ describe Flapjack::Data::Contact, :redis => true do
22
23
 
23
24
  let(:general_notification_rule_data) {
24
25
  {:entities => [],
25
- :entity_tags => [],
26
+ :tags => Flapjack::Data::TagSet.new([]),
26
27
  :time_restrictions => [],
27
28
  :warning_media => ['email', 'sms', 'jabber', 'pagerduty'],
28
29
  :critical_media => ['email', 'sms', 'jabber', 'pagerduty'],
@@ -156,7 +157,7 @@ describe Flapjack::Data::Contact, :redis => true do
156
157
  rules = contact.notification_rules
157
158
  rules.should have(1).rule
158
159
  rule = rules.first
159
- [:entities, :entity_tags, :time_restrictions,
160
+ [:entities, :tags, :time_restrictions,
160
161
  :warning_media, :critical_media,
161
162
  :warning_blackhole, :critical_blackhole].each do |k|
162
163
  rule.send(k).should == general_notification_rule_data[k]
@@ -2,6 +2,7 @@ require 'spec_helper'
2
2
 
3
3
  require 'flapjack/data/entity'
4
4
  require 'flapjack/data/entity_check'
5
+ require 'flapjack/data/tag_set'
5
6
 
6
7
  describe Flapjack::Data::EntityCheck, :redis => true do
7
8
 
@@ -574,4 +575,13 @@ describe Flapjack::Data::EntityCheck, :redis => true do
574
575
  contacts.should have(1).contact
575
576
  contacts.first.name.should == 'John Johnson'
576
577
  end
578
+
579
+ it "generates ephemeral tags for itself" do
580
+ ec = Flapjack::Data::EntityCheck.for_entity_name('foo-app-01.example.com', 'Disk / Utilisation', :redis => @redis)
581
+ tags = ec.tags
582
+ tags.should_not be_nil
583
+ tags.should be_a(Flapjack::Data::TagSet)
584
+ ['foo-app-01', 'example.com', 'disk', '/', 'utilisation'].to_set.subset?(tags).should be_true
585
+ end
586
+
577
587
  end
@@ -176,16 +176,71 @@ describe Flapjack::Data::Entity, :redis => true do
176
176
  end
177
177
 
178
178
  it "finds entities by tag" do
179
- entity = Flapjack::Data::Entity.add({'id' => '5000',
179
+ entity0 = Flapjack::Data::Entity.add({'id' => '5000',
180
+ 'name' => 'abc-123',
181
+ 'contacts' => []},
182
+ :redis => @redis)
183
+
184
+ entity1 = Flapjack::Data::Entity.add({'id' => '5001',
185
+ 'name' => 'def-456',
186
+ 'contacts' => []},
187
+ :redis => @redis)
188
+
189
+ entity0.add_tags('source:foobar', 'abc')
190
+ entity1.add_tags('source:foobar', 'def')
191
+
192
+ entity0.should_not be_nil
193
+ entity0.should be_an(Flapjack::Data::Entity)
194
+ entity0.tags.should include("source:foobar")
195
+ entity0.tags.should include("abc")
196
+ entity0.tags.should_not include("def")
197
+ entity1.should_not be_nil
198
+ entity1.should be_an(Flapjack::Data::Entity)
199
+ entity1.tags.should include("source:foobar")
200
+ entity1.tags.should include("def")
201
+ entity1.tags.should_not include("abc")
202
+
203
+ entities = Flapjack::Data::Entity.find_all_with_tags(['abc'], :redis => @redis)
204
+ entities.should be_an(Array)
205
+ entities.should have(1).entity
206
+ entities.first.should == 'abc-123'
207
+
208
+ entities = Flapjack::Data::Entity.find_all_with_tags(['donkey'], :redis => @redis)
209
+ entities.should be_an(Array)
210
+ entities.should have(0).entities
211
+ end
212
+
213
+ it "finds entities with several tags" do
214
+ entity0 = Flapjack::Data::Entity.add({'id' => '5000',
180
215
  'name' => 'abc-123',
181
216
  'contacts' => []},
182
217
  :redis => @redis)
183
218
 
184
- entity.add_tags('source:foobar', 'foo')
219
+ entity1 = Flapjack::Data::Entity.add({'id' => '5001',
220
+ 'name' => 'def-456',
221
+ 'contacts' => []},
222
+ :redis => @redis)
185
223
 
186
- entity.should_not be_nil
187
- entity.should be_an(Flapjack::Data::Entity)
188
- # TODO - the rest of it
224
+ entity0.add_tags('source:foobar', 'abc')
225
+ entity1.add_tags('source:foobar', 'def')
226
+
227
+ entity0.should_not be_nil
228
+ entity0.should be_an(Flapjack::Data::Entity)
229
+ entity0.tags.should include("source:foobar")
230
+ entity0.tags.should include("abc")
231
+ entity1.should_not be_nil
232
+ entity1.should be_an(Flapjack::Data::Entity)
233
+ entity1.tags.should include("source:foobar")
234
+ entity1.tags.should include("def")
235
+
236
+ entities = Flapjack::Data::Entity.find_all_with_tags(['source:foobar'], :redis => @redis)
237
+ entities.should be_an(Array)
238
+ entities.should have(2).entity
239
+
240
+ entities = Flapjack::Data::Entity.find_all_with_tags(['source:foobar', 'def'], :redis => @redis)
241
+ entities.should be_an(Array)
242
+ entities.should have(1).entity
243
+ entities.first.should == 'def-456'
189
244
  end
190
245
 
191
246
  end
@@ -58,13 +58,13 @@ describe Flapjack::Data::Event do
58
58
  events_len = 23
59
59
  mock_redis.should_receive(:llen).with('events').and_return(events_len)
60
60
 
61
- pc = Flapjack::Data::Event.pending_count(:redis => mock_redis)
61
+ pc = Flapjack::Data::Event.pending_count('events', :redis => mock_redis)
62
62
  pc.should == events_len
63
63
  end
64
64
 
65
65
  it "creates a notification testing event" do
66
66
  Time.should_receive(:now).and_return(time)
67
- mock_redis.should_receive(:rpush).with('events', /"testing"/ )
67
+ mock_redis.should_receive(:lpush).with('events', /"testing"/ )
68
68
 
69
69
  Flapjack::Data::Event.test_notifications(entity_name, check,
70
70
  :summary => 'test', :details => 'testing', :redis => mock_redis)
@@ -72,7 +72,7 @@ describe Flapjack::Data::Event do
72
72
 
73
73
  it "creates an acknowledgement event" do
74
74
  Time.should_receive(:now).and_return(time)
75
- mock_redis.should_receive(:rpush).with('events', /"acking"/ )
75
+ mock_redis.should_receive(:lpush).with('events', /"acking"/ )
76
76
 
77
77
  Flapjack::Data::Event.create_acknowledgement(entity_name, check,
78
78
  :summary => 'acking', :time => time.to_i, :redis => mock_redis)
@@ -98,4 +98,4 @@ describe Flapjack::Data::Event do
98
98
  it { should be_a_failure }
99
99
  end
100
100
 
101
- end
101
+ end
@@ -14,7 +14,7 @@ describe Flapjack::Data::NotificationRule, :redis => true do
14
14
 
15
15
  let(:rule_data) {
16
16
  {:contact_id => '23',
17
- :entity_tags => ["database","physical"],
17
+ :tags => ["database","physical"],
18
18
  :entities => ["foo-app-01.example.com"],
19
19
  :time_restrictions => [ weekdays_8_18 ],
20
20
  :warning_media => ["email"],
@@ -76,7 +76,14 @@ describe Flapjack::Data::NotificationRule, :redis => true do
76
76
  rule.match_entity?('foo-app-02.example.com').should be_false
77
77
  end
78
78
 
79
- pending "check whether entity tags match"
79
+ it "checks whether entity tags match" do
80
+ rule = existing_rule
81
+
82
+ rule.match_tags?(['database', 'physical'].to_set).should be_true
83
+ rule.match_tags?(['database', 'physical', 'beetroot'].to_set).should be_true
84
+ rule.match_tags?(['database'].to_set).should be_false
85
+ rule.match_tags?(['virtual'].to_set).should be_false
86
+ end
80
87
 
81
88
  it "checks if blackhole settings for a rule match a severity level" do
82
89
  rule_data[:warning_blackhole] = true
@@ -1,5 +1,4 @@
1
1
  require 'spec_helper'
2
- require 'flapjack/data/entity'
3
2
 
4
3
  describe Flapjack::Data::Tag, :redis => true do
5
4
 
@@ -37,7 +37,7 @@ describe 'Flapjack::Gateways::API::ContactMethods', :sinatra => true, :logger =>
37
37
 
38
38
  let(:notification_rule_data) {
39
39
  {"contact_id" => "21",
40
- "entity_tags" => ["database","physical"],
40
+ "tags" => ["database","physical"],
41
41
  "entities" => ["foo-app-01.example.com"],
42
42
  "time_restrictions" => nil,
43
43
  "warning_media" => ["email"],
@@ -15,6 +15,7 @@ describe Flapjack::Gateways::Email, :logger => true do
15
15
  entity_check = mock(Flapjack::Data::EntityCheck)
16
16
  entity_check.should_receive(:in_scheduled_maintenance?).and_return(false)
17
17
  entity_check.should_receive(:in_unscheduled_maintenance?).and_return(false)
18
+ entity_check.should_receive(:last_change).and_return(Time.now.to_i)
18
19
 
19
20
  redis = mock('redis')
20
21
  ::Resque.should_receive(:redis).and_return(redis)
@@ -46,4 +47,4 @@ describe Flapjack::Gateways::Email, :logger => true do
46
47
  Flapjack::Gateways::Email.perform(notification)
47
48
  end
48
49
 
49
- end
50
+ end
@@ -151,9 +151,11 @@ describe Flapjack::Gateways::Jabber, :logger => true do
151
151
  end
152
152
 
153
153
  it "prompts the blocking redis connection to quit" do
154
- redis = mock('redis')
155
- redis.should_receive(:rpush).with('jabber_notifications', %q{{"notification_type":"shutdown"}})
154
+ shutdown_redis = mock('shutdown_redis')
155
+ shutdown_redis.should_receive(:rpush).with('jabber_notifications', %q{{"notification_type":"shutdown"}})
156
+ EM::Hiredis.should_receive(:connect).and_return(shutdown_redis)
156
157
 
158
+ redis = mock('redis')
157
159
  Flapjack::RedisPool.should_receive(:new).and_return(redis)
158
160
  fj = Flapjack::Gateways::Jabber.new(:config => config, :logger => @logger)
159
161
 
@@ -163,7 +165,7 @@ describe Flapjack::Gateways::Jabber, :logger => true do
163
165
  it "runs a blocking loop listening for notifications" do
164
166
  timer = mock('timer')
165
167
  timer.should_receive(:cancel)
166
- EM::Synchrony.should_receive(:add_periodic_timer).with(60).and_return(timer)
168
+ EM::Synchrony.should_receive(:add_periodic_timer).with(1).and_return(timer)
167
169
 
168
170
  redis = mock('redis')
169
171
 
@@ -11,7 +11,9 @@ describe Flapjack::Gateways::Pagerduty, :logger => true do
11
11
  let(:redis) { mock('redis') }
12
12
 
13
13
  it "prompts the blocking redis connection to quit" do
14
- redis.should_receive(:rpush).with(config['queue'], %q{{"notification_type":"shutdown"}})
14
+ shutdown_redis = mock('shutdown_redis')
15
+ shutdown_redis.should_receive(:rpush).with(config['queue'], %q{{"notification_type":"shutdown"}})
16
+ EM::Hiredis.should_receive(:connect).and_return(shutdown_redis)
15
17
 
16
18
  Flapjack::RedisPool.should_receive(:new).and_return(redis)
17
19
  fp = Flapjack::Gateways::Pagerduty.new(:config => config, :logger => @logger)
@@ -10,21 +10,21 @@ describe Flapjack::Logger do
10
10
  it "creates a logger logging to STDOUT and syslog" do
11
11
  logger.should_receive(:formatter=).with(an_instance_of(Proc))
12
12
  logger.should_receive(:level=).and_return(Logger::DEBUG)
13
- logger.should_receive(:warn).with("Yowza!")
13
+ logger.should_receive(:add).with(2, nil, "Yowza!")
14
14
  ::Logger.should_receive(:new).with(STDOUT).and_return(logger)
15
15
 
16
16
  if Syslog.const_defined?('Logger', false)
17
17
  sys_logger.should_receive(:formatter=).with(an_instance_of(Proc))
18
18
  sys_logger.should_receive(:level=).with(Logger::DEBUG)
19
- sys_logger.should_receive(:warn).with("Yowza!")
19
+ sys_logger.should_receive(:add).with(Logger::WARN, 'Yowza!', 'flapjack')
20
20
  Syslog.const_get('Logger', false).should_receive(:new).with('flapjack').and_return(sys_logger)
21
21
  else
22
- syslog.should_receive(:log).with(Syslog::Constants::LOG_WARNING, /\[WARN\] :: spec :: %s/, "Yowza!")
23
- Syslog.should_receive(:"opened?").and_return(false)
24
22
  Syslog.should_receive(:open).with('flapjack',
25
23
  (Syslog::Constants::LOG_PID | Syslog::Constants::LOG_CONS),
26
24
  Syslog::Constants::LOG_USER).and_return(syslog)
27
- Syslog.should_receive(:mask=).with(Syslog::LOG_UPTO(Syslog::Constants::LOG_DEBUG))
25
+ Syslog.should_receive(:mask=).with(Syslog::LOG_UPTO(Syslog::Constants::LOG_WARNING))
26
+ Syslog.should_receive(:log).with(Syslog::Constants::LOG_WARNING, /\[WARN\] :: spec :: %s/, "Yowza!")
27
+ Syslog.should_receive(:close)
28
28
  end
29
29
 
30
30
  flogger = Flapjack::Logger.new('spec', 'level' => 'debug')
@@ -21,17 +21,19 @@ describe Flapjack::Pikelet do
21
21
 
22
22
  config.should_receive(:[]).with('logger').and_return(nil)
23
23
 
24
+ fc = mock('coordinator')
25
+
24
26
  processor = mock('processor')
25
27
  processor.should_receive(:start)
26
28
  Flapjack::Processor.should_receive(:new).with(:config => config,
27
- :redis_config => redis_config, :boot_time => time, :logger => logger).
29
+ :redis_config => redis_config, :boot_time => time, :logger => logger, :coordinator => fc).
28
30
  and_return(processor)
29
31
 
30
32
  fiber.should_receive(:resume)
31
33
  Fiber.should_receive(:new).and_yield.and_return(fiber)
32
34
 
33
35
  pik = Flapjack::Pikelet.create('processor', :config => config,
34
- :redis_config => redis_config, :boot_time => time)
36
+ :redis_config => redis_config, :boot_time => time, :coordinator => fc)
35
37
  pik.should be_a(Flapjack::Pikelet::Generic)
36
38
  pik.start
37
39
  end
@@ -1,11 +1,14 @@
1
1
  require 'spec_helper'
2
2
  require 'flapjack/processor'
3
+ require 'flapjack/coordinator'
3
4
 
4
5
  describe Flapjack::Processor, :logger => true do
5
6
 
6
7
  # NB: this is only testing the public API of the Processor class, which is pretty limited.
7
8
  # (initialize, main, stop). Most test coverage for this class comes from the cucumber features.
8
9
 
10
+ let(:config) { mock(Flapjack::Configuration) }
11
+
9
12
  # TODO this does too much -- split it up
10
13
  it "starts up, runs and shuts down" do
11
14
  t = Time.now.to_i
@@ -37,20 +40,26 @@ describe Flapjack::Processor, :logger => true do
37
40
  # redis.should_receive(:hincrby).with('event_counters', 'all', 1)
38
41
  # redis.should_receive(:hincrby).with(/^event_counters:/, 'all', 1)
39
42
 
40
- Flapjack::Data::Event.should_receive(:pending_count).with(:redis => redis).and_return(0)
43
+ Flapjack::Data::Event.should_receive(:pending_count).with('events', :redis => redis).and_return(0)
41
44
 
42
45
  Flapjack::RedisPool.should_receive(:new).and_return(redis)
43
- executive = Flapjack::Processor.new(:config => {}, :logger => @logger)
44
46
 
45
- shutdown_evt = mock(Flapjack::Data::Event)
46
- shutdown_evt.should_receive(:inspect)
47
- shutdown_evt.should_receive(:type).and_return('shutdown')
47
+ fc = mock('coordinator')
48
+
49
+ executive = Flapjack::Processor.new(:config => {}, :logger => @logger, :coordinator => fc)
50
+
51
+ noop_evt = mock(Flapjack::Data::Event)
52
+ noop_evt.should_receive(:inspect)
53
+ noop_evt.should_receive(:type).and_return('noop')
48
54
  Flapjack::Data::Event.should_receive(:next) {
49
55
  executive.instance_variable_set('@should_quit', true)
50
- shutdown_evt
56
+ noop_evt
51
57
  }
52
58
 
53
- executive.start
59
+ begin
60
+ executive.start
61
+ rescue SystemExit
62
+ end
54
63
  end
55
64
 
56
65
  end
@@ -0,0 +1,228 @@
1
+ require 'redis'
2
+ require 'oj'
3
+ require 'time'
4
+
5
+ namespace :benchmarks do
6
+
7
+ # add lib to the default include path
8
+ unless $:.include?(File.dirname(__FILE__) + '/../lib/')
9
+ $: << File.dirname(__FILE__) + '/../lib'
10
+ end
11
+
12
+ require 'flapjack/configuration'
13
+ require 'flapjack/data/event'
14
+ require 'flapjack/data/entity_check'
15
+ require 'flapjack/version'
16
+
17
+ FLAPJACK_ENV = ENV['FLAPJACK_ENV'] || 'test'
18
+ config_file = File.join('tasks', 'support', 'flapjack_config_benchmark.yaml')
19
+
20
+ config = Flapjack::Configuration.new
21
+ config.load( config_file )
22
+
23
+ @config_env = config.all
24
+ @redis_config = config.for_redis
25
+
26
+ if @config_env.nil? || @config_env.empty?
27
+ puts "No config data for environment '#{FLAPJACK_ENV}' found in '#{config_file}'"
28
+ exit(false)
29
+ end
30
+
31
+ redis = Redis.new(@redis_config)
32
+
33
+ desc "nukes the redis db, generates the events, runs and shuts down flapjack, generates perftools reports"
34
+ task :run => [:reset_redis, :benchmark, :run_flapjack, :reports] do
35
+ puts Oj.dump(@benchmark_data, :indent => 2)
36
+ end
37
+
38
+ desc "reset the redis database"
39
+ task :reset_redis do
40
+ raise "I'm not going to let you reset your production redis db, sorry about that." if FLAPJACK_ENV.downcase == "production"
41
+ puts "db size before: #{redis.dbsize}"
42
+ redis.flushdb
43
+ puts "db size after: #{redis.dbsize}"
44
+ end
45
+
46
+ desc "starts flapjack"
47
+ task :run_flapjack do
48
+ puts "Discovering path to perftools"
49
+ perftools = `gem which perftools | tail -1`
50
+ if system("if [ ! -d 'artifacts' ] ; then mkdir artifacts ; fi")
51
+ puts "we now have an artifacts dir"
52
+ else
53
+ raise "Problem creating artifacts: #{$?}"
54
+ end
55
+ time_flapjack_start = Time.now.to_f
56
+ puts "Starting flapjack..."
57
+ if system({"FLAPJACK_ENV" => FLAPJACK_ENV,
58
+ "CPUPROFILE" => "artifacts/flapjack-perftools-cpuprofile",
59
+ "RUBYOPT" => "-r#{perftools}"},
60
+ "bin/flapjack start --no-daemonize --config tasks/support/flapjack_config_benchmark.yaml")
61
+ puts "Flapjack run completed successfully"
62
+ else
63
+ raise "Problem starting flapjack: #{$?}"
64
+ end
65
+ @timer_flapjack = Time.now.to_f - time_flapjack_start
66
+ end
67
+
68
+ desc "generates perftools reports"
69
+ task :reports do
70
+ @benchmark_data = { 'events_created' => @events_created,
71
+ 'flapjack_runtime' => @timer_flapjack,
72
+ 'processing_rate' => @events_created.to_f / @timer_flapjack }.merge(@benchmark_parameters)
73
+ bytes_written = IO.write('artifacts/benchmark_data.json', Oj.dump(@benchmark_data, :indent => 2))
74
+ puts "benchmark data written to artifacts/benchmark_data.json (#{bytes_written} bytes)"
75
+
76
+ if system("pprof.rb --text artifacts/flapjack-perftools-cpuprofile > artifacts/flapjack-perftools-cpuprofile.txt")
77
+ puts "Generated perftools.rb text report at artifacts/flapjack-perftools-cpuprofile.txt"
78
+ system("head -24 artifacts/flapjack-perftools-cpuprofile.txt")
79
+ else
80
+ raise "Problem generating perftools.rb text report: #{$?}"
81
+ end
82
+ if system("pprof.rb --pdf artifacts/flapjack-perftools-cpuprofile > artifacts/flapjack-perftools-cpuprofile.pdf")
83
+ puts "Generated perftools.rb pdf report at artifacts/flapjack-perftools-cpuprofile.pdf"
84
+ else
85
+ raise "Problem generating perftools.rb pdf report: #{$?}"
86
+ end
87
+ end
88
+
89
+
90
+ desc "run benchmark - simulate a stream of events from the check execution system"
91
+ # Assumptions:
92
+ # - time to failure varies evenly between 1 hour and 1 month
93
+ # - time to recovery varies evenly between 10 seconds and 1 week
94
+ task :benchmark do
95
+
96
+ num_checks_per_entity = (ENV['CHECKS_PER_ENTITY'] || 5).to_i
97
+ num_entities = (ENV['ENTITIES'] || 100).to_i
98
+ interval = (ENV['INTERVAL'] || 60).to_i
99
+ hours = (ENV['HOURS'] || 1).to_f
100
+ seed = (ENV['SEED'] || 42).to_i
101
+
102
+ puts "Behaviour can be modified by setting any combination of the following environment variables: "
103
+ puts "CHECKS_PER_ENTITY - #{num_checks_per_entity}"
104
+ puts "ENTITIES - #{num_entities}"
105
+ puts "INTERVAL - #{interval}"
106
+ puts "HOURS - #{hours}"
107
+ puts "SEED - #{seed}"
108
+ puts "FLAPJACK_ENV - #{FLAPJACK_ENV}"
109
+
110
+ raise "INTERVAL must be less than (or equal to) 3600 seconds (1 hour)" unless interval <= 3600
111
+
112
+ cycles_per_hour = (60.0 * 60) / interval
113
+ cycles_per_day = (60.0 * 60 * 24) / interval
114
+ cycles_per_week = (60.0 * 60 * 24 * 7) / interval
115
+ cycles_per_month = (60.0 * 60 * 24 * 7 * 30) / interval
116
+ cycles = (hours * cycles_per_hour).to_i
117
+ failure_prob_min = 1.0 / cycles_per_month
118
+ failure_prob_max = 1.0 / cycles_per_hour
119
+ recovery_prob_min = 1.0 / cycles_per_week
120
+ recovery_prob_max = 1.0
121
+ initial_ok_prob = 1
122
+ num_checks = num_checks_per_entity * num_entities
123
+
124
+ prng = Random.new(seed)
125
+
126
+ ok = 0
127
+ critical = 0
128
+ check_id = 1
129
+ entities = (1..num_entities).to_a.inject({}) {|memo, id|
130
+ checks = (1..num_checks_per_entity).to_a.inject({}) {|memo_check, id_check|
131
+ memo_check[check_id] = {:name => "Check Type #{id_check}",
132
+ :state => ( prng.rand < initial_ok_prob ? 'OK' : 'CRITICAL' ),
133
+ :p_failure => prng.rand(failure_prob_min..failure_prob_max),
134
+ :p_recovery => prng.rand(recovery_prob_min..recovery_prob_max)}
135
+ ok += 1 if memo_check[check_id][:state] == 'OK'
136
+ critical += 1 if memo_check[check_id][:state] == 'CRITICAL'
137
+ check_id += 1
138
+ memo_check
139
+ }
140
+ memo[id] = checks
141
+ memo
142
+ }
143
+ #puts "ok: #{ok * 100.0 / num_checks}% (#{ok}), critical: #{100.0 * critical / num_checks}% (#{critical})"
144
+
145
+ events_created = 0
146
+ ok_to_critical = 0
147
+ critical_to_ok = 0
148
+ ok_events = 0
149
+ critical_events = 0
150
+ state_changes = 0
151
+ (0..cycles).to_a.each {|i|
152
+ changes = 0
153
+ ok = 0
154
+ critical = 0
155
+ summary = "You tell me summer's here \nand the time is wrong \n"
156
+ summary << "You tell me winter's here \nAnd your days are getting long"
157
+ entities.each_pair {|entity_id, checks|
158
+ checks.each_pair {|check_id, check|
159
+ changed = false
160
+ previous_state = check[:state]
161
+ case previous_state
162
+ when "OK"
163
+ if prng.rand < check[:p_failure]
164
+ check[:state] = "CRITICAL"
165
+ changed = true
166
+ changes += 1
167
+ ok_to_critical += 1
168
+ end
169
+ when "CRITICAL"
170
+ if prng.rand < check[:p_recovery]
171
+ check[:state] = "OK"
172
+ changed = true
173
+ changes += 1
174
+ critical_to_ok += 1
175
+ end
176
+ end
177
+ ok += 1 if check[:state] == 'OK'
178
+ critical += 1 if check[:state] == 'CRITICAL'
179
+
180
+ Flapjack::Data::Event.add({'entity' => "entity_#{entity_id}.example.com",
181
+ 'check' => check[:name],
182
+ 'type' => 'service',
183
+ 'state' => check[:state],
184
+ 'summary' => summary }, :redis => redis)
185
+ events_created += 1
186
+ }
187
+ }
188
+ ok_events += ok
189
+ critical_events += critical
190
+ state_changes += changes
191
+
192
+ #puts "ok: #{100.0 * ok / num_checks}% (#{ok}), critical: #{100.0 * critical / num_checks}% (#{critical}), changed: #{100.0 * changes / num_checks}% (#{changes})"
193
+
194
+ }
195
+ puts "created #{events_created} events:"
196
+ puts " OK: #{ok_events} (#{ (100.0 * ok_events / events_created).round(1)}%)"
197
+ puts " CRITICAL: #{critical_events} (#{ (100.0 * critical_events / events_created).round(1)}%)"
198
+ puts "containing #{state_changes} state changes (#{ (100.0 * state_changes / events_created).round(1)}%):"
199
+ puts " OK -> CRITICAL: #{ok_to_critical} (#{ (100.0 * ok_to_critical / events_created).round(1)}%)"
200
+ puts " CRITICAL -> OK: #{critical_to_ok} (#{ (100.0 * critical_to_ok / events_created).round(1)}%)"
201
+
202
+ @events_created = events_created
203
+ @benchmark_parameters = { 'events_created' => events_created,
204
+ 'ok_to_critical' => ok_to_critical,
205
+ 'critical_to_ok' => critical_to_ok,
206
+ 'checks_per_entity' => num_checks_per_entity,
207
+ 'entities' => num_entities,
208
+ 'interval' => interval,
209
+ 'hours' => hours,
210
+ 'cycles' => cycles,
211
+ 'failure_prob_min' => failure_prob_min,
212
+ 'failure_prob_max' => failure_prob_max,
213
+ 'recovery_prob_min' => recovery_prob_min,
214
+ 'recovery_prob_max' => recovery_prob_max,
215
+ 'initial_ok_prob' => initial_ok_prob,
216
+ 'seed' => seed,
217
+ 'flapjack_env' => FLAPJACK_ENV,
218
+ 'version' => Flapjack::VERSION,
219
+ 'git_last_commit' => `git rev-parse HEAD`.chomp,
220
+ 'git_version' => `git describe --long --dirty --abbrev=10 --tags`.chomp,
221
+ 'git_branch' => `git status --porcelain -b | head -1 | cut -d ' ' -f 2`.chomp,
222
+ 'ruby_build' => `ruby --version`.chomp,
223
+ 'time' => Time.new.iso8601,
224
+ 'hostname' => `hostname -f`.chomp,
225
+ 'uname' => `uname -a`.chomp }
226
+ end
227
+
228
+ end