flapjack 0.7.20 → 0.7.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. data/.gitignore +1 -0
  2. data/.travis.yml +3 -1
  3. data/CHANGELOG.md +10 -0
  4. data/Gemfile +1 -0
  5. data/bin/flapjack +11 -0
  6. data/bin/simulate-failed-check +5 -5
  7. data/features/notification_rules.feature +77 -19
  8. data/features/steps/events_steps.rb +15 -3
  9. data/lib/flapjack/coordinator.rb +3 -3
  10. data/lib/flapjack/data/contact.rb +1 -1
  11. data/lib/flapjack/data/entity.rb +12 -1
  12. data/lib/flapjack/data/entity_check.rb +9 -2
  13. data/lib/flapjack/data/event.rb +4 -4
  14. data/lib/flapjack/data/notification.rb +27 -20
  15. data/lib/flapjack/data/notification_rule.rb +26 -24
  16. data/lib/flapjack/data/tag.rb +5 -0
  17. data/lib/flapjack/gateways/api.rb +1 -1
  18. data/lib/flapjack/gateways/api/contact_methods.rb +3 -3
  19. data/lib/flapjack/gateways/email.rb +73 -46
  20. data/lib/flapjack/gateways/email/alert.html.erb +13 -4
  21. data/lib/flapjack/gateways/email/alert.text.erb +2 -2
  22. data/lib/flapjack/gateways/jabber.rb +22 -16
  23. data/lib/flapjack/gateways/pagerduty.rb +7 -3
  24. data/lib/flapjack/gateways/web.rb +1 -1
  25. data/lib/flapjack/gateways/web/views/check.html.erb +2 -2
  26. data/lib/flapjack/gateways/web/views/contact.html.erb +3 -3
  27. data/lib/flapjack/logger.rb +67 -35
  28. data/lib/flapjack/notifier.rb +9 -3
  29. data/lib/flapjack/pikelet.rb +3 -1
  30. data/lib/flapjack/processor.rb +34 -10
  31. data/lib/flapjack/version.rb +1 -1
  32. data/spec/lib/flapjack/coordinator_spec.rb +17 -13
  33. data/spec/lib/flapjack/data/contact_spec.rb +4 -3
  34. data/spec/lib/flapjack/data/entity_check_spec.rb +10 -0
  35. data/spec/lib/flapjack/data/entity_spec.rb +60 -5
  36. data/spec/lib/flapjack/data/event_spec.rb +4 -4
  37. data/spec/lib/flapjack/data/notification_rule_spec.rb +9 -2
  38. data/spec/lib/flapjack/data/tag_spec.rb +0 -1
  39. data/spec/lib/flapjack/gateways/api/contact_methods_spec.rb +1 -1
  40. data/spec/lib/flapjack/gateways/email_spec.rb +2 -1
  41. data/spec/lib/flapjack/gateways/jabber_spec.rb +5 -3
  42. data/spec/lib/flapjack/gateways/pagerduty_spec.rb +3 -1
  43. data/spec/lib/flapjack/logger_spec.rb +5 -5
  44. data/spec/lib/flapjack/pikelet_spec.rb +4 -2
  45. data/spec/lib/flapjack/processor_spec.rb +16 -7
  46. data/tasks/benchmarks.rake +228 -0
  47. data/tasks/events.rake +11 -10
  48. data/tasks/support/flapjack_config_benchmark.yaml +58 -0
  49. metadata +6 -4
@@ -6,11 +6,12 @@ require 'ice_cube'
6
6
  require 'flapjack/data/contact'
7
7
  require 'flapjack/data/entity_check'
8
8
  require 'flapjack/data/notification_rule'
9
+ require 'flapjack/data/tag_set'
9
10
 
10
11
  describe Flapjack::Data::Contact, :redis => true do
11
12
 
12
13
  let(:notification_rule_data) {
13
- {:entity_tags => ["database","physical"],
14
+ {:tags => ["database","physical"],
14
15
  :entities => ["foo-app-01.example.com"],
15
16
  :time_restrictions => [],
16
17
  :warning_media => ["email"],
@@ -22,7 +23,7 @@ describe Flapjack::Data::Contact, :redis => true do
22
23
 
23
24
  let(:general_notification_rule_data) {
24
25
  {:entities => [],
25
- :entity_tags => [],
26
+ :tags => Flapjack::Data::TagSet.new([]),
26
27
  :time_restrictions => [],
27
28
  :warning_media => ['email', 'sms', 'jabber', 'pagerduty'],
28
29
  :critical_media => ['email', 'sms', 'jabber', 'pagerduty'],
@@ -156,7 +157,7 @@ describe Flapjack::Data::Contact, :redis => true do
156
157
  rules = contact.notification_rules
157
158
  rules.should have(1).rule
158
159
  rule = rules.first
159
- [:entities, :entity_tags, :time_restrictions,
160
+ [:entities, :tags, :time_restrictions,
160
161
  :warning_media, :critical_media,
161
162
  :warning_blackhole, :critical_blackhole].each do |k|
162
163
  rule.send(k).should == general_notification_rule_data[k]
@@ -2,6 +2,7 @@ require 'spec_helper'
2
2
 
3
3
  require 'flapjack/data/entity'
4
4
  require 'flapjack/data/entity_check'
5
+ require 'flapjack/data/tag_set'
5
6
 
6
7
  describe Flapjack::Data::EntityCheck, :redis => true do
7
8
 
@@ -574,4 +575,13 @@ describe Flapjack::Data::EntityCheck, :redis => true do
574
575
  contacts.should have(1).contact
575
576
  contacts.first.name.should == 'John Johnson'
576
577
  end
578
+
579
+ it "generates ephemeral tags for itself" do
580
+ ec = Flapjack::Data::EntityCheck.for_entity_name('foo-app-01.example.com', 'Disk / Utilisation', :redis => @redis)
581
+ tags = ec.tags
582
+ tags.should_not be_nil
583
+ tags.should be_a(Flapjack::Data::TagSet)
584
+ ['foo-app-01', 'example.com', 'disk', '/', 'utilisation'].to_set.subset?(tags).should be_true
585
+ end
586
+
577
587
  end
@@ -176,16 +176,71 @@ describe Flapjack::Data::Entity, :redis => true do
176
176
  end
177
177
 
178
178
  it "finds entities by tag" do
179
- entity = Flapjack::Data::Entity.add({'id' => '5000',
179
+ entity0 = Flapjack::Data::Entity.add({'id' => '5000',
180
+ 'name' => 'abc-123',
181
+ 'contacts' => []},
182
+ :redis => @redis)
183
+
184
+ entity1 = Flapjack::Data::Entity.add({'id' => '5001',
185
+ 'name' => 'def-456',
186
+ 'contacts' => []},
187
+ :redis => @redis)
188
+
189
+ entity0.add_tags('source:foobar', 'abc')
190
+ entity1.add_tags('source:foobar', 'def')
191
+
192
+ entity0.should_not be_nil
193
+ entity0.should be_an(Flapjack::Data::Entity)
194
+ entity0.tags.should include("source:foobar")
195
+ entity0.tags.should include("abc")
196
+ entity0.tags.should_not include("def")
197
+ entity1.should_not be_nil
198
+ entity1.should be_an(Flapjack::Data::Entity)
199
+ entity1.tags.should include("source:foobar")
200
+ entity1.tags.should include("def")
201
+ entity1.tags.should_not include("abc")
202
+
203
+ entities = Flapjack::Data::Entity.find_all_with_tags(['abc'], :redis => @redis)
204
+ entities.should be_an(Array)
205
+ entities.should have(1).entity
206
+ entities.first.should == 'abc-123'
207
+
208
+ entities = Flapjack::Data::Entity.find_all_with_tags(['donkey'], :redis => @redis)
209
+ entities.should be_an(Array)
210
+ entities.should have(0).entities
211
+ end
212
+
213
+ it "finds entities with several tags" do
214
+ entity0 = Flapjack::Data::Entity.add({'id' => '5000',
180
215
  'name' => 'abc-123',
181
216
  'contacts' => []},
182
217
  :redis => @redis)
183
218
 
184
- entity.add_tags('source:foobar', 'foo')
219
+ entity1 = Flapjack::Data::Entity.add({'id' => '5001',
220
+ 'name' => 'def-456',
221
+ 'contacts' => []},
222
+ :redis => @redis)
185
223
 
186
- entity.should_not be_nil
187
- entity.should be_an(Flapjack::Data::Entity)
188
- # TODO - the rest of it
224
+ entity0.add_tags('source:foobar', 'abc')
225
+ entity1.add_tags('source:foobar', 'def')
226
+
227
+ entity0.should_not be_nil
228
+ entity0.should be_an(Flapjack::Data::Entity)
229
+ entity0.tags.should include("source:foobar")
230
+ entity0.tags.should include("abc")
231
+ entity1.should_not be_nil
232
+ entity1.should be_an(Flapjack::Data::Entity)
233
+ entity1.tags.should include("source:foobar")
234
+ entity1.tags.should include("def")
235
+
236
+ entities = Flapjack::Data::Entity.find_all_with_tags(['source:foobar'], :redis => @redis)
237
+ entities.should be_an(Array)
238
+ entities.should have(2).entity
239
+
240
+ entities = Flapjack::Data::Entity.find_all_with_tags(['source:foobar', 'def'], :redis => @redis)
241
+ entities.should be_an(Array)
242
+ entities.should have(1).entity
243
+ entities.first.should == 'def-456'
189
244
  end
190
245
 
191
246
  end
@@ -58,13 +58,13 @@ describe Flapjack::Data::Event do
58
58
  events_len = 23
59
59
  mock_redis.should_receive(:llen).with('events').and_return(events_len)
60
60
 
61
- pc = Flapjack::Data::Event.pending_count(:redis => mock_redis)
61
+ pc = Flapjack::Data::Event.pending_count('events', :redis => mock_redis)
62
62
  pc.should == events_len
63
63
  end
64
64
 
65
65
  it "creates a notification testing event" do
66
66
  Time.should_receive(:now).and_return(time)
67
- mock_redis.should_receive(:rpush).with('events', /"testing"/ )
67
+ mock_redis.should_receive(:lpush).with('events', /"testing"/ )
68
68
 
69
69
  Flapjack::Data::Event.test_notifications(entity_name, check,
70
70
  :summary => 'test', :details => 'testing', :redis => mock_redis)
@@ -72,7 +72,7 @@ describe Flapjack::Data::Event do
72
72
 
73
73
  it "creates an acknowledgement event" do
74
74
  Time.should_receive(:now).and_return(time)
75
- mock_redis.should_receive(:rpush).with('events', /"acking"/ )
75
+ mock_redis.should_receive(:lpush).with('events', /"acking"/ )
76
76
 
77
77
  Flapjack::Data::Event.create_acknowledgement(entity_name, check,
78
78
  :summary => 'acking', :time => time.to_i, :redis => mock_redis)
@@ -98,4 +98,4 @@ describe Flapjack::Data::Event do
98
98
  it { should be_a_failure }
99
99
  end
100
100
 
101
- end
101
+ end
@@ -14,7 +14,7 @@ describe Flapjack::Data::NotificationRule, :redis => true do
14
14
 
15
15
  let(:rule_data) {
16
16
  {:contact_id => '23',
17
- :entity_tags => ["database","physical"],
17
+ :tags => ["database","physical"],
18
18
  :entities => ["foo-app-01.example.com"],
19
19
  :time_restrictions => [ weekdays_8_18 ],
20
20
  :warning_media => ["email"],
@@ -76,7 +76,14 @@ describe Flapjack::Data::NotificationRule, :redis => true do
76
76
  rule.match_entity?('foo-app-02.example.com').should be_false
77
77
  end
78
78
 
79
- pending "check whether entity tags match"
79
+ it "checks whether entity tags match" do
80
+ rule = existing_rule
81
+
82
+ rule.match_tags?(['database', 'physical'].to_set).should be_true
83
+ rule.match_tags?(['database', 'physical', 'beetroot'].to_set).should be_true
84
+ rule.match_tags?(['database'].to_set).should be_false
85
+ rule.match_tags?(['virtual'].to_set).should be_false
86
+ end
80
87
 
81
88
  it "checks if blackhole settings for a rule match a severity level" do
82
89
  rule_data[:warning_blackhole] = true
@@ -1,5 +1,4 @@
1
1
  require 'spec_helper'
2
- require 'flapjack/data/entity'
3
2
 
4
3
  describe Flapjack::Data::Tag, :redis => true do
5
4
 
@@ -37,7 +37,7 @@ describe 'Flapjack::Gateways::API::ContactMethods', :sinatra => true, :logger =>
37
37
 
38
38
  let(:notification_rule_data) {
39
39
  {"contact_id" => "21",
40
- "entity_tags" => ["database","physical"],
40
+ "tags" => ["database","physical"],
41
41
  "entities" => ["foo-app-01.example.com"],
42
42
  "time_restrictions" => nil,
43
43
  "warning_media" => ["email"],
@@ -15,6 +15,7 @@ describe Flapjack::Gateways::Email, :logger => true do
15
15
  entity_check = mock(Flapjack::Data::EntityCheck)
16
16
  entity_check.should_receive(:in_scheduled_maintenance?).and_return(false)
17
17
  entity_check.should_receive(:in_unscheduled_maintenance?).and_return(false)
18
+ entity_check.should_receive(:last_change).and_return(Time.now.to_i)
18
19
 
19
20
  redis = mock('redis')
20
21
  ::Resque.should_receive(:redis).and_return(redis)
@@ -46,4 +47,4 @@ describe Flapjack::Gateways::Email, :logger => true do
46
47
  Flapjack::Gateways::Email.perform(notification)
47
48
  end
48
49
 
49
- end
50
+ end
@@ -151,9 +151,11 @@ describe Flapjack::Gateways::Jabber, :logger => true do
151
151
  end
152
152
 
153
153
  it "prompts the blocking redis connection to quit" do
154
- redis = mock('redis')
155
- redis.should_receive(:rpush).with('jabber_notifications', %q{{"notification_type":"shutdown"}})
154
+ shutdown_redis = mock('shutdown_redis')
155
+ shutdown_redis.should_receive(:rpush).with('jabber_notifications', %q{{"notification_type":"shutdown"}})
156
+ EM::Hiredis.should_receive(:connect).and_return(shutdown_redis)
156
157
 
158
+ redis = mock('redis')
157
159
  Flapjack::RedisPool.should_receive(:new).and_return(redis)
158
160
  fj = Flapjack::Gateways::Jabber.new(:config => config, :logger => @logger)
159
161
 
@@ -163,7 +165,7 @@ describe Flapjack::Gateways::Jabber, :logger => true do
163
165
  it "runs a blocking loop listening for notifications" do
164
166
  timer = mock('timer')
165
167
  timer.should_receive(:cancel)
166
- EM::Synchrony.should_receive(:add_periodic_timer).with(60).and_return(timer)
168
+ EM::Synchrony.should_receive(:add_periodic_timer).with(1).and_return(timer)
167
169
 
168
170
  redis = mock('redis')
169
171
 
@@ -11,7 +11,9 @@ describe Flapjack::Gateways::Pagerduty, :logger => true do
11
11
  let(:redis) { mock('redis') }
12
12
 
13
13
  it "prompts the blocking redis connection to quit" do
14
- redis.should_receive(:rpush).with(config['queue'], %q{{"notification_type":"shutdown"}})
14
+ shutdown_redis = mock('shutdown_redis')
15
+ shutdown_redis.should_receive(:rpush).with(config['queue'], %q{{"notification_type":"shutdown"}})
16
+ EM::Hiredis.should_receive(:connect).and_return(shutdown_redis)
15
17
 
16
18
  Flapjack::RedisPool.should_receive(:new).and_return(redis)
17
19
  fp = Flapjack::Gateways::Pagerduty.new(:config => config, :logger => @logger)
@@ -10,21 +10,21 @@ describe Flapjack::Logger do
10
10
  it "creates a logger logging to STDOUT and syslog" do
11
11
  logger.should_receive(:formatter=).with(an_instance_of(Proc))
12
12
  logger.should_receive(:level=).and_return(Logger::DEBUG)
13
- logger.should_receive(:warn).with("Yowza!")
13
+ logger.should_receive(:add).with(2, nil, "Yowza!")
14
14
  ::Logger.should_receive(:new).with(STDOUT).and_return(logger)
15
15
 
16
16
  if Syslog.const_defined?('Logger', false)
17
17
  sys_logger.should_receive(:formatter=).with(an_instance_of(Proc))
18
18
  sys_logger.should_receive(:level=).with(Logger::DEBUG)
19
- sys_logger.should_receive(:warn).with("Yowza!")
19
+ sys_logger.should_receive(:add).with(Logger::WARN, 'Yowza!', 'flapjack')
20
20
  Syslog.const_get('Logger', false).should_receive(:new).with('flapjack').and_return(sys_logger)
21
21
  else
22
- syslog.should_receive(:log).with(Syslog::Constants::LOG_WARNING, /\[WARN\] :: spec :: %s/, "Yowza!")
23
- Syslog.should_receive(:"opened?").and_return(false)
24
22
  Syslog.should_receive(:open).with('flapjack',
25
23
  (Syslog::Constants::LOG_PID | Syslog::Constants::LOG_CONS),
26
24
  Syslog::Constants::LOG_USER).and_return(syslog)
27
- Syslog.should_receive(:mask=).with(Syslog::LOG_UPTO(Syslog::Constants::LOG_DEBUG))
25
+ Syslog.should_receive(:mask=).with(Syslog::LOG_UPTO(Syslog::Constants::LOG_WARNING))
26
+ Syslog.should_receive(:log).with(Syslog::Constants::LOG_WARNING, /\[WARN\] :: spec :: %s/, "Yowza!")
27
+ Syslog.should_receive(:close)
28
28
  end
29
29
 
30
30
  flogger = Flapjack::Logger.new('spec', 'level' => 'debug')
@@ -21,17 +21,19 @@ describe Flapjack::Pikelet do
21
21
 
22
22
  config.should_receive(:[]).with('logger').and_return(nil)
23
23
 
24
+ fc = mock('coordinator')
25
+
24
26
  processor = mock('processor')
25
27
  processor.should_receive(:start)
26
28
  Flapjack::Processor.should_receive(:new).with(:config => config,
27
- :redis_config => redis_config, :boot_time => time, :logger => logger).
29
+ :redis_config => redis_config, :boot_time => time, :logger => logger, :coordinator => fc).
28
30
  and_return(processor)
29
31
 
30
32
  fiber.should_receive(:resume)
31
33
  Fiber.should_receive(:new).and_yield.and_return(fiber)
32
34
 
33
35
  pik = Flapjack::Pikelet.create('processor', :config => config,
34
- :redis_config => redis_config, :boot_time => time)
36
+ :redis_config => redis_config, :boot_time => time, :coordinator => fc)
35
37
  pik.should be_a(Flapjack::Pikelet::Generic)
36
38
  pik.start
37
39
  end
@@ -1,11 +1,14 @@
1
1
  require 'spec_helper'
2
2
  require 'flapjack/processor'
3
+ require 'flapjack/coordinator'
3
4
 
4
5
  describe Flapjack::Processor, :logger => true do
5
6
 
6
7
  # NB: this is only testing the public API of the Processor class, which is pretty limited.
7
8
  # (initialize, main, stop). Most test coverage for this class comes from the cucumber features.
8
9
 
10
+ let(:config) { mock(Flapjack::Configuration) }
11
+
9
12
  # TODO this does too much -- split it up
10
13
  it "starts up, runs and shuts down" do
11
14
  t = Time.now.to_i
@@ -37,20 +40,26 @@ describe Flapjack::Processor, :logger => true do
37
40
  # redis.should_receive(:hincrby).with('event_counters', 'all', 1)
38
41
  # redis.should_receive(:hincrby).with(/^event_counters:/, 'all', 1)
39
42
 
40
- Flapjack::Data::Event.should_receive(:pending_count).with(:redis => redis).and_return(0)
43
+ Flapjack::Data::Event.should_receive(:pending_count).with('events', :redis => redis).and_return(0)
41
44
 
42
45
  Flapjack::RedisPool.should_receive(:new).and_return(redis)
43
- executive = Flapjack::Processor.new(:config => {}, :logger => @logger)
44
46
 
45
- shutdown_evt = mock(Flapjack::Data::Event)
46
- shutdown_evt.should_receive(:inspect)
47
- shutdown_evt.should_receive(:type).and_return('shutdown')
47
+ fc = mock('coordinator')
48
+
49
+ executive = Flapjack::Processor.new(:config => {}, :logger => @logger, :coordinator => fc)
50
+
51
+ noop_evt = mock(Flapjack::Data::Event)
52
+ noop_evt.should_receive(:inspect)
53
+ noop_evt.should_receive(:type).and_return('noop')
48
54
  Flapjack::Data::Event.should_receive(:next) {
49
55
  executive.instance_variable_set('@should_quit', true)
50
- shutdown_evt
56
+ noop_evt
51
57
  }
52
58
 
53
- executive.start
59
+ begin
60
+ executive.start
61
+ rescue SystemExit
62
+ end
54
63
  end
55
64
 
56
65
  end
@@ -0,0 +1,228 @@
1
+ require 'redis'
2
+ require 'oj'
3
+ require 'time'
4
+
5
+ namespace :benchmarks do
6
+
7
+ # add lib to the default include path
8
+ unless $:.include?(File.dirname(__FILE__) + '/../lib/')
9
+ $: << File.dirname(__FILE__) + '/../lib'
10
+ end
11
+
12
+ require 'flapjack/configuration'
13
+ require 'flapjack/data/event'
14
+ require 'flapjack/data/entity_check'
15
+ require 'flapjack/version'
16
+
17
+ FLAPJACK_ENV = ENV['FLAPJACK_ENV'] || 'test'
18
+ config_file = File.join('tasks', 'support', 'flapjack_config_benchmark.yaml')
19
+
20
+ config = Flapjack::Configuration.new
21
+ config.load( config_file )
22
+
23
+ @config_env = config.all
24
+ @redis_config = config.for_redis
25
+
26
+ if @config_env.nil? || @config_env.empty?
27
+ puts "No config data for environment '#{FLAPJACK_ENV}' found in '#{config_file}'"
28
+ exit(false)
29
+ end
30
+
31
+ redis = Redis.new(@redis_config)
32
+
33
+ desc "nukes the redis db, generates the events, runs and shuts down flapjack, generates perftools reports"
34
+ task :run => [:reset_redis, :benchmark, :run_flapjack, :reports] do
35
+ puts Oj.dump(@benchmark_data, :indent => 2)
36
+ end
37
+
38
+ desc "reset the redis database"
39
+ task :reset_redis do
40
+ raise "I'm not going to let you reset your production redis db, sorry about that." if FLAPJACK_ENV.downcase == "production"
41
+ puts "db size before: #{redis.dbsize}"
42
+ redis.flushdb
43
+ puts "db size after: #{redis.dbsize}"
44
+ end
45
+
46
+ desc "starts flapjack"
47
+ task :run_flapjack do
48
+ puts "Discovering path to perftools"
49
+ perftools = `gem which perftools | tail -1`
50
+ if system("if [ ! -d 'artifacts' ] ; then mkdir artifacts ; fi")
51
+ puts "we now have an artifacts dir"
52
+ else
53
+ raise "Problem creating artifacts: #{$?}"
54
+ end
55
+ time_flapjack_start = Time.now.to_f
56
+ puts "Starting flapjack..."
57
+ if system({"FLAPJACK_ENV" => FLAPJACK_ENV,
58
+ "CPUPROFILE" => "artifacts/flapjack-perftools-cpuprofile",
59
+ "RUBYOPT" => "-r#{perftools}"},
60
+ "bin/flapjack start --no-daemonize --config tasks/support/flapjack_config_benchmark.yaml")
61
+ puts "Flapjack run completed successfully"
62
+ else
63
+ raise "Problem starting flapjack: #{$?}"
64
+ end
65
+ @timer_flapjack = Time.now.to_f - time_flapjack_start
66
+ end
67
+
68
+ desc "generates perftools reports"
69
+ task :reports do
70
+ @benchmark_data = { 'events_created' => @events_created,
71
+ 'flapjack_runtime' => @timer_flapjack,
72
+ 'processing_rate' => @events_created.to_f / @timer_flapjack }.merge(@benchmark_parameters)
73
+ bytes_written = IO.write('artifacts/benchmark_data.json', Oj.dump(@benchmark_data, :indent => 2))
74
+ puts "benchmark data written to artifacts/benchmark_data.json (#{bytes_written} bytes)"
75
+
76
+ if system("pprof.rb --text artifacts/flapjack-perftools-cpuprofile > artifacts/flapjack-perftools-cpuprofile.txt")
77
+ puts "Generated perftools.rb text report at artifacts/flapjack-perftools-cpuprofile.txt"
78
+ system("head -24 artifacts/flapjack-perftools-cpuprofile.txt")
79
+ else
80
+ raise "Problem generating perftools.rb text report: #{$?}"
81
+ end
82
+ if system("pprof.rb --pdf artifacts/flapjack-perftools-cpuprofile > artifacts/flapjack-perftools-cpuprofile.pdf")
83
+ puts "Generated perftools.rb pdf report at artifacts/flapjack-perftools-cpuprofile.pdf"
84
+ else
85
+ raise "Problem generating perftools.rb pdf report: #{$?}"
86
+ end
87
+ end
88
+
89
+
90
+ desc "run benchmark - simulate a stream of events from the check execution system"
91
+ # Assumptions:
92
+ # - time to failure varies evenly between 1 hour and 1 month
93
+ # - time to recovery varies evenly between 10 seconds and 1 week
94
+ task :benchmark do
95
+
96
+ num_checks_per_entity = (ENV['CHECKS_PER_ENTITY'] || 5).to_i
97
+ num_entities = (ENV['ENTITIES'] || 100).to_i
98
+ interval = (ENV['INTERVAL'] || 60).to_i
99
+ hours = (ENV['HOURS'] || 1).to_f
100
+ seed = (ENV['SEED'] || 42).to_i
101
+
102
+ puts "Behaviour can be modified by setting any combination of the following environment variables: "
103
+ puts "CHECKS_PER_ENTITY - #{num_checks_per_entity}"
104
+ puts "ENTITIES - #{num_entities}"
105
+ puts "INTERVAL - #{interval}"
106
+ puts "HOURS - #{hours}"
107
+ puts "SEED - #{seed}"
108
+ puts "FLAPJACK_ENV - #{FLAPJACK_ENV}"
109
+
110
+ raise "INTERVAL must be less than (or equal to) 3600 seconds (1 hour)" unless interval <= 3600
111
+
112
+ cycles_per_hour = (60.0 * 60) / interval
113
+ cycles_per_day = (60.0 * 60 * 24) / interval
114
+ cycles_per_week = (60.0 * 60 * 24 * 7) / interval
115
+ cycles_per_month = (60.0 * 60 * 24 * 7 * 30) / interval
116
+ cycles = (hours * cycles_per_hour).to_i
117
+ failure_prob_min = 1.0 / cycles_per_month
118
+ failure_prob_max = 1.0 / cycles_per_hour
119
+ recovery_prob_min = 1.0 / cycles_per_week
120
+ recovery_prob_max = 1.0
121
+ initial_ok_prob = 1
122
+ num_checks = num_checks_per_entity * num_entities
123
+
124
+ prng = Random.new(seed)
125
+
126
+ ok = 0
127
+ critical = 0
128
+ check_id = 1
129
+ entities = (1..num_entities).to_a.inject({}) {|memo, id|
130
+ checks = (1..num_checks_per_entity).to_a.inject({}) {|memo_check, id_check|
131
+ memo_check[check_id] = {:name => "Check Type #{id_check}",
132
+ :state => ( prng.rand < initial_ok_prob ? 'OK' : 'CRITICAL' ),
133
+ :p_failure => prng.rand(failure_prob_min..failure_prob_max),
134
+ :p_recovery => prng.rand(recovery_prob_min..recovery_prob_max)}
135
+ ok += 1 if memo_check[check_id][:state] == 'OK'
136
+ critical += 1 if memo_check[check_id][:state] == 'CRITICAL'
137
+ check_id += 1
138
+ memo_check
139
+ }
140
+ memo[id] = checks
141
+ memo
142
+ }
143
+ #puts "ok: #{ok * 100.0 / num_checks}% (#{ok}), critical: #{100.0 * critical / num_checks}% (#{critical})"
144
+
145
+ events_created = 0
146
+ ok_to_critical = 0
147
+ critical_to_ok = 0
148
+ ok_events = 0
149
+ critical_events = 0
150
+ state_changes = 0
151
+ (0..cycles).to_a.each {|i|
152
+ changes = 0
153
+ ok = 0
154
+ critical = 0
155
+ summary = "You tell me summer's here \nand the time is wrong \n"
156
+ summary << "You tell me winter's here \nAnd your days are getting long"
157
+ entities.each_pair {|entity_id, checks|
158
+ checks.each_pair {|check_id, check|
159
+ changed = false
160
+ previous_state = check[:state]
161
+ case previous_state
162
+ when "OK"
163
+ if prng.rand < check[:p_failure]
164
+ check[:state] = "CRITICAL"
165
+ changed = true
166
+ changes += 1
167
+ ok_to_critical += 1
168
+ end
169
+ when "CRITICAL"
170
+ if prng.rand < check[:p_recovery]
171
+ check[:state] = "OK"
172
+ changed = true
173
+ changes += 1
174
+ critical_to_ok += 1
175
+ end
176
+ end
177
+ ok += 1 if check[:state] == 'OK'
178
+ critical += 1 if check[:state] == 'CRITICAL'
179
+
180
+ Flapjack::Data::Event.add({'entity' => "entity_#{entity_id}.example.com",
181
+ 'check' => check[:name],
182
+ 'type' => 'service',
183
+ 'state' => check[:state],
184
+ 'summary' => summary }, :redis => redis)
185
+ events_created += 1
186
+ }
187
+ }
188
+ ok_events += ok
189
+ critical_events += critical
190
+ state_changes += changes
191
+
192
+ #puts "ok: #{100.0 * ok / num_checks}% (#{ok}), critical: #{100.0 * critical / num_checks}% (#{critical}), changed: #{100.0 * changes / num_checks}% (#{changes})"
193
+
194
+ }
195
+ puts "created #{events_created} events:"
196
+ puts " OK: #{ok_events} (#{ (100.0 * ok_events / events_created).round(1)}%)"
197
+ puts " CRITICAL: #{critical_events} (#{ (100.0 * critical_events / events_created).round(1)}%)"
198
+ puts "containing #{state_changes} state changes (#{ (100.0 * state_changes / events_created).round(1)}%):"
199
+ puts " OK -> CRITICAL: #{ok_to_critical} (#{ (100.0 * ok_to_critical / events_created).round(1)}%)"
200
+ puts " CRITICAL -> OK: #{critical_to_ok} (#{ (100.0 * critical_to_ok / events_created).round(1)}%)"
201
+
202
+ @events_created = events_created
203
+ @benchmark_parameters = { 'events_created' => events_created,
204
+ 'ok_to_critical' => ok_to_critical,
205
+ 'critical_to_ok' => critical_to_ok,
206
+ 'checks_per_entity' => num_checks_per_entity,
207
+ 'entities' => num_entities,
208
+ 'interval' => interval,
209
+ 'hours' => hours,
210
+ 'cycles' => cycles,
211
+ 'failure_prob_min' => failure_prob_min,
212
+ 'failure_prob_max' => failure_prob_max,
213
+ 'recovery_prob_min' => recovery_prob_min,
214
+ 'recovery_prob_max' => recovery_prob_max,
215
+ 'initial_ok_prob' => initial_ok_prob,
216
+ 'seed' => seed,
217
+ 'flapjack_env' => FLAPJACK_ENV,
218
+ 'version' => Flapjack::VERSION,
219
+ 'git_last_commit' => `git rev-parse HEAD`.chomp,
220
+ 'git_version' => `git describe --long --dirty --abbrev=10 --tags`.chomp,
221
+ 'git_branch' => `git status --porcelain -b | head -1 | cut -d ' ' -f 2`.chomp,
222
+ 'ruby_build' => `ruby --version`.chomp,
223
+ 'time' => Time.new.iso8601,
224
+ 'hostname' => `hostname -f`.chomp,
225
+ 'uname' => `uname -a`.chomp }
226
+ end
227
+
228
+ end