flapjack 0.5.5 → 0.6.23

data/bin/install-flapjack-systemwide

@@ -44,14 +44,14 @@ system("cp -aiv #{etc_path}/* /etc")
 
 # set sequence number to 50 so beanstalkd has a chance to boot
 system("update-rc.d flapjack-workers defaults 50")
-system("update-rc.d flapjack-notifier defaults 50")
+system("update-rc.d flapjack-executive defaults 50")
 
 puts 
 puts "Setup complete!"
 puts 
 puts "You will want to customise:"
 puts "  * /etc/flapjack/recipients.yaml"
-puts "  * /etc/flapjack/flapjack-notifier.yaml"
+puts "  * /etc/flapjack/flapjack-executive.yaml"
 puts 
 puts ".examples of these files exist in /etc/flapjack/'
 

data/config.ru

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'bundler'
+Bundler.require(:default)
+
+$: << File.expand_path(File.join(File.dirname(__FILE__), 'lib'))
+require 'flapjack/web'
+
+use Flapjack::Web
+run Sinatra::Application

data/dist/etc/init.d/flapjack

@@ -0,0 +1,46 @@
+#!/bin/bash
+#
+# Copyright (c) 2009-2012 Lindsay Holmwood <lindsay@holmwood.id.au>
+#
+# flapjack
+# Boots flapjack (coordinator, flapjack-executive, notification workers, ...)
+#
+
+PATH=$PATH:/sbin:/bin:/usr/sbin:/usr/bin:/usr/local:/usr/local/sbin:/usr/local/bin:/usr/lib/flapjack/bin
+
+# load global rbenv environment if present
+if [ -x /etc/profile.d/rbenv.sh ] ; then
+  source /etc/profile.d/rbenv.sh
+fi
+
+# Default return value
+RETVAL=0
+
+export FLAPJACK_ENV="production"
+
+if [ ! $(which flapjack) ]; then
+  echo "Error: flapjack isn't in PATH."
+  echo "Refusing to do anything!"
+  exit 1
+fi
+
+# Evaluate command
+case "$1" in
+  start)
+    flapjack --config /etc/flapjack/flapjack-config.yaml --daemonize
+    RETVAL=$?
+  ;;
+  stop)
+    kill `cat /var/run/flapjack/flapjack.pid`
+    # FIXME: make this more robust, preferably use daemons foo
+    # or at least wait until the flapjack process has exited before exiting
+    RETVAL=$?
+  ;;
+  *)
+    echo "Usage: flapjack {start|stop}"
+    RETVAL=1
+  ;;
+esac
+
+exit $RETVAL
+

data/dist/etc/init.d/flapjack-nagios-receiver

@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# Copyright (c) 2009-2012 Lindsay Holmwood <lindsay@holmwood.id.au>
+#
+# flapjack-nagios-receiver
+# reads from a nagios perfdata named-pipe and submits each event to the events queue in redis
+#
+
+PATH=$PATH:/sbin:/bin:/usr/sbin:/usr/bin:/usr/local:/usr/local/sbin:/usr/local/bin:/usr/lib/flapjack/bin
+
+# load global rbenv environment if present
+if [ -x /etc/profile.d/rbenv.sh ] ; then
+  source /etc/profile.d/rbenv.sh
+fi
+
+# Default return value
+RETVAL=0
+
+export FLAPJACK_ENV="production"
+
+NAGIOS_PERFDATA_FIFO="/var/cache/nagios3/event_stream.fifo"
+
+if [ ! $(which flapjack-nagios-receiver-control) ]; then
+  echo "Error: flapjack-nagios-receiver-control isn't in PATH."
+  echo "Refusing to do anything!"
+  exit 1
+fi
+
+# Evaluate command
+
+#/bin/bash -c "source /etc/profile.d/rbenv.sh && rbenv shell 1.9.3-p125 && flapjack-nagios-receiver-control $1 -- --config /etc/flapjack/flapjack-config.yaml ${NAGIOS_PERFDATA_FIFO}"
+rbenv shell 1.9.3-p125 && flapjack-nagios-receiver-control $1 -- --config /etc/flapjack/flapjack-config.yaml ${NAGIOS_PERFDATA_FIFO}
+RETVAL=$?
+
+exit $RETVAL
+

data/doc/GLOSSARY.md

@@ -0,0 +1,19 @@
+
+As implemented for the reporting API functions, these are placeholder terms only and open for discussion.
+
+<dl>
+<dt>Entity</dt>
+<dd>TODO</dd>
+<dt>Check</dt>
+<dd>TODO</dd>
+
+<dt>Outage</dt>
+<dd>Period of time from when a check goes into the 'CRITICAL'
+state to when it comes out of that state again</dd>
+<dt>Scheduled maintenance</dt>
+<dd>Periods of time explicitly created by external actors to denote that maintenance is scheduled to occur at these times.</dd>
+<dt>Unscheduled maintenance</dt>
+<dd>Periods of time explicitly created by external actors to denote that maintenance is happening or has happened at these times</dd>
+<dt>Downtime</dt>
+<dd>Outages minus scheduled maintenances across any given time period</dd>
+</dl>

data/etc/flapjack_config.yaml.example

@@ -0,0 +1,90 @@
+---
+
+development:
+  pid_file: tmp/pids/flapjack.pid
+  log_file: log/flapjack.log
+  daemonize: no
+  redis:
+    host: 127.0.0.1
+    port: 6379
+    db: 13
+  executive:
+    enabled: yes
+    email_queue: email_notifications
+    sms_queue: sms_notifications
+    jabber_queue: jabber_notifications
+    notification_log_file: log/flapjack-notification.log
+  email_notifier:
+    enabled: yes
+    queue: email_notifications
+    smtp_config:
+      port: 2525
+      # address: "localhost"
+      # port: 25
+      # domain: 'localhost.localdomain'
+      # user_name: nil
+      # password: nil
+      # authentication: nil
+      # enable_starttls_auto: true
+  sms_notifier:
+    enabled: yes
+    queue: sms_notifications
+    username: "ermahgerd"
+    password: "xxxx"
+  jabber_gateway:
+    enabled: yes
+    queue: jabber_notifications
+    server: "jabber.domain.tld"
+    port: 5222
+    jabberid: "flapjack@jabber.domain.tld"
+    password: "good-password"
+    alias: "flapjack"
+    rooms:
+      - "gimp@conference.jabber.domain.tld"
+      - "log@conference.jabber.domain.tld"
+  pagerduty_gateway:
+    enabled: yes
+    queue: pagerduty_notifications
+  web:
+    enabled: yes
+    port: 5080
+  api:
+    enabled: yes
+    port: 5081
+
+test:
+  redis:
+    database: 14
+  email_notifier:
+    instances: 1
+  sms_notifier:
+    instances: 1
+    username: "ermahgerd"
+    password: "xxxx"
+  web:
+    instances: 1
+
+
+staging:
+  redis:
+    database: 15
+  email_notifier:
+    instances: 1
+  sms_notifier:
+    instances: 1
+    username: "ermahgerd"
+    password: "xxxx"
+  web:
+    instances: 1
+
+production:
+  redis:
+    database: 12
+  email_notifier:
+    instances: 1
+  sms_notifier:
+    instances: 1
+    username: "ermahgerd"
+    password: "xxxx"
+  web:
+    instances: 1

data/features/events.feature

@@ -0,0 +1,132 @@
+@events
+Feature: events
+  So people can be notified when things break and recover
+  flapjack-executive must process events correctly
+
+  # TODO make entity and check implicit, so the test reads more cleanly
+  Background:
+    Given an entity 'def' exists
+
+  Scenario: Check ok to ok
+    Given check 'abc' for entity 'def' is in an ok state
+    When  an ok event is received for check 'abc' on entity 'def'
+    Then  a notification should not be generated for check 'abc' on entity 'def'
+    # And   show me the output
+
+  Scenario: Check ok to failed
+    Given check 'abc' for entity 'def' is in an ok state
+    When  a failure event is received for check 'abc' on entity 'def'
+    Then  a notification should not be generated for check 'abc' on entity 'def'
+
+  @time
+  Scenario: Check failed to failed after 10 seconds
+    Given check 'abc' for entity 'def' is in an ok state
+    When  a failure event is received for check 'abc' on entity 'def'
+    And   10 seconds passes
+    And   a failure event is received for check 'abc' on entity 'def'
+    Then  a notification should not be generated for check 'abc' on entity 'def'
+
+  @time
+  Scenario: Check ok to failed for 1 minute
+    Given check 'abc' for entity 'def' is in an ok state
+    When  a failure event is received for check 'abc' on entity 'def'
+    And   1 minute passes
+    And   a failure event is received for check 'abc' on entity 'def'
+    Then  a notification should be generated for check 'abc' on entity 'def'
+
+  @time
+  Scenario: Check failed and alerted to failed for 1 minute
+    Given check 'abc' for entity 'def' is in an ok state
+    When  a failure event is received for check 'abc' on entity 'def'
+    And   1 minute passes
+    And   a failure event is received for check 'abc' on entity 'def'
+    And   show me the notifications
+    Then  a notification should be generated for check 'abc' on entity 'def'
+    When  1 minute passes
+    And   a failure event is received for check 'abc' on entity 'def'
+    And   show me the notifications
+    Then  a notification should not be generated for check 'abc' on entity 'def'
+
+  @time
+  Scenario: Check failed and alerted to failed for 6 minutes
+    Given check 'abc' for entity 'def' is in an ok state
+    When  a failure event is received for check 'abc' on entity 'def'
+    And   1 minute passes
+    And   a failure event is received for check 'abc' on entity 'def'
+    Then  a notification should be generated for check 'abc' on entity 'def'
+    When  6 minutes passes
+    And   a failure event is received for check 'abc' on entity 'def'
+    Then  a notification should be generated for check 'abc' on entity 'def'
+
+  @time
+  Scenario: Check ok to failed for 1 minute when in scheduled maintenance
+    Given check 'abc' for entity 'def' is in an ok state
+    And   check 'abc' for entity 'def' is in scheduled maintenance
+    When  a failure event is received for check 'abc' on entity 'def'
+    And   1 minute passes
+    And   a failure event is received for check 'abc' on entity 'def'
+    Then  a notification should not be generated for check 'abc' on entity 'def'
+
+  @time
+  Scenario: Check ok to failed for 1 minute when in unscheduled maintenance
+    Given check 'abc' for entity 'def' is in an ok state
+    And   check 'abc' for entity 'def' is in unscheduled maintenance
+    When  a failure event is received for check 'abc' on entity 'def'
+    And   1 minute passes
+    And   a failure event is received for check 'abc' on entity 'def'
+    Then  a notification should not be generated for check 'abc' on entity 'def'
+
+  @time
+  Scenario: Check ok to failed for 1 minute, acknowledged, and failed for 6 minutes
+    Given check 'abc' for entity 'def' is in an ok state
+    When  a failure event is received for check 'abc' on entity 'def'
+    And   1 minute passes
+    And   a failure event is received for check 'abc' on entity 'def'
+    Then  a notification should be generated for check 'abc' on entity 'def'
+    When  an acknowledgement is received for check 'abc' on entity 'def'
+    And   6 minute passes
+    And   a failure event is received for check 'abc' on entity 'def'
+    Then  a notification should not be generated for check 'abc' on entity 'def'
+
+  Scenario: Check failed to ok
+    Given check 'abc' for entity 'def' is in a failure state
+    And   5 minutes passes
+    And   a failure event is received for check 'abc' on entity 'def'
+    Then  a notification should be generated for check 'abc' on entity 'def'
+    When  5 minutes passes
+    And   an ok event is received for check 'abc' on entity 'def'
+    Then  a notification should be generated for check 'abc' on entity 'def'
+
+  @time
+  Scenario: Check failed to ok when acknowledged
+    Given check 'abc' for entity 'def' is in a failure state
+    When  an acknowledgement event is received for check 'abc' on entity 'def'
+    Then  a notification should be generated for check 'abc' on entity 'def'
+    When  1 minute passes
+    And   an ok event is received for check 'abc' on entity 'def'
+    Then  a notification should be generated for check 'abc' on entity 'def'
+
+  @time
+  Scenario: Check failed to ok when acknowledged, and fails after 6 minutes
+    Given check 'abc' for entity 'def' is in a failure state
+    When  an acknowledgement event is received for check 'abc' on entity 'def'
+    Then  a notification should be generated for check 'abc' on entity 'def'
+    When  1 minute passes
+    And   an ok event is received for check 'abc' on entity 'def'
+    Then  a notification should be generated for check 'abc' on entity 'def'
+    When  6 minutes passes
+    And   a failure event is received for check 'abc' on entity 'def'
+    Then  a notification should not be generated for check 'abc' on entity 'def'
+    When  6 minutes passes
+    And   a failure event is received for check 'abc' on entity 'def'
+    Then  a notification should be generated for check 'abc' on entity 'def'
+
+  Scenario: Acknowledgement when ok
+    Given check 'abc' for entity 'def' is in an ok state
+    When  an acknowledgement event is received for check 'abc' on entity 'def'
+    Then  a notification should not be generated for check 'abc' on entity 'def'
+
+  Scenario: Acknowledgement when failed
+    Given check 'abc' for entity 'def' is in a failure state
+    When  an acknowledgement event is received for check 'abc' on entity 'def'
+    Then  a notification should be generated for check 'abc' on entity 'def'

data/features/notifications.feature

@@ -0,0 +1,57 @@
+@notifications
+Feature: notifications
+  So people can be notified when things break and recover
+  flapjack-notifier must send notifications correctly
+
+  # TODO test across multiple contacts
+
+  @resque
+  Scenario: Queue an SMS notification
+    Given the user wants to receive SMS notifications for entity 'example.com'
+    When an event notification is generated for entity 'example.com'
+    Then an SMS notification for entity 'example.com' should be queued for the user
+    And an email notification for entity 'example.com' should not be queued for the user
+
+  @resque
+  Scenario: Queue an email notification
+    Given the user wants to receive email notifications for entity 'example.com'
+    When an event notification is generated for entity 'example.com'
+    Then an email notification for entity 'example.com' should be queued for the user
+    And an SMS notification for entity 'example.com' should not be queued for the user
+
+  @resque
+  Scenario: Queue SMS and email notifications
+    Given the user wants to receive SMS notifications for entity 'example.com' and email notifications for entity 'example2.com'
+    When an event notification is generated for entity 'example.com'
+    And an event notification is generated for entity 'example2.com'
+    Then an SMS notification for entity 'example.com' should be queued for the user
+    And an SMS notification for entity 'example2.com' should not be queued for the user
+    Then an email notification for entity 'example.com' should not be queued for the user
+    And an email notification for entity 'example2.com' should be queued for the user
+
+  # NB: Scenarios below here are those that cover code run by the Resque workers
+  # We could maybe test resque integration as well, see
+  #   http://corner.squareup.com/2010/08/cucumber-and-resque.html
+  #   http://gist.github.com/532100
+
+  Scenario: Send a queued SMS notification
+    Given a user SMS notification has been queued for entity 'example.com'
+    When the SMS notification handler runs successfully
+    Then the user should receive an SMS notification
+
+  Scenario: Handle a failure to send a queued SMS notification
+    Given a user SMS notification has been queued for entity 'example.com'
+    When the SMS notification handler fails to send an SMS
+    Then the user should not receive an SMS notification
+
+  @email
+  Scenario: Send a queued email notification
+    Given a user email notification has been queued for entity 'example.com'
+    When the email notification handler runs successfully
+    Then the user should receive an email notification
+
+  @email
+  Scenario: Handle a failure to send a queued email notification
+    Given a user email notification has been queued for entity 'example.com'
+    When the email notification handler fails to send an email
+    Then the user should not receive an email notification

data/features/packaging-lintian.feature

@@ -1,15 +1,17 @@
-Feature: Packagability 
+Feature: Packagability
   To make Flapjack usable to the masses
   It must be easily packagable
 
   Scenario: No rubygems references
     Given I am at the project root
     When I run "grep require lib/* bin/* -R |grep rubygems"
-    Then I should see 0 lines of output
+    Then the exit status should be 1
+    And I should see 0 lines of output
 
   Scenario: A shebang that works everywhere
     Given I am at the project root
     When I run "find lib/ -type 'f' -name '*.rb'"
-    Then every file in the output should start with "#!/usr/bin/env ruby"
+    Then the exit status should be 0
+    And every file in the output should start with "#!/usr/bin/env ruby"
 
 

data/features/steps/events_steps.rb

@@ -0,0 +1,164 @@
+#!/usr/bin/env ruby
+
+require 'flapjack/data/entity_check'
+require 'flapjack/data/event'
+
+def drain_events
+  loop do
+    event = Flapjack::Data::Event.next(:block => false, :persistence => @redis)
+    break unless event
+    @app.send(:process_event, event)
+  end
+end
+
+def submit_event(event)
+  @redis.rpush 'events', event.to_json
+end
+
+def set_scheduled_maintenance(entity, check, duration = 60*60*2)
+  entity_check = Flapjack::Data::EntityCheck.for_entity_name(entity, check, :redis => @redis)
+  t = Time.now.to_i
+  entity_check.create_scheduled_maintenance(:start_time => t, :duration => duration, :summary => "upgrading everything")
+  @redis.setex("#{entity}:#{check}:scheduled_maintenance", duration, t)
+end
+
+def remove_scheduled_maintenance(entity, check)
+  entity_check = Flapjack::Data::EntityCheck.for_entity_name(entity, check, :redis => @redis)
+  sm = entity_check.maintenances(nil, nil, :scheduled => true)
+  sm.each do |m|
+    entity_check.delete_scheduled_maintenance(:start_time => m[:start_time])
+  end
+end
+
+def remove_unscheduled_maintenance(entity, check)
+  # end any unscheduled downtime
+  event_id = entity + ":" + check
+  if (um_start = @redis.get("#{event_id}:unscheduled_maintenance"))
+    @redis.del("#{event_id}:unscheduled_maintenance")
+    duration = Time.now.to_i - um_start.to_i
+    @redis.zadd("#{event_id}:unscheduled_maintenances", duration, um_start)
+  end
+end
+
+def remove_notifications(entity, check)
+  event_id = entity + ":" + check
+  @redis.del("#{event_id}:last_problem_notification")
+  @redis.del("#{event_id}:last_recovery_notification")
+  @redis.del("#{event_id}:last_acknowledgement_notification")
+end
+
+def set_ok_state(entity, check)
+  entity_check = Flapjack::Data::EntityCheck.for_entity_name(entity, check, :redis => @redis)
+  entity_check.update_state(Flapjack::Data::EntityCheck::STATE_OK,
+    :timestamp => (Time.now.to_i - (60*60*24)))
+end
+
+def set_failure_state(entity, check)
+  entity_check = Flapjack::Data::EntityCheck.for_entity_name(entity, check, :redis => @redis)
+  entity_check.update_state(Flapjack::Data::EntityCheck::STATE_CRITICAL,
+    :timestamp => (Time.now.to_i - (60*60*24)))
+end
+
+def submit_ok(entity, check)
+  event = {
+    'type'    => 'service',
+    'state'   => 'ok',
+    'summary' => '0% packet loss',
+    'entity'  => entity,
+    'check'   => check,
+    'client'  => 'clientx'
+  }
+  submit_event(event)
+end
+
+def submit_critical(entity, check)
+  event = {
+    'type'    => 'service',
+    'state'   => 'critical',
+    'summary' => '100% packet loss',
+    'entity'  => entity,
+    'check'   => check,
+    'client'  => 'clientx'
+  }
+  submit_event(event)
+end
+
+def submit_acknowledgement(entity, check)
+  event = {
+    'type'               => 'action',
+    'state'              => 'acknowledgement',
+    'summary'            => "I'll have this fixed in a jiffy, saw the same thing yesterday",
+    'entity'             => entity,
+    'check'              => check,
+    'client'             => 'clientx',
+    # 'acknowledgement_id' =>
+  }
+  submit_event(event)
+end
+
+Given /^an entity '([\w\.\-]+)' exists$/ do |entity|
+  Flapjack::Data::Entity.add({'id'       => '5000',
+                              'name'     => entity},
+                             :redis => @redis )
+end
+
+Given /^^check '([\w\.\-]+)' for entity '([\w\.\-]+)' is in an ok state$/ do |check, entity|
+  remove_unscheduled_maintenance(entity, check)
+  remove_scheduled_maintenance(entity, check)
+  remove_notifications(entity, check)
+  set_ok_state(entity, check)
+end
+
+Given /^check '([\w\.\-]+)' for entity '([\w\.\-]+)' is in a failure state$/ do |check, entity|
+  remove_unscheduled_maintenance(entity, check)
+  remove_scheduled_maintenance(entity, check)
+  remove_notifications(entity, check)
+  set_failure_state(entity, check)
+end
+
+Given /^check '([\w\.\-]+)' for entity '([\w\.\-]+)' is in scheduled maintenance$/ do |check, entity|
+  remove_unscheduled_maintenance(entity, check)
+  set_scheduled_maintenance(entity, check)
+end
+
+# TODO set the state directly rather than submit & drain
+Given /^check '([\w\.\-]+)' for entity '([\w\.\-]+)' is in unscheduled maintenance$/ do |check, entity|
+  remove_scheduled_maintenance(entity, check)
+  set_failure_state(entity, check)
+  submit_acknowledgement(entity, check)
+  drain_events  # TODO these should only be in When clauses
+end
+
+When /^an ok event is received for check '([\w\.\-]+)' on entity '([\w\.\-]+)'$/ do |check, entity|
+  submit_ok(entity, check)
+  drain_events
+end
+
+When /^a failure event is received for check '([\w\.\-]+)' on entity '([\w\.\-]+)'$/ do |check, entity|
+  submit_critical(entity, check)
+  drain_events
+end
+
+When /^an acknowledgement .*is received for check '([\w\.\-]+)' on entity '([\w\.\-]+)'$/ do |check, entity|
+  submit_acknowledgement(entity, check)
+  drain_events
+end
+
+
+# TODO logging is a side-effect, should test for notification generation itself
+Then /^a notification should not be generated for check '([\w\.\-]+)' on entity '([\w\.\-]+)'$/ do |check, entity|
+  message = @app.logger.messages.find_all {|m| m =~ /ending notifications for event #{entity}:#{check}/ }.last
+  message ? happy = message.match(/Not sending notifications/) : happy = false
+  happy.should be_true
+end
+
+Then /^a notification should be generated for check '([\w\.\-]+)' on entity '([\w\.\-]+)'$/ do |check, entity|
+  message = @app.logger.messages.find_all {|m| m =~ /ending notifications for event #{entity}:#{check}/ }.last
+  message ? happy = message.match(/Sending notifications/) : happy = false
+  happy.should be_true
+end
+
+Then /^show me the notifications?$/ do
+  puts @app.logger.messages.join("\n")
+end
+