flapjack 1.6.0 → 2.0.0b1

data/lib/flapjack/gateways/web/views/self_stats.html.erb

@@ -38,41 +38,41 @@
     <table class="table table-bordered table-hover table-condensed">
       <tr>
         <td>Events queued:</td>
-        <td><%= h @events_queued %></td>
-      </tr>
-      <tr>
-        <td>Number of enabled entities:</td>
-        <td><%= h @count_current_entities %></td>
-      </tr>
-      <tr>
-        <td>Number of failing entities:</td>
-        <td><%= h @count_failing_entities %></td>
+        <td><%= h @metrics[:event_queue_length] %></td>
       </tr>
       <tr>
         <td>Number of enabled checks:</td>
-        <td><%= h @count_current_checks %></td>
+        <td><%= h @metrics[:check_counts][:enabled] %></td>
       </tr>
       <tr>
         <td>Number of failing checks:</td>
-        <td><%= h @count_failing_checks %></td>
+        <td><%= h @metrics[:check_counts][:failing] %></td>
       </tr>
       <tr>
-        <td>Events processed (all time)</td>
-        <td>
-          <ul>
-            <li>all: <%= h @event_counters['all'] %> events</li>
-            <li>ok: <%= h @event_counters['ok'] %> events</li>
-            <li>failure: <%= h @event_counters['failure'] %> events</li>
-            <li>action: <%= h @event_counters['action'] %> events</li>
-            <li>invalid: <%= h @event_counters['invalid'] %> events</li>
-          </ul>
-        </td>
+        <td>Number of disabled checks:</td>
+        <td><%= h @metrics[:check_counts][:all] - @metrics[:check_counts][:enabled] %></td>
+      </tr>
+      <tr>
+        <% if @global_stats.nil? %>
+          <td colspan="2">No statistics found.</td>
+        <% else %>
+          <td>Events processed (all time)</td>
+          <td>
+            <ul>
+              <li>all: <%= h @global_stats[:all_events] %> events</li>
+              <li>ok: <%= h @global_stats[:ok_events] %> events</li>
+              <li>failure: <%= h @global_stats[:failure_events] %> events</li>
+              <li>action: <%= h @global_stats[:action_events] %> events</li>
+              <li>invalid: <%= h @global_stats[:invalid_events] %> events</li>
+            </ul>
+          </td>
+        <% end %>
       </tr>
       <tr>
         <td>Check Freshness</td>
         <td>
           <ul>
-            <% @current_checks_ages.each_pair do |age, check_count| %>
+            <% @metrics[:check_freshness].each_pair do |age, check_count| %>
               <li>&gt;= <%= h age %>: <%= h check_count %></li>
             <% end %>
           </ul>
@@ -80,7 +80,7 @@
       </tr>
       <tr>
         <td>Total keys in redis</td>
-        <td><%= h @dbsize %></td>
+        <td><%= h @metrics[:total_keys] %></td>
       </tr>
       <tr>
         <td>Current time</td>
@@ -97,13 +97,12 @@
         <th>Uptime</th>
         <th>Events Processed</th>
       </tr>
-      <% @executive_instances.sort_by {|i, d| d['uptime']}.each do |ei| %>
+      <% @executive_instances.sort_by {|i, d| d[:uptime]}.each do |ei| %>
         <%
           instance_id, details = ei
           hostname, pid  = instance_id.split(':')
-          started        = details['uptime_string']
-          event_counters = details['event_counters']
-          event_rates    = details['event_rates']
+          started        = details[:uptime_string]
+          event_rates    = details[:event_rates]
          %>
         <tr>
           <td><%= h hostname %></td>
@@ -111,11 +110,11 @@
           <td><%= h started %></td>
           <td>
             <ul>
-              <li>all: <%= h event_counters['all'] %> (<%= h event_rates['all'] %> events/s)</li>
-              <li>ok: <%= h event_counters['ok'] %> (<%= h event_rates['ok'] %> events/s)</li>
-              <li>failure: <%= h event_counters['failure'] %> (<%= h event_rates['failure'] %> events/s)</li>
-              <li>action: <%= h event_counters['action'] %> (<%= h event_rates['action'] %> events/s)</li>
-              <li>invalid: <%= h event_counters['invalid'] %> (<%= h event_rates['invalid'] %> events/s)</li>
+              <li>all: <%= h details[:all_events] %> (<%= h event_rates[:all_events] %> events/s)</li>
+              <li>ok: <%= h details[:ok_events] %> (<%= h event_rates[:ok_events] %> events/s)</li>
+              <li>failure: <%= h details[:failure_events] %> (<%= h event_rates[:failure_events] %> events/s)</li>
+              <li>action: <%= h details[:action_events] %> (<%= h event_rates[:action_events] %> events/s)</li>
+              <li>invalid: <%= h details[:invalid_events] %> (<%= h event_rates[:invalid_events] %> events/s)</li>
             </ul>
           </td>
         </tr>
@@ -125,7 +124,7 @@
 </div>
 
 <p>
-  <a class="btn btn-success" href="<% @base_url %>self_stats.json">View as JSON</a>
+  <a class="btn btn-success" href="<%= @base_url %>self_stats.json">View as JSON</a>
   Learn how to
   <a href="http://flapjack.io/docs/1.0/development/Gathering-internal-statistics-with-collectd"> use these metrics</a>.
 </p>

data/lib/flapjack/gateways/web/views/tag.html.erb

@@ -0,0 +1,32 @@
+<% page_title "Tag: #{h @tag[:name]}" %>
+
+<div class="page-header">
+  <h2>Tag: <%= h @tag[:name] %></h2>
+</div>
+
+<% if @checks.nil? || @checks.empty? %>
+  <p>No checks</p>
+<% else %>
+  <table class="table table-bordered table-hover table-condensed">
+    <tr>
+      <th>Name</th>
+      <th>Failing?</th>
+    </tr>
+    <% @checks.each do |check| %>
+    <tr>
+      <td><a href="<%= @base_url %>checks/<%= check[:id] %>" title="check details"><%= h check[:name] %></a></td>
+      <td><%=
+case check[:failing]
+when true
+  'Y'
+when false
+  'N'
+else
+  '?'
+end %></a></td>
+    </tr>
+    <% end %>
+  </table>
+<% end %>
+<br>
+

data/lib/flapjack/gateways/web/views/tags.html.erb

@@ -0,0 +1,51 @@
+<% page_title 'Tags' %>
+
+<div class="page-header">
+  <h2>Tags</h2>
+</div>
+
+<div class="row">
+  <div class="col-md-8">
+
+    <form action="<%= @base_url %>tags" method="get" role="form" class="form-horizontal">
+
+      <div class="form-group">
+        <label class="col-sm-2 control-label" for="start_time">Name:</label>
+        <div class="col-sm-10">
+          <input type="text" name="name" class="form-control" size="20" maxlength="80" value="<%= h params[:name] %>">
+          <span class="help-block">
+            e.g. the exact name, or a /regexp/
+          </span>
+        </div>
+
+        <div class="form-group">
+          <div class="col-sm-2"></div>
+          <div class="col-sm-10">
+            <button type="submit" class="btn btn-success">Refresh</button>
+          </div>
+        </div>
+
+      </div>
+
+    </form><!-- form-horizontal-->
+
+  </div>
+</div>
+
+<% if !@tags || @tags.empty? %>
+  <p>No tags</p>
+<% else %>
+  <table class="table table-bordered table-hover table-condensed">
+    <tr>
+      <th>Name</th>
+    </tr>
+    <% @tags.each do |tag| %>
+    <tr>
+      <td><a href="<%= @base_url %>tags/<%= tag[:id] %>" title="tag details"><%= h tag[:name] %></a></td>
+    </tr>
+    <% end %>
+  </table>
+<% end %>
+<br>
+
+<%= erb '_pagination.html'.to_sym, :layout => false %>

data/lib/flapjack/logger.rb

@@ -6,6 +6,26 @@ require 'monitor'
 
 module Flapjack
 
+  class << self
+    # Thread and fiber-local
+
+    def configure_log(name, config = {})
+      Thread.current[:flapjack_logger_name]   = name
+      Thread.current[:flapjack_logger_config] = config
+    end
+
+    def logger=(l)
+      Thread.current[:flapjack_logger] = l
+    end
+
+    def logger
+      Thread.current[:flapjack_logger] ||= Flapjack::Logger.new(
+        Thread.current[:flapjack_logger_name]   || 'default',
+        Thread.current[:flapjack_logger_config] || {}
+      )
+    end
+  end
+
   class Logger
 
     LEVELS = [:debug, :info, :warn, :error, :fatal]
@@ -26,10 +46,20 @@ module Flapjack
 
       @formatter = proc do |severity, datetime, progname, msg|
         t = datetime.iso8601(6)
-        "#{t} [#{severity}] :: #{name} :: #{msg}\n"
+        "#{t} [#{severity}] :: #{@name} :: #{msg}\n"
+      end
+
+      output = if config[:file].nil? || config[:file].empty? ||
+        !File.exists?(File.dirname(config[:file])) ||
+        !File.writable?(File.dirname(config[:file])) ||
+        (File.exists?(config[:file]) && !File.writable?(config[:file]))
+
+        STDOUT
+      else
+        config[:file]
       end
 
-      @logger = ::Logger.new(STDOUT)
+      @logger = ::Logger.new(output)
       @logger.formatter = @formatter
 
       configure(config)
@@ -103,7 +133,7 @@ module Flapjack
     end
 
     LEVELS.each do |level|
-      define_method(level) {|progname, &block|
+      define_method(level) {|progname = nil, &block|
         add(::Logger.const_get(level.upcase), nil, progname, &block)
       }
     end
@@ -112,6 +142,7 @@ module Flapjack
       (LEVELS + [:configure, :close, :add]).include?(sym)
     end
 
+
     ['debug', 'info', 'warn', 'error', 'fatal'].each { |level|
       define_method("#{level}?") {
         @logger.send("#{level}?")

data/lib/flapjack/notifier.rb

@@ -2,23 +2,16 @@
 
 require 'active_support/time'
 
-require 'em-hiredis'
+require 'flapjack/exceptions'
+require 'flapjack/redis_proxy'
+require 'flapjack/record_queue'
+require 'flapjack/utility'
 
 require 'flapjack/data/alert'
-
+require 'flapjack/data/check'
 require 'flapjack/data/contact'
-require 'flapjack/data/entity_check'
-require 'flapjack/data/notification'
 require 'flapjack/data/event'
-require 'flapjack/redis_pool'
-require 'flapjack/utility'
-
-require 'flapjack/gateways/email'
-require 'flapjack/gateways/sms_messagenet'
-require 'flapjack/gateways/slack'
-require 'flapjack/gateways/sms_twilio'
-require 'flapjack/gateways/sms_nexmo'
-require 'flapjack/gateways/aws_sns'
+require 'flapjack/data/notification'
 
 module Flapjack
 
@@ -27,61 +20,45 @@ module Flapjack
     include Flapjack::Utility
 
     def initialize(opts = {})
-      @config = opts[:config]
-      @redis_config = opts[:redis_config] || {}
-      @logger = opts[:logger]
-      @redis = Flapjack::RedisPool.new(:config => @redis_config, :size => 2, :logger => @logger)
+      @lock = opts[:lock]
+      @config = opts[:config] || {}
 
-      @notifications_queue = @config['queue'] || 'notifications'
+      @queue = Flapjack::RecordQueue.new(@config['queue'] || 'notifications',
+                 Flapjack::Data::Notification)
 
       queue_configs = @config.find_all {|k, v| k =~ /_queue$/ }
-      @queues = Hash[queue_configs.map {|k, v| [k[/^(.*)_queue$/, 1], v] }]
+      @queues = Hash[queue_configs.map {|k, v|
+        [k[/^(.*)_queue$/, 1], Flapjack::RecordQueue.new(v, Flapjack::Data::Alert)]
+      }]
 
-      notify_logfile  = @config['notification_log_file'] || 'log/notify.log'
-      if not File.directory?(File.dirname(notify_logfile))
-        puts "Parent directory for log file '#{notify_logfile}' doesn't exist"
-        puts "Exiting!"
-        exit
-      end
-      @notifylog = ::Logger.new(notify_logfile)
-      @notifylog.formatter = proc do |severity, datetime, progname, msg|
-        "#{datetime.to_s} | #{msg}\n"
-      end
+      raise "No queues for media transports" if @queues.empty?
 
-      tz = nil
       tz_string = @config['default_contact_timezone'] || ENV['TZ'] || 'UTC'
-      begin
-        tz = ActiveSupport::TimeZone.new(tz_string)
-      rescue ArgumentError
-        logger.error("Invalid timezone string specified in default_contact_timezone or TZ (#{tz_string})")
-        exit_now!
+      tz = ActiveSupport::TimeZone[tz_string.untaint]
+      if tz.nil?
+        raise "Invalid timezone string specified in default_contact_timezone or TZ (#{tz_string})"
       end
       @default_contact_timezone = tz
     end
 
     def start
-      @logger.info("Booting main loop.")
-
-      until @should_quit
-        @logger.debug("Waiting for notification...")
-        notification = Flapjack::Data::Notification.next(@notifications_queue,
-                                                         :redis => @redis,
-                                                         :logger => @logger)
-        process_notification(notification) unless notification.nil?
-      end
+      begin
+        Zermelo.redis = Flapjack.redis
 
-      @logger.info("Exiting main loop.")
-    end
+        loop do
+          @lock.synchronize do
+            @queue.foreach {|notif| process_notification(notif) }
+          end
 
-    # this must use a separate connection to the main Executive one, as it's running
-    # from a different fiber while the main one is blocking.
-    def stop
-      @should_quit = true
+          @queue.wait
+        end
+      ensure
+        Flapjack.redis.quit
+      end
+    end
 
-      redis_uri = @redis_config[:path] ||
-        "redis://#{@redis_config[:host] || '127.0.0.1'}:#{@redis_config[:port] || '6379'}/#{@redis_config[:db] || '0'}"
-      shutdown_redis = EM::Hiredis.connect(redis_uri)
-      shutdown_redis.rpush(@notifications_queue, Flapjack.dump_json('type' => 'shutdown'))
+    def stop_type
+      :exception
     end
 
   private
@@ -90,76 +67,167 @@ module Flapjack
     # notification, updates the notification history in redis, generates the
     # notifications
     def process_notification(notification)
-      @logger.debug ("Processing notification: #{notification.inspect}")
-
-      timestamp    = Time.now
-      event_id     = notification.event_id
-      entity_check = Flapjack::Data::EntityCheck.for_event_id(event_id,
-                      :redis => @redis, :logger => @logger)
-      contacts     = entity_check.contacts
-
-      if contacts.empty?
-        @logger.debug("No contacts for #{event_id}")
-        @notifylog.info("#{event_id} | #{notification.type} | NO CONTACTS")
-        return
+      Flapjack.logger.debug { "Processing notification: #{notification.inspect}" }
+
+      check       = notification.check
+      check_name  = check.name
+
+      # TODO check whether time should come from something stored in the notification
+      alerts = alerts_for(notification, check, :time => Time.now)
+
+      if alerts.nil? || alerts.empty?
+        Flapjack.logger.info { "No alerts" }
+      else
+        Flapjack.logger.info { "Alerts: #{alerts.size}" }
+
+        alerts.each do |alert|
+          medium = alert.medium
+
+          Flapjack.logger.info {
+            "#{check_name} | #{medium.contact.id} | " \
+            "#{medium.transport} | #{medium.address}\n" \
+            "Enqueueing #{medium.transport} alert for " \
+            "#{check_name} to #{medium.address} " \
+            " rollup: #{alert.rollup || '-'}"
+          }
+
+          @queues[medium.transport].push(alert)
+        end
       end
 
-      messages = notification.messages(contacts,
-        :default_timezone => @default_contact_timezone, :logger => @logger)
-
-      notification_contents = notification.contents
-
-      messages.each do |message|
-        media_type = message.medium
-        address    = message.address
-        contents   = message.contents.merge(notification_contents)
-
-        if message.rollup
-          rollup_alerts = message.contact.alerting_checks_for_media(media_type).inject({}) do |memo, alert|
-            ec = Flapjack::Data::EntityCheck.for_event_id(alert, :redis => @redis)
-            last_change = ec.last_change
-            memo[alert] = {
-              'duration' => last_change ? (Time.now.to_i - last_change) : nil,
-              'state'    => ec.state,
-              'summary'  => ec.summary
-            }
-            memo
-          end
-          contents['rollup_alerts'] = rollup_alerts
-          contents['rollup_threshold'] = message.contact.rollup_threshold_for_media(media_type)
+      notification.destroy
+    end
+
+    def alerts_for(notification, check, opts = {})
+      time       = opts[:time]
+
+      Flapjack::Data::Medium.lock(Flapjack::Data::Check,
+                                  Flapjack::Data::ScheduledMaintenance,
+                                  Flapjack::Data::UnscheduledMaintenance,
+                                  Flapjack::Data::Rule,
+                                  Flapjack::Data::Alert,
+                                  Flapjack::Data::Tag,
+                                  Flapjack::Data::Notification,
+                                  Flapjack::Data::Contact,
+                                  Flapjack::Data::State) do
+
+        notification_state = notification.state
+
+        this_notification_ack = 'acknowledgement'.eql?(notification_state.action)
+        this_notification_ok = this_notification_ack ||
+          Flapjack::Data::Condition.healthy?(notification_state.condition)
+
+        # checks in sched/unsched maint will not be notified -- time should be taken
+        # from the processor's created notification, maint period check done there only
+        is_a_test = !(notification_state.action =~ /\Atest_notifications/).nil?
+
+        alerting_severity = is_a_test ? 'critical' : notification.severity
+
+        # clear alertable if OK, to get accurate rollup counts
+        if !is_a_test && !this_notification_ok
+          check.alertable = true
+          check.save
         end
 
-        @notifylog.info("#{event_id} | " +
-          "#{notification.type} | #{message.contact.id} | #{media_type} | #{address}")
+        media = check.alerting_media(:time => time, :severity => alerting_severity).all
+
+        Flapjack.logger.debug {
+          "Alerting media for check #{check.name}:\n" +
+            media.collect {|m| "#{m.transport} #{m.address}"}.join("\n")
+        }
 
-        if @queues[media_type.to_s].nil?
-          @logger.error("no queue for media type: #{media_type}")
-          return
+        # clear alertable if OK, to get accurate rollup counts
+        if !is_a_test && this_notification_ok
+          check.alertable = false
+          check.save
         end
 
-        @logger.info("Enqueueing #{media_type} alert for #{event_id} to #{address} type: #{notification.type} rollup: #{message.rollup || '-'}")
+        media.inject([]) do |memo, alerting_medium|
+          alert_rollup = nil
+          alerting_check_ids = []
+
+          unless is_a_test
+            rollup_count_needed = !(alerting_medium.rollup_threshold.nil? ||
+              (alerting_medium.rollup_threshold <= 0))
+
+            if rollup_count_needed
+              alerting_check_ids = alerting_medium.alerting_checks(:time => time).ids
+
+              Flapjack.logger.debug {
+                "Alerting checks for medium #{alerting_medium.id}:\n" +
+                  Flapjack::Data::Check.intersect(:id => alerting_check_ids).map(&:name).join("\n")
+              }
+
+            end
+
+            alert_rollup = if rollup_count_needed &&
+              (alerting_check_ids.size >= alerting_medium.rollup_threshold)
+
+              'problem'
+            else
+              'problem'.eql?(alerting_medium.last_rollup_type) ? 'recovery' : nil
+            end
 
-        contact = message.contact
+            last_state = alerting_medium.last_state
 
-        if notification.ok? || (notification.state == 'acknowledgement')
-          ['warning', 'critical', 'unknown'].each do |alert_state|
-            contact.update_sent_alert_keys(
-              :media => media_type,
-              :check => event_id,
-              :state => alert_state,
-              :delete => true)
+            Flapjack.logger.debug "last_state #{last_state.inspect}"
+
+            last_state_ok = last_state.nil? ? nil :
+              (Flapjack::Data::Condition.healthy?(last_state.condition) ||
+               'acknowledgement'.eql?(last_state.action))
+
+            interval_allows = last_state.nil? ||
+              ((last_state.created_at + (alerting_medium.interval || 0)) < notification_state.created_at)
+
+            Flapjack.logger.debug "  last_state_ok = #{last_state_ok}\n" \
+              "  interval_allows  = #{interval_allows}\n" \
+              "  alert_rollup , last_rollup_type = #{alert_rollup} , #{alerting_medium.last_rollup_type}\n" \
+              "  condition , last_notification_condition  = #{notification_state.condition} , #{last_state.nil? ? '-' : last_state.condition}\n" \
+              "  no_previous_notification  = #{last_state.nil?}\n"
+
+            next memo unless last_state.nil? ||
+              (!last_state_ok && this_notification_ok) ||
+              (alert_rollup != alerting_medium.last_rollup_type) ||
+              (this_notification_ack && !last_state_ok) ||
+              (last_state_ok && !this_notification_ok) ||
+              (notification_state.condition != last_state.condition) ||
+              interval_allows
           end
-        else
-          contact.update_sent_alert_keys(
-            :media => media_type,
-            :check => event_id,
-            :state => notification.state)
-        end
 
-        contents_tags = contents['tags']
-        contents['tags'] = contents_tags.is_a?(Set) ? contents_tags.to_a : contents_tags
+          alert = Flapjack::Data::Alert.new(:condition => notification_state.condition,
+            :action => notification_state.action,
+            :last_condition => (last_state.nil? ? nil : last_state.condition),
+            :last_action => (last_state.nil? ? nil : last_state.action),
+            :condition_duration => notification.condition_duration,
+            :acknowledgement_duration => notification.duration,
+            :rollup => alert_rollup)
+
+          unless alert_rollup.nil? || alerting_check_ids.empty?
+            alert.rollup_states = Flapjack::Data::Check.intersect(:id => alerting_check_ids).all.each_with_object({}) do |ch, m|
+              cond = ch.condition
+              m[cond] ||= []
+              m[cond] << ch.name
+            end
+          end
 
-        Flapjack::Data::Alert.add(@queues[media_type.to_s], contents, :redis => @redis)
+          unless alert.save
+            raise "Couldn't save alert: #{alert.errors.full_messages.inspect}"
+          end
+
+          alerting_medium.alerts << alert
+          check.alerts  << alert
+
+          Flapjack.logger.info "alerting for #{alerting_medium.transport}, #{alerting_medium.address}"
+
+          unless is_a_test
+            notification_state.latest_media << alerting_medium
+            alerting_medium.last_rollup_type = alert.rollup
+            alerting_medium.save
+          end
+
+          memo << alert
+          memo
+        end
       end
     end