flapjack 0.5.5 → 0.6.23

data/lib/flapjack/patches.rb

@@ -1,6 +1,10 @@
+#!/usr/bin/env ruby
+
 require 'ostruct'
 require 'daemons'
-require 'log4r'
+require 'thin'
+require 'resque'
+# require 'log4r'
 
 class OpenStruct
   def to_h
@@ -9,33 +13,33 @@ class OpenStruct
 end
 
 module Daemons
-  class PidFile 
+  class PidFile
     # we override this method so creating pid files is fork-safe
-    def filename 
+    def filename
       File.join(@dir, "#{@progname}#{Process.pid}.pid")
     end
   end
 end
 
-module Log4r
-  class Logger
-    def error(args)
-      err(args)
-    end
-
-    def warning(args)
-      warn(args)
-    end
-  end
-end
+#module Log4r
+#  class Logger
+#    def error(args)
+#      err(args)
+#    end
+#
+#    def warning(args)
+#      warn(args)
+#    end
+#  end
+#end
 
-# extracted from Extlib. 
+# extracted from Extlib.
 # FIXME: what's the licensing here?
 class String
-  def camel_case                                                                                                                                    
-    return self if self !~ /_/ && self =~ /[A-Z]+.*/                                                                                                
-    split('_').map{|e| e.capitalize}.join                                                                                                           
-  end   
+  def camel_case
+    return self if self !~ /_/ && self =~ /[A-Z]+.*/
+    split('_').map{|e| e.capitalize}.join
+  end
 end
 
 # http://gist.github.com/151324
@@ -49,3 +53,88 @@ class Hash
     end
   end
 end
+
+# we don't want to stop the entire EM reactor when we stop a web server
+module Thin
+  module Backends
+    class Base
+      def stop!
+        @running  = false
+        @stopping = false
+
+        # EventMachine.stop if EventMachine.reactor_running?
+        @connections.each { |connection| connection.close_connection }
+        close
+      end
+    end
+  end
+end
+
+# Resque is really designed around a multiprocess model, so we here we
+# stub some that behaviour away.
+module Resque
+
+  class Worker
+
+    def procline(string)
+      # $0 = "resque-#{Resque::Version}: #{string}"
+      # log! $0
+    end
+
+    # Redefining the entire method to stop the direct access to $0 :(
+    def work(interval = 5.0, &block)
+      interval = Float(interval)
+      # $0 = "resque: Starting"
+      startup
+
+      loop do
+        break if shutdown?
+
+        if not paused? and job = reserve
+          log "got: #{job.inspect}"
+          job.worker = self
+          run_hook :before_fork, job
+          working_on job
+
+          if @child = fork
+            srand # Reseeding
+            procline "Forked #{@child} at #{Time.now.to_i}"
+            Process.wait(@child)
+          else
+            unregister_signal_handlers if !@cant_fork && term_child
+            procline "Processing #{job.queue} since #{Time.now.to_i}"
+            redis.client.reconnect if !@cant_fork # Don't share connection with parent
+            perform(job, &block)
+            exit! unless @cant_fork
+          end
+
+          done_working
+          @child = nil
+        else
+          break if interval.zero?
+          log! "Sleeping for #{interval} seconds"
+          procline paused? ? "Paused" : "Waiting for #{@queues.join(',')}"
+          sleep interval
+        end
+      end
+
+    ensure
+      unregister_worker
+    end
+
+  end
+end
+
+# As Redis::Future objects inherit from BasicObject, it's difficult to
+# distinguish between them and other objects in collected data from
+# pipelined queries.
+#
+# (One alternative would be to put other values in Futures ourselves, and
+#  evaluate everything...)
+class Redis
+  class Future
+    def class
+      ::Redis::Future
+    end
+  end
+end

data/lib/flapjack/persistence/data_mapper/models/check.rb

@@ -1,3 +1,5 @@
+#!/usr/bin/env ruby
+
 class Check
   include DataMapper::Resource
 
@@ -6,9 +8,9 @@ class Check
   has n, :related_checks, :child_key => [:child_id, :parent_id]
   has n, :events
 
-  #has n, :parent_checks, :through => :related_checks, 
+  #has n, :parent_checks, :through => :related_checks,
   #       :child_key => :child_id,  :class_name => "Check"
-  #has n, :child_checks,  :through => :related_checks, 
+  #has n, :child_checks,  :through => :related_checks,
   #       :child_key => :parent_id, :class_name => "Check"
 
   belongs_to :node
@@ -27,7 +29,7 @@ class Check
   property :deleted_at, ParanoidDateTime
 
   # copy command onto check
-  before :valid? do 
+  before :valid? do
     if self.check_template && self.command.blank?
       self.command = self.check_template.command
       self.name = self.check_template.name

data/lib/flapjack/persistence/data_mapper/models/check_template.rb

@@ -1,3 +1,5 @@
+#!/usr/bin/env ruby
+
 class CheckTemplate
   include DataMapper::Resource
 

data/lib/flapjack/persistence/data_mapper/models/event.rb

@@ -1,3 +1,5 @@
+#!/usr/bin/env ruby
+
 class Event
   include DataMapper::Resource
 

data/lib/flapjack/persistence/data_mapper/models/node.rb

@@ -1,3 +1,5 @@
+#!/usr/bin/env ruby
+
 class Node
   include DataMapper::Resource
 
@@ -6,7 +8,7 @@ class Node
   property :fqdn, String, :key => true
 
   validates_is_unique :fqdn
-  validates_format :fqdn, :with => /^[0-9|a-z|A-Z|\-|\.]*$/, 
+  validates_format :fqdn, :with => /^[0-9|a-z|A-Z|\-|\.]*$/,
                           :message => "not a RFC1035-formatted FQDN (see section 2.3.1)"
 
   def hostname

data/lib/flapjack/persistence/data_mapper/models/related_check.rb

@@ -1,6 +1,8 @@
+#!/usr/bin/env ruby
+
 class RelatedCheck
   include DataMapper::Resource
- 
+
   belongs_to :parent_check, :class_name => "Check", :child_key => [:parent_id]
   belongs_to :child_check, :class_name => "Check", :child_key => [:child_id]
 

data/lib/flapjack/pikelet.rb

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+
+# This class encapsulates the config data and environmental setup used
+# by the various Flapjack components.
+#
+# "In Australia and New Zealand, small pancakes (about 75 mm in diameter) known as pikelets
+# are also eaten. They are traditionally served with jam and/or whipped cream, or solely
+# with butter, at afternoon tea, but can also be served at morning tea."
+#    from http://en.wikipedia.org/wiki/Pancake
+
+require 'log4r'
+require 'log4r/outputter/consoleoutputters'
+require 'log4r/outputter/syslogoutputter'
+
+module Flapjack
+  module Pikelet
+    attr_accessor :logger, :redis, :config
+
+    def should_quit?
+      @should_quit
+    end
+
+    def stop
+      @should_quit = true
+    end
+
+    def build_redis_connection_pool(options = {})
+      return unless @bootstrapped
+      if defined?(EventMachine) && defined?(EventMachine::Synchrony)
+        EventMachine::Synchrony::ConnectionPool.new(:size => options[:size] || 5) do
+          ::Redis.new(@redis_config.merge(:driver => 'synchrony'))
+        end
+      else
+        ::Redis.new(@redis_config)
+      end
+    end
+
+    def bootstrap(opts = {})
+      return if @bootstrapped
+
+      unless @logger = opts[:logger]
+        @logger = Log4r::Logger.new("#{self.class.to_s.downcase.gsub('::', '-')}")
+        @logger.add(Log4r::StdoutOutputter.new("flapjack"))
+        @logger.add(Log4r::SyslogOutputter.new("flapjack"))
+      end
+
+      @redis_config = opts[:redis] || {}
+      @config = opts[:config] || {}
+
+      @should_quit = false
+
+      @bootstrapped = true
+    end
+
+  end
+end

data/lib/flapjack/transports/beanstalkd.rb

@@ -23,7 +23,7 @@ module Flapjack
         begin
           @queue = Beanstalk::Pool.new(["#{@config.host}:#{@config.port}"], @config.queue_name)
         rescue Beanstalk::NotConnected => e
-          @log.error("Couldn't connect to the '#{@config.queue_name}' Beanstalk queue. Waiting 5 seconds, then retrying.")
+          @log.error("Couldn't connect to the '#{@config.queue_name}' Beanstalk queue. Retrying in 5 seconds.")
           sleep 5
           retry
         end

data/lib/flapjack/transports/result.rb

@@ -10,22 +10,22 @@ module Flapjack
         @job = options[:job]
         @result = OpenStruct.new(options[:result])
       end
-  
-      # Whether a check returns an ok status. 
+
+      # Whether a check returns an ok status.
       def ok?
         @result.retval == 0
       end
-    
+
       # Whether a check has a warning status.
       def warning?
         @result.retval == 1
       end
-    
+
       # Whether a check has a critical status.
       def critical?
         @result.retval == 2
       end
-    
+
       # Human readable representation of the check's return value.
       def status
         case @result.retval
@@ -34,7 +34,7 @@ module Flapjack
         when 2 ; "critical"
         end
       end
-   
+
       # FIXME: there is a *lot* of duplication here - implement a proxy
       # object pattern?
       def id

data/lib/flapjack/utility.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+
+module Flapjack
+  module Utility
+
+    def time_period_in_words(period)
+      period_mm, period_ss  = period.divmod(60)
+      period_hh, period_mm  = period_mm.divmod(60)
+      period_dd, period_hh  = period_hh.divmod(24)
+      ["#{period_dd} days",
+       "#{period_hh} hours",
+       "#{period_mm} minutes",
+       "#{period_ss} seconds"].reject {|s| s =~ /^0 /}.join(', ')
+    end
+
+    # Returns relative time in words referencing the given date
+    # relative_time_ago(Time.now) => 'about a minute ago'
+    def relative_time_ago(from_time)
+      distance_in_minutes = (((Time.now - from_time.to_time).abs)/60).round
+      case distance_in_minutes
+        when 0..1 then 'about a minute'
+        when 2..44 then "#{distance_in_minutes} minutes"
+        when 45..89 then 'about 1 hour'
+        when 90..1439 then "about #{(distance_in_minutes.to_f / 60.0).round} hours"
+        when 1440..2439 then '1 day'
+        when 2440..2879 then 'about 2 days'
+        when 2880..43199 then "#{(distance_in_minutes / 1440).round} days"
+        when 43200..86399 then 'about 1 month'
+        when 86400..525599 then "#{(distance_in_minutes / 43200).round} months"
+        when 525600..1051199 then 'about 1 year'
+        else "over #{(distance_in_minutes / 525600).round} years"
+      end
+    end
+
+    # returns a string showing the local timezone we're running in
+    # eg "CST (UTC+09:30)"
+    def local_timezone
+      tzname = Time.new.zone
+      q, r = Time.new.utc_offset.divmod(3600)
+      sign = (q < 0) ? '-' : '+'
+      tzoffset = sign + "%02d" % q.abs.to_s + ':' + r.to_f.div(60).to_s
+      "#{tzname} (UTC#{tzoffset})"
+    end
+
+  end
+end

data/lib/flapjack/version.rb

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+module Flapjack
+  VERSION = "0.6.23"
+end

data/lib/flapjack/web.rb

@@ -0,0 +1,198 @@
+#!/usr/bin/env ruby
+
+require 'fiber'
+
+require 'chronic'
+require 'chronic_duration'
+require 'sinatra/base'
+require 'haml'
+require 'rack/fiber_pool'
+
+require 'flapjack/pikelet'
+require 'flapjack/data/entity_check'
+require 'flapjack/utility'
+
+module Flapjack
+  class Web < Sinatra::Base
+
+    # doesn't work with Rack::Test for some reason
+    unless 'test'.eql?(FLAPJACK_ENV)
+      rescue_exception = Proc.new { |env, exception|
+        p exception.message
+        puts exception.backtrace.join("\n")
+        [503, {}, exception.message]
+      }
+
+      use Rack::FiberPool, :size => 25, :rescue_exception => rescue_exception
+    end
+    use Rack::MethodOverride
+    extend Flapjack::Pikelet
+    include Flapjack::Utility
+
+    set :views, settings.root + '/web/views'
+
+    before do
+      # will only initialise the first time it's run
+      Flapjack::Web.bootstrap
+    end
+
+    get '/' do
+      self_stats
+
+      # TODO (?) recast as Entity.all do |e|; e.checks.do |ec|; ...
+      @states = @@redis.keys('*:*:states').map { |r|
+        parts  = r.split(':')[0..1]
+        [parts[0], parts[1]] + entity_check_state(parts[0], parts[1])
+      }.compact.sort_by {|parts| parts }
+      haml :index
+    end
+
+    get '/failing' do
+      self_stats
+      @states = @@redis.zrange('failed_checks', 0, -1).map {|key|
+        parts  = key.split(':')
+        [parts[0], parts[1]] + entity_check_state(parts[0], parts[1])
+      }.compact.sort_by {|parts| parts}
+      haml :index
+    end
+
+    get '/self_stats' do
+      self_stats
+      haml :self_stats
+    end
+
+    get '/check' do
+      @entity = params[:entity]
+      @check = params[:check]
+
+      entity_check = get_entity_check(@entity, @check)
+      return 404 if entity_check.nil?
+
+      last_change = entity_check.last_change
+
+      @check_state                = entity_check.state
+      @check_last_update          = entity_check.last_update
+      @check_last_change          = last_change
+      @check_summary              = entity_check.summary
+      @last_notifications         =
+        {:problem         => entity_check.last_problem_notification,
+         :recovery        => entity_check.last_recovery_notification,
+         :acknowledgement => entity_check.last_acknowledgement_notification
+        }
+      @in_scheduled_maintenance   = entity_check.in_scheduled_maintenance?
+      @in_unscheduled_maintenance = entity_check.in_unscheduled_maintenance?
+      @scheduled_maintenances     = entity_check.maintenances(nil, nil, :scheduled => true)
+      @acknowledgement_id         = entity_check.failed? ?
+        entity_check.event_count_at(entity_check.last_change) : nil
+
+      haml :check
+    end
+
+    post '/acknowledgements/:entity/:check' do
+      @entity = params[:entity]
+      @check = params[:check]
+      @summary = params[:summary]
+      @acknowledgement_id = params[:acknowledgement_id]
+
+      dur   = ChronicDuration.parse(params[:duration] || '')
+      @duration = (dur.nil? || (dur <= 0)) ? (4 * 60 * 60) : dur
+
+      entity_check = get_entity_check(@entity, @check)
+      return 404 if entity_check.nil?
+
+      ack = entity_check.create_acknowledgement('summary' => (@summary || ''),
+        'acknowledgement_id' => @acknowledgement_id, 'duration' => @duration)
+      @acknowledge_success = !!ack
+      [201, haml(:acknowledge)]
+    end
+
+    # FIXME: there is bound to be a more idiomatic / restful way of doing this
+    post '/end_unscheduled_maintenance/:entity/:check' do
+      @entity = params[:entity]
+      @check = params[:check]
+
+      entity_check = get_entity_check(@entity, @check)
+      return 404 if entity_check.nil?
+
+      entity_check.end_unscheduled_maintenance
+
+      redirect back
+    end
+
+    # create scheduled maintenance
+    post '/scheduled_maintenances/:entity/:check' do
+      start_time = Chronic.parse(params[:start_time]).to_i
+      raise ArgumentError, "start time parsed to zero" unless start_time > 0
+      duration   = ChronicDuration.parse(params[:duration])
+      summary    = params[:summary]
+
+      entity_check = get_entity_check(params[:entity],  params[:check])
+      return 404 if entity_check.nil?
+
+      entity_check.create_scheduled_maintenance(:start_time => start_time,
+                                                :duration   => duration,
+                                                :summary    => summary)
+      redirect back
+    end
+
+    # delete a scheduled maintenance
+    delete '/scheduled_maintenances/:entity/:check' do
+      entity_check = get_entity_check(params[:entity], params[:check])
+      return 404 if entity_check.nil?
+
+      entity_check.delete_scheduled_maintenance(:start_time => params[:start_time].to_i)
+      redirect back
+    end
+
+  private
+
+    def get_entity_check(entity, check)
+      entity_obj = (entity && entity.length > 0) ?
+        Flapjack::Data::Entity.find_by_name(entity, :redis => @@redis) : nil
+      return if entity_obj.nil? || (check.nil? || check.length == 0)
+      Flapjack::Data::EntityCheck.for_entity(entity_obj, check, :redis => @@redis)
+    end
+
+    def entity_check_state(entity_name, check)
+      entity = Flapjack::Data::Entity.find_by_name(entity_name,
+        :redis => @@redis)
+      return if entity.nil?
+      entity_check = Flapjack::Data::EntityCheck.for_entity(entity,
+        check, :redis => @@redis)
+      latest_notif =
+        {:problem         => entity_check.last_problem_notification,
+         :recovery        => entity_check.last_recovery_notification,
+         :acknowledgement => entity_check.last_acknowledgement_notification
+        }.max_by {|n| n[1] || 0}
+      [(entity_check.state       || '-'),
+       (entity_check.last_change || '-'),
+       (entity_check.last_update || '-'),
+       entity_check.in_unscheduled_maintenance?,
+       entity_check.in_scheduled_maintenance?,
+       latest_notif[0],
+       latest_notif[1]
+      ]
+    end
+
+    def self_stats
+      @keys = @@redis.keys '*'
+      @count_failing_checks  = @@redis.zcard 'failed_checks'
+      @count_all_checks      = @@redis.keys('check:*:*').length
+      @event_counter_all     = @@redis.hget('event_counters', 'all')
+      @event_counter_ok      = @@redis.hget('event_counters', 'ok')
+      @event_counter_failure = @@redis.hget('event_counters', 'failure')
+      @event_counter_action  = @@redis.hget('event_counters', 'action')
+      @boot_time             = Time.at(@@redis.get('boot_time').to_i)
+      @uptime                = Time.now.to_i - @boot_time.to_i
+      @uptime_string         = time_period_in_words(@uptime)
+      @event_rate_all        = (@uptime > 0) ?
+                                 (@event_counter_all.to_f / @uptime) : 0
+      @events_queued = @@redis.llen('events')
+    end
+
+    def logger
+      Flapjack::Web.logger
+    end
+
+  end
+end