flapjack 1.1.0 → 1.2.0rc1

data/lib/flapjack/cli/receiver.rb

@@ -2,9 +2,7 @@
 
 require 'dante'
 require 'redis'
-
-require 'oj'
-Oj.default_options = { :indent => 0, :mode => :strict }
+require 'hiredis'
 
 require 'flapjack/configuration'
 require 'flapjack/data/event'
@@ -21,16 +19,28 @@ module Flapjack
         @options = options
 
         @config = Flapjack::Configuration.new
-        @config.load(global_options[:config])
-        @config_env = @config.all
 
-        if @config_env.nil? || @config_env.empty?
-          exit_now! "No config data for environment '#{FLAPJACK_ENV}' found in '#{global_options[:config]}'"
+        if 'mirror'.eql?(@options[:type]) &&
+          (global_options[:config].nil? || global_options[:config].strip.empty?)
+
+          @config_env = {}
+          @config_runner = {}
+        else
+          @config.load(global_options[:config])
+          @config_env = @config.all
+
+          if @config_env.nil? || @config_env.empty?
+            exit_now! "No config data for environment '#{FLAPJACK_ENV}' found in '#{global_options[:config]}'"
+          end
+
+          @config_runner = @config_env["#{@options[:type]}-receiver"] || {}
         end
 
-        @config_runner = @config_env["#{@options[:type]}-receiver"] || {}
+        @redis_options = @config.for_redis
+      end
 
-        @pidfile = case
+      def pidfile
+        @pidfile ||= case
         when !@options[:pidfile].nil?
           @options[:pidfile]
         when !@config_env['pid_dir'].nil?
@@ -38,8 +48,10 @@ module Flapjack
         else
           "/var/run/flapjack/#{@options[:type]}-receiver.pid"
         end
+      end
 
-        @logfile = case
+      def logfile
+        @logfile ||= case
         when !@options[:logfile].nil?
           @options[:logfile]
         when !@config_env['log_dir'].nil?
@@ -47,8 +59,6 @@ module Flapjack
         else
           "/var/run/flapjack/#{@options[:type]}-receiver.log"
         end
-
-        @redis_options = @config.for_redis
       end
 
       def start
@@ -97,7 +107,7 @@ module Flapjack
       def status
         if runner(@options[:type]).daemon_running?
           pid = get_pid
-          uptime = Time.now - File.stat(@pidfile).ctime
+          uptime = Time.now - File.stat(pidfile).ctime
           puts "#{@options[:type]}-receiver is running: pid #{pid}, uptime #{uptime}"
         else
           exit_now! "#{@options[:type]}-receiver is not running"
@@ -109,22 +119,30 @@ module Flapjack
       end
 
       def mirror
+        if (@options[:dest].nil? || @options[:dest].strip.empty?) &&
+          @redis_options.nil?
+
+          exit_now! "No destination redis URL passed, and none configured"
+        end
+
         mirror_receive(:source => @options[:source],
-          :all => @options[:all], :follow => @options[:follow],
-          :last => @options[:last], :time => @options[:time])
+          :dest => @options[:dest] || @redis_options,
+          :include => @options[:include], :all => @options[:all],
+          :follow => @options[:follow], :last => @options[:last],
+          :time => @options[:time])
       end
 
       private
 
       def redis
-        @redis ||= Redis.new(@redis_options)
+        @redis ||= Redis.new(@redis_options.merge(:driver => :hiredis))
       end
 
       def runner(type)
         return @runner if @runner
 
-        @runner = Dante::Runner.new("#{@options[:type]}-receiver", :pid_path => @pidfile,
-          :log_path => @logfile)
+        @runner = Dante::Runner.new("#{@options[:type]}-receiver", :pid_path => pidfile,
+          :log_path => logfile)
         @runner
       end
 
@@ -254,7 +272,7 @@ module Flapjack
       end
 
       def get_pid
-        IO.read(@pidfile).chomp.to_i
+        IO.read(pidfile).chomp.to_i
       rescue StandardError
         pid = nil
       end
@@ -298,7 +316,6 @@ module Flapjack
       end
 
       def json_feeder(opts = {})
-
         input = if opts[:from]
           File.open(opts[:from]) # Explodes if file does not exist.
         elsif $stdin.tty?
@@ -328,16 +345,30 @@ module Flapjack
         puts "Done."
       end
 
-
       def mirror_receive(opts)
         unless opts[:follow] || opts[:all]
           exit_now! "one or both of --follow or --all is required"
         end
 
-        source_redis = Redis.new(:url => opts[:source])
+        include_re = nil
+        unless opts[:include].nil? || opts[:include].strip.empty?
+          begin
+            include_re = Regexp.new(opts[:include].strip)
+          rescue RegexpError
+            exit_now! "could not parse include Regexp: #{opts[:include].strip}"
+          end
+        end
+
+        source_addr = opts[:source]
+        source_redis = Redis.new(:url => source_addr, :driver => :hiredis)
 
-        archives = mirror_get_archive_keys_stats(source_redis)
-        raise "found no archives!" unless archives && archives.length > 0
+        dest_addr  = opts[:dest]
+        dest_redis = Redis.new(:url => dest_addr, :driver => :hiredis)
+
+        refresh_archive_index(source_addr, :source => source_redis, :dest => dest_redis)
+        archives = mirror_get_archive_keys_stats(source_addr, :source => source_redis,
+          :dest => dest_redis)
+        raise "found no archives!" if archives.empty?
 
         puts "found archives: #{archives.inspect}"
 
@@ -350,53 +381,85 @@ module Flapjack
         events_sent = 0
         case
         when opts[:all]
-          archive_key = archives[0][:name]
+          archive_idx = 0
           cursor      = -1
         when opts[:last], opts[:time]
           raise "Sorry, unimplemented"
         else
           # wait for the next event to be archived, so point the cursor at a non-existant
           # slot in the list, the one before the 0'th
-          archive_key = archives[-1][:name]
+          archive_idx = archives.size - 1
           cursor      = -1 - archives[-1][:size]
         end
 
+        archive_key = archives[archive_idx][:name]
         puts archive_key
 
         loop do
-          new_archive_key = false
-          # something to read at cursor?
-          event = source_redis.lindex(archive_key, cursor)
-          if event
-            Flapjack::Data::Event.add(event, :redis => redis)
-            events_sent += 1
-            print "#{events_sent} " if events_sent % 1000 == 0
+          event_json = source_redis.lindex(archive_key, cursor)
+          if event_json
+            event = Flapjack::Data::Event.parse_and_validate(event_json)
+            if !event.nil? && (include_re.nil? ||
+              (include_re === "#{event['entity']}:#{event['check']}"))
+
+              Flapjack::Data::Event.add(event, :redis => dest_redis)
+              events_sent += 1
+              print "#{events_sent} " if events_sent % 1000 == 0
+            end
             cursor -= 1
+            next
+          end
+
+          archives = mirror_get_archive_keys_stats(source_addr,
+            :source => source_redis, :dest => dest_redis)
+
+          if archives.any? {|a| a[:size] == 0}
+            # data may be out of date -- refresh, then reject any immediately
+            # expired keys directly; don't keep chasing updated data
+            refresh_archive_index(source_addr, :source => source_redis, :dest => dest_redis)
+            archives = mirror_get_archive_keys_stats(source_addr,
+              :source => source_redis, :dest => dest_redis).select {|a| a[:size] > 0}
+          end
+
+          if archives.empty?
+            sleep 1
+            next
+          end
+
+          archive_idx = archives.index {|a| a[:name] == archive_key }
+          archive_idx = archive_idx.nil? ? 0 : (archive_idx + 1)
+          if archives[archive_idx]
+            archive_key = archives[archive_idx][:name]
+            puts archive_key
+            cursor = -1
           else
-            puts "\narchive key: #{archive_key}, cursor: #{cursor}"
-            # do we need to look at the next archive bucket?
-            archives = mirror_get_archive_keys_stats(source_redis)
-            i = archives.index {|a| a[:name] == archive_key }
-            if archives[i][:size] = (cursor.abs + 1)
-              if archives[i + 1]
-                archive_key = archives[i + 1][:name]
-                puts archive_key
-                cursor = -1
-                new_archive_key = true
-              else
-                return unless opts[:follow]
-              end
-            end
-            sleep 1 unless new_archive_key
+            break unless opts[:follow]
+            sleep 1
           end
         end
       end
 
-      def mirror_get_archive_keys_stats(source_redis)
-        source_redis.keys("events_archive:*").sort.map {|a|
-          { :name => a,
-            :size => source_redis.llen(a) }
-        }
+      def mirror_get_archive_keys_stats(name, opts = {})
+        source_redis = opts[:source]
+        dest_redis   = opts[:dest]
+        dest_redis.smembers("known_events_archive_keys:#{name}").sort.collect do |eak|
+          {:name => eak, :size => source_redis.llen(eak)}
+        end
+      end
+
+      def refresh_archive_index(name, opts = {})
+        source_redis = opts[:source]
+        dest_redis   = opts[:dest]
+        # refresh the key name cache, avoid repeated calls to redis KEYS
+        # this cache will be updated any time a new archive bucket is created
+        archive_keys = source_redis.keys("events_archive:*").group_by do |ak|
+          (source_redis.llen(ak) > 0) ? 't' : 'f'
+        end
+
+        {'f' => :srem, 't' => :sadd}.each_pair do |k, cmd|
+          next unless archive_keys.has_key?(k) && !archive_keys[k].empty?
+          dest_redis.send(cmd, "known_events_archive_keys:#{name}", archive_keys[k])
+        end
       end
 
     end
@@ -596,9 +659,13 @@ command :receiver do |receiver|
   receiver.desc 'Mirror receiver'
   receiver.command :mirror do |mirror|
 
-    mirror.flag     [:s, 'source'], :desc => 'URL of source redis database, eg redis://localhost:6379/0',
+    mirror.flag     [:s, 'source'], :desc => 'URL of source redis database, e.g. redis://localhost:6379/0',
       :required => true
 
+    mirror.flag     [:d, 'dest'],   :desc => 'URL of destination redis database, e.g. redis://localhost:6379/1'
+
+    mirror.flag     [:i, 'include'], :desc => 'Regexp which must match event id for it to be mirrored'
+
     # one or both of follow, all is required
     mirror.switch   [:f, 'follow'], :desc => 'keep reading events as they are archived on the source',
       :default_value => nil
@@ -615,6 +682,7 @@ command :receiver do |receiver|
       :default_value => nil
 
     mirror.action do |global_options,options,args|
+      options.merge!(:type => 'mirror')
       receiver = Flapjack::CLI::Receiver.new(global_options, options)
       receiver.mirror
     end

data/lib/flapjack/cli/server.rb

@@ -2,11 +2,6 @@
 
 require 'dante'
 
-require 'oj'
-Oj.mimic_JSON
-Oj.default_options = { :indent => 0, :mode => :strict }
-require 'active_support/json'
-
 require 'flapjack/coordinator'
 
 module Flapjack

data/lib/flapjack/coordinator.rb

@@ -32,6 +32,12 @@ module Flapjack
       EM.synchrony do
         setup_signals if options[:signals]
 
+        redis = Flapjack::RedisPool.new(:config => @redis_options, :size => 1)
+        ['entity', 'check'].each do |type|
+          discovered = redis.keys("#{type}_tag:*")
+          redis.sadd("known_tags:#{type}_tag", discovered) unless discovered.empty?
+        end
+
         begin
           add_pikelets(pikelets(@config.all))
           loop do

data/lib/flapjack/data/contact.rb

@@ -7,8 +7,6 @@ require 'set'
 require 'ice_cube'
 require 'flapjack/data/entity'
 require 'flapjack/data/notification_rule'
-require 'flapjack/data/tag'
-require 'flapjack/data/tag_set'
 
 require 'securerandom'
 
@@ -21,7 +19,6 @@ module Flapjack
       attr_accessor :id, :first_name, :last_name, :email, :media,
         :media_intervals, :media_rollup_thresholds, :pagerduty_credentials
 
-      TAG_PREFIX = 'contact_tag'
       ALL_MEDIA  = ['email', 'sms', 'sms_twilio', 'jabber', 'pagerduty', 'sns']
 
       def self.all(options = {})
@@ -75,9 +72,12 @@ module Flapjack
         end
 
         self.add_or_update(contact_id, contact_data, :redis => redis)
-        if contact = self.find_by_id(contact_id, :redis => redis)
+        contact = self.find_by_id(contact_id, :redis => redis)
+
+        unless contact.nil?
           contact.notification_rules # invoke to create general rule
         end
+
         contact
       end
 
@@ -128,9 +128,6 @@ module Flapjack
         # the main alerts sorted set, remove all alerts_by sorted sets
         # for the contact
 
-        # remove this contact from all tags it's marked with
-        self.delete_tags(*self.tags.to_a)
-
         # remove all associated notification rules
         self.notification_rules.each do |nr|
           self.delete_notification_rule(nr)
@@ -252,8 +249,8 @@ module Flapjack
           rule = self.add_notification_rule({
               :entities           => [],
               :regex_entities     => [],
-              :tags               => Flapjack::Data::TagSet.new([]),
-              :regex_tags         => Flapjack::Data::TagSet.new([]),
+              :tags               => Set.new([]),
+              :regex_tags         => Set.new([]),
               :time_restrictions  => [],
               :warning_media      => ALL_MEDIA,
               :critical_media     => ALL_MEDIA,
@@ -415,37 +412,6 @@ module Flapjack
         @redis.zcard("contact_alerting_checks:#{self.id}:media:#{media}")
       end
 
-      # FIXME
-      # do a mixin with the following tag methods, they will be the same
-      # across all objects we allow tags on
-
-      # return the set of tags for this contact
-      def tags
-        @tags ||= Flapjack::Data::TagSet.new( @redis.keys("#{TAG_PREFIX}:*").inject([]) {|memo, tag|
-          if Flapjack::Data::Tag.find(tag, :redis => @redis).include?(@id.to_s)
-            memo << tag.sub(/^#{TAG_PREFIX}:/, '')
-          end
-          memo
-        } )
-      end
-
-      # adds tags to this contact
-      def add_tags(*enum)
-        enum.each do |t|
-          Flapjack::Data::Tag.create("#{TAG_PREFIX}:#{t}", [@id], :redis => @redis)
-          tags.add(t)
-        end
-      end
-
-      # removes tags from this contact
-      def delete_tags(*enum)
-        enum.each do |t|
-          tag = Flapjack::Data::Tag.find("#{TAG_PREFIX}:#{t}", :redis => @redis)
-          tag.delete(@id)
-          tags.delete(t)
-        end
-      end
-
       # return a list of media enabled for this contact
       # eg [ 'email', 'sms' ]
       def media_list
@@ -488,32 +454,20 @@ module Flapjack
         end
       end
 
-      def to_json(*args)
-        { "id"                      => self.id,
-          "first_name"              => self.first_name,
-          "last_name"               => self.last_name,
-          "email"                   => self.email,
-          "media"                   => self.media,
-          "media_intervals"         => self.media_intervals,
-          "media_rollup_thresholds" => self.media_rollup_thresholds,
-          "timezone"                => self.timezone.name,
-          "tags"                    => self.tags.to_a
-        }.to_json
-      end
-
       def to_jsonapi(opts = {})
-        { "id"                    => self.id,
+        json_data = {
+          "id"                    => self.id,
           "first_name"            => self.first_name,
           "last_name"             => self.last_name,
           "email"                 => self.email,
           "timezone"              => self.timezone.name,
-          "tags"                  => self.tags.to_a,
           "links"                 => {
             :entities               => opts[:entity_ids]          || [],
             :media                  => self.media_ids             || [],
             :notification_rules     => self.notification_rule_ids || [],
           }
-        }.to_json
+        }
+        Flapjack.dump_json(json_data)
       end
 
     private
@@ -536,12 +490,6 @@ module Flapjack
 
         redis.hmset("contact:#{contact_id}", *attrs) unless attrs.empty?
 
-        if ( ! contact_data['tags'].nil? && contact_data['tags'].is_a?(Enumerable))
-          contact_data['tags'].each do |t|
-            Flapjack::Data::Tag.create("#{TAG_PREFIX}:#{t}", [contact_id], :redis => redis)
-          end
-        end
-
         unless contact_data['media'].nil?
           redis.del("contact_media:#{contact_id}")
           redis.del("contact_media_intervals:#{contact_id}")

data/lib/flapjack/data/entity.rb

@@ -3,8 +3,7 @@
 require 'securerandom'
 
 require 'flapjack/data/contact'
-require 'flapjack/data/tag'
-require 'flapjack/data/tag_set'
+require 'flapjack/data/tagged'
 
 module Flapjack
 
@@ -14,7 +13,7 @@ module Flapjack
 
       attr_accessor :name, :id
 
-      TAG_PREFIX = 'entity_tag'
+      include Tagged
 
       def self.all(options = {})
         raise "Redis connection not set" unless redis = options[:redis]
@@ -452,9 +451,14 @@ module Flapjack
             redis.sadd("contacts_for:#{entity_id}", contact_id)
           }
         end
-        self.new(:name  => entity_name,
-                 :id    => entity_id,
-                 :redis => redis)
+
+        e = self.new(:name  => entity_name,
+                     :id    => entity_id,
+                     :redis => redis)
+        if entity['tags'] && entity['tags'].respond_to?(:each)
+          e.add_tags(*entity['tags'])
+        end
+        e
       end
 
       def self.find_by_name(entity_name, options = {})
@@ -508,25 +512,14 @@ module Flapjack
         }.sort
       end
 
-      def self.find_all_with_tags(tags, options = {})
-        raise "Redis connection not set" unless redis = options[:redis]
-        tags_prefixed = tags.collect {|tag|
-          "#{TAG_PREFIX}:#{tag}"
-        }
-        logger.debug "tags_prefixed: #{tags_prefixed.inspect}" if logger = options[:logger]
-        Flapjack::Data::Tag.find_intersection(tags_prefixed, :redis => redis).collect {|entity_id|
-          Flapjack::Data::Entity.find_by_id(entity_id, :redis => redis).name
-        }.compact
-      end
-
       def self.current_names(options = {})
         raise "Redis connection not set" unless redis = options[:redis]
         redis.zrange('current_entities', 0, -1)
       end
 
-      def self.find_all_with_failing_checks(options)
+      def self.find_all_names_with_failing_checks(options)
         raise "Redis connection not set" unless redis = options[:redis]
-        Flapjack::Data::EntityCheck.find_current_failing_by_entity(:redis => redis).keys
+        Flapjack::Data::EntityCheck.find_current_names_failing_by_entity(:redis => redis).keys
       end
 
       def contacts
@@ -552,6 +545,19 @@ module Flapjack
         end
       end
 
+      def self.check_ids_for(entity_ids, options = {})
+        raise "Redis connection not set" unless redis = options[:redis]
+
+        entity_ids.inject({}) do |memo, entity_id|
+          entity_name = redis.hget("entity:#{entity_id}", 'name')
+          unless entity_name.nil? || entity_name.empty?
+            checks = Flapjack::Data::EntityCheck.find_all_names_for_entity_name(entity_name, :redis => redis)
+            memo[entity_id] = checks.collect {|check| "#{entity_name}:#{check}" }
+          end
+          memo
+        end
+      end
+
       def check_list
         @redis.zrange("current_checks:#{@name}", 0, -1)
       end
@@ -562,45 +568,23 @@ module Flapjack
         checks.length
       end
 
-      def tags
-        @tags ||= Flapjack::Data::TagSet.new( @redis.keys("#{TAG_PREFIX}:*").inject([]) {|memo, entity_tag|
-          if Flapjack::Data::Tag.find(entity_tag, :redis => @redis).include?(@id.to_s)
-            memo << entity_tag.sub(/^#{TAG_PREFIX}:/, '')
-          end
-          memo
-        } )
-      end
-
-      def add_tags(*enum)
-        enum.each do |t|
-          Flapjack::Data::Tag.create("#{TAG_PREFIX}:#{t}", [@id], :redis => @redis)
-          tags.add(t)
-        end
-      end
-
-      def delete_tags(*enum)
-        enum.each do |t|
-          tag = Flapjack::Data::Tag.find("#{TAG_PREFIX}:#{t}", :redis => @redis)
-          tag.delete(@id)
-          tags.delete(t)
-        end
-      end
-
-      def as_json(*args)
-        {
-          "id"    => self.id,
-          "name"  => self.name,
-        }
-      end
+      # def as_json(*args)
+      #   {
+      #     "id"    => self.id,
+      #     "name"  => self.name,
+      #   }
+      # end
 
       def to_jsonapi(opts = {})
-        {
+        json_data = {
           "id"        => self.id,
           "name"      => self.name,
           "links"     => {
-            :contacts   => opts[:contact_ids] || [],
+            :contacts  => opts[:contact_ids] || [],
+            :checks    => opts[:check_ids]   || [],
           }
-        }.to_json
+        }
+        Flapjack.dump_json(json_data)
       end
 
     private