flapjack 1.2.0rc1 → 1.2.0rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2d2385cedabb54ffdf76fab97912ff06918d6d1d
-  data.tar.gz: 5df58568fccb796ee1f45e332a711b5b01003d15
+  metadata.gz: b10a01932fb4fd6a466e2b975764c3c6ff777e9c
+  data.tar.gz: e25026116bccc9209d8ca98c05585e89dc4b7ce3
 SHA512:
-  metadata.gz: 2192ac69c3abfffd68ab6dd23dbbd4bf2ffb0bbb411620b8c6356857511f6792868a678f15796a8a116bec52b7ff2208b5ff251583fc57825e747816218b3532
-  data.tar.gz: f7bfa433a64c42747c3b9359c8e28049d618f10ee6c03787df83818a2f63dd7a2c04194c234371f7151d3bfd21b22cf28715f0783d80ef91fb8148e3b80ce941
+  metadata.gz: d1e4aeba827545b5b18bc56bd549adff1b93b43bf368cd426d37b847c2d2801ea4e83f22c1af4e87579e947cf8d911a2abcb6025d778d476abcaad4b2ac934ab
+  data.tar.gz: e6956faf10c9a12bf9cd08bf56fbe87be986063967ec27e35e77b5ed313b8a3cff6b50ea14a5d58019b9b8f4c29f6bad483d0ecf739e7a0f609f26e3d9dedfeb

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 ## Flapjack Changelog
 
+# 1.2.0rc2 - 2014-10-17
+- Feature: Cache entity and check name <-> id lookups #682 (@ali-graham)
+- Chore: Add last_change field to status_reports in the API (closes: #678) #679 (@Hobbsee)
+- Chore: Move archive cache responsibility to mirror source. #683 (@ali-graham)
+- Bug: jsonapi: GET /entities is non performant #674 (@ali-graham)
+
 # 1.2.0rc1 - 2014-10-08
 - Feature: Allow updating an entities name via PATCH /entities/id[,id...] #628 (@ali-graham)
 - Feature: Pact specs for flapjack-diner compatability testing #663 (@ali-graham)

data/Gemfile-ruby1.9.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    flapjack (1.2.0rc1)
+    flapjack (1.2.0rc2)
       activesupport (~> 3.2.14)
       blather (~> 0.8.3)
       chronic

data/lib/flapjack/cli/import.rb

@@ -5,6 +5,7 @@ require 'redis'
 require 'flapjack/configuration'
 require 'flapjack/data/contact'
 require 'flapjack/data/entity'
+require 'flapjack/data/migration'
 
 module Flapjack
   module CLI
@@ -61,7 +62,10 @@ module Flapjack
       private
 
       def redis
-        @redis ||= Redis.new(@redis_options.merge(:driver => :ruby))
+        return @redis unless @redis.nil?
+        @redis = Redis.new(@redis_options.merge(:driver => :hiredis))
+        Flapjack::Data::Migration.migrate_entity_check_data_if_required(:redis => @redis)
+        @redis
       end
 
     end

data/lib/flapjack/cli/maintenance.rb

@@ -1,13 +1,12 @@
 #!/usr/bin/env ruby
 
-require 'eventmachine'
-require 'em-synchrony'
 require 'redis'
-require 'redis/connection/synchrony'
+require 'hiredis'
 
 require 'flapjack/configuration'
 require 'flapjack/data/event'
 require 'flapjack/data/entity_check'
+require 'flapjack/data/migration'
 require 'terminal-table'
 
 module Flapjack
@@ -26,7 +25,7 @@ module Flapjack
           exit_now! "No config data for environment '#{FLAPJACK_ENV}' found in '#{global_options[:config]}'"
         end
 
-        @redis_options = config.for_redis.merge(:driver => :ruby)
+        @redis_options = config.for_redis
         @options[:redis] = redis
       end
 
@@ -67,7 +66,10 @@ module Flapjack
       private
 
       def redis
-        @redis ||= Redis.new(@redis_options)
+        return @redis unless @redis.nil?
+        @redis = Redis.new(@redis_options.merge(:driver => :hiredis))
+        Flapjack::Data::Migration.migrate_entity_check_data_if_required(:redis => @redis)
+        @redis
       end
 
     end

data/lib/flapjack/cli/purge.rb

@@ -3,6 +3,8 @@
 require 'hiredis'
 require 'flapjack/configuration'
 
+require 'flapjack/data/migration'
+
 module Flapjack
   module CLI
     class Purge
@@ -50,7 +52,10 @@ module Flapjack
       private
 
       def redis
-        @redis ||= Redis.new(@redis_options.merge(:driver => :ruby))
+        return @redis unless @redis.nil?
+        @redis = Redis.new(@redis_options.merge(:driver => :hiredis))
+        Flapjack::Data::Migration.migrate_entity_check_data_if_required(:redis => @redis)
+        @redis
       end
 
     end

data/lib/flapjack/cli/receiver.rb

@@ -6,6 +6,7 @@ require 'hiredis'
 
 require 'flapjack/configuration'
 require 'flapjack/data/event'
+require 'flapjack/data/migration'
 require 'flapjack/patches'
 
 # TODO options should be overridden by similar config file options
@@ -19,20 +20,17 @@ module Flapjack
         @options = options
 
         @config = Flapjack::Configuration.new
+        @config.load(global_options[:config])
+        @config_env = @config.all
 
-        if 'mirror'.eql?(@options[:type]) &&
-          (global_options[:config].nil? || global_options[:config].strip.empty?)
+        if @config_env.nil? || @config_env.empty?
+          unless 'mirror'.eql?(@options[:type])
+            exit_now! "No config data for environment '#{FLAPJACK_ENV}' found in '#{global_options[:config]}'"
+          end
 
           @config_env = {}
           @config_runner = {}
         else
-          @config.load(global_options[:config])
-          @config_env = @config.all
-
-          if @config_env.nil? || @config_env.empty?
-            exit_now! "No config data for environment '#{FLAPJACK_ENV}' found in '#{global_options[:config]}'"
-          end
-
           @config_runner = @config_env["#{@options[:type]}-receiver"] || {}
         end
 
@@ -135,7 +133,10 @@ module Flapjack
       private
 
       def redis
-        @redis ||= Redis.new(@redis_options.merge(:driver => :hiredis))
+        return @redis unless @redis.nil?
+        @redis = Redis.new(@redis_options.merge(:driver => :hiredis))
+        Flapjack::Data::Migration.migrate_entity_check_data_if_required(:redis => @redis)
+        @redis
       end
 
       def runner(type)
@@ -363,11 +364,18 @@ module Flapjack
         source_redis = Redis.new(:url => source_addr, :driver => :hiredis)
 
         dest_addr  = opts[:dest]
-        dest_redis = Redis.new(:url => dest_addr, :driver => :hiredis)
+        dest_redis = case dest_addr
+        when Hash
+          Redis.new(dest_addr.merge(:driver => :hiredis))
+        when String
+          Redis.new(:url => dest_addr, :driver => :hiredis)
+        else
+          exit_now! "could not understand destination Redis config"
+        end
+
+        Flapjack::Data::Migration.migrate_entity_check_data_if_required(:redis => dest_redis)
 
-        refresh_archive_index(source_addr, :source => source_redis, :dest => dest_redis)
-        archives = mirror_get_archive_keys_stats(source_addr, :source => source_redis,
-          :dest => dest_redis)
+        archives = mirror_get_archive_keys_stats(:source => source_redis)
         raise "found no archives!" if archives.empty?
 
         puts "found archives: #{archives.inspect}"
@@ -410,16 +418,9 @@ module Flapjack
             next
           end
 
-          archives = mirror_get_archive_keys_stats(source_addr,
-            :source => source_redis, :dest => dest_redis)
-
-          if archives.any? {|a| a[:size] == 0}
-            # data may be out of date -- refresh, then reject any immediately
-            # expired keys directly; don't keep chasing updated data
-            refresh_archive_index(source_addr, :source => source_redis, :dest => dest_redis)
-            archives = mirror_get_archive_keys_stats(source_addr,
-              :source => source_redis, :dest => dest_redis).select {|a| a[:size] > 0}
-          end
+          archives = mirror_get_archive_keys_stats(:source => source_redis).select {|a|
+            a[:size] > 0
+          }
 
           if archives.empty?
             sleep 1
@@ -439,29 +440,13 @@ module Flapjack
         end
       end
 
-      def mirror_get_archive_keys_stats(name, opts = {})
+      def mirror_get_archive_keys_stats(opts = {})
         source_redis = opts[:source]
-        dest_redis   = opts[:dest]
-        dest_redis.smembers("known_events_archive_keys:#{name}").sort.collect do |eak|
+        source_redis.smembers("known_events_archive_keys").sort.collect do |eak|
           {:name => eak, :size => source_redis.llen(eak)}
         end
       end
 
-      def refresh_archive_index(name, opts = {})
-        source_redis = opts[:source]
-        dest_redis   = opts[:dest]
-        # refresh the key name cache, avoid repeated calls to redis KEYS
-        # this cache will be updated any time a new archive bucket is created
-        archive_keys = source_redis.keys("events_archive:*").group_by do |ak|
-          (source_redis.llen(ak) > 0) ? 't' : 'f'
-        end
-
-        {'f' => :srem, 't' => :sadd}.each_pair do |k, cmd|
-          next unless archive_keys.has_key?(k) && !archive_keys[k].empty?
-          dest_redis.send(cmd, "known_events_archive_keys:#{name}", archive_keys[k])
-        end
-      end
-
     end
   end
 end

data/lib/flapjack/cli/simulate.rb

@@ -7,6 +7,7 @@ require 'redis/connection/synchrony'
 
 require 'flapjack/configuration'
 require 'flapjack/data/event'
+require 'flapjack/data/migration'
 
 module Flapjack
   module CLI
@@ -51,7 +52,10 @@ module Flapjack
       private
 
       def redis
-        @redis ||= Redis.new(@redis_options)
+        return @redis unless @redis.nil?
+        @redis = Redis.new(@redis_options)
+        Flapjack::Data::Migration.migrate_entity_check_data_if_required(:redis => @redis)
+        @redis
       end
 
       def events(opts = {})

data/lib/flapjack/coordinator.rb

@@ -32,7 +32,7 @@ module Flapjack
       EM.synchrony do
         setup_signals if options[:signals]
 
-        redis = Flapjack::RedisPool.new(:config => @redis_options, :size => 1)
+        redis = Flapjack::RedisPool.new(:config => @redis_options, :size => 1, :logger => @logger)
         ['entity', 'check'].each do |type|
           discovered = redis.keys("#{type}_tag:*")
           redis.sadd("known_tags:#{type}_tag", discovered) unless discovered.empty?

data/lib/flapjack/data/entity.rb

@@ -21,19 +21,16 @@ module Flapjack
         current_entity_names = (options.has_key?(:enabled) && !options[:enabled].nil?) ?
           Flapjack::Data::Entity.current_names(:redis => redis) : nil
 
-        keys = redis.keys("entity_id:*")
-        return [] unless keys.any?
-        ids = redis.mget(keys)
-        keys.inject([]) {|memo, k|
-          k =~ /^entity_id:(.+)$/; entity_name = $1; entity_id = ids.shift
+        entity_names_by_id = redis.hgetall("all_entity_names_by_id")
+        return [] if entity_names_by_id.empty?
 
+        entity_names_by_id.inject([]) {|memo, (eid, ename)|
           if options[:enabled].nil? ||
-            (options[:enabled].is_a?(TrueClass) && current_entity_names.include?(entity_name) ) ||
-            (options[:enabled].is_a?(FalseClass) && !current_entity_names.include?(entity_name))
+            (options[:enabled].is_a?(TrueClass) && current_entity_names.include?(ename) ) ||
+            (options[:enabled].is_a?(FalseClass) && !current_entity_names.include?(ename))
 
-            memo << self.new(:name => entity_name, :id => entity_id, :redis => redis)
+            memo << self.new(:name => ename, :id => eid, :redis => redis)
           end
-
           memo
         }.sort_by(&:name)
       end
@@ -59,6 +56,7 @@ module Flapjack
 
         alerting_check_keys = redis.keys("contact_alerting_checks:*")
 
+        all_checks       = {}
         failed_checks    = {}
         hashes_to_remove = []
         hashes_to_add    = {}
@@ -77,6 +75,9 @@ module Flapjack
           existing_check = "#{existing_name}:#{ch}"
           new_check      = "#{entity_name}:#{ch}"
 
+          ch_all_score = redis.zscore("all_checks", existing_check)
+          all_checks[ch] = ch_all_score unless ch_all_score.nil?
+
           ch_fail_score = redis.zscore("failed_checks", existing_check)
           failed_checks[ch] = ch_fail_score unless ch_fail_score.nil?
 
@@ -111,12 +112,18 @@ module Flapjack
           redis.rename(chk, chk.sub(/^#{Regexp.escape(existing_name)}:/, "#{entity_name}:"))
         end
 
+        all_checks.each_pair do |ch, score|
+          redis.zrem('all_checks', "#{existing_name}:#{ch}")
+          redis.zadd('all_checks', score, "#{entity_name}:#{ch}")
+        end
+
         # currently failing checks
         failed_checks.each_pair do |ch, score|
           redis.zrem('failed_checks', "#{existing_name}:#{ch}")
           redis.zadd('failed_checks', score, "#{entity_name}:#{ch}")
         end
 
+        redis.rename("all_checks:#{existing_name}", "all_checks:#{entity_name}")
         redis.rename("current_checks:#{existing_name}", "current_checks:#{entity_name}")
 
         unless current_score.nil?
@@ -160,6 +167,9 @@ module Flapjack
         keys_to_delete = []
         keys_to_rename = {}
 
+        all_checks_to_remove = []
+        all_checks_to_add    = {}
+
         failed_checks_to_remove = []
         failed_checks_to_add    = {}
 
@@ -182,6 +192,7 @@ module Flapjack
           old_states = "#{old_check}:states"
           new_states = "#{current_check}:states"
 
+          all_checks_to_remove    << old_check
           failed_checks_to_remove << old_check
 
           if redis.exists("check:#{current_check}")
@@ -195,6 +206,9 @@ module Flapjack
             keys_to_delete << old_states
           else
 
+            ch_all_score = redis.zscore("all_checks", old_check)
+            all_checks_to_add[current_check] = ch_all_score unless ch_all_score.nil?
+
             # can move a failing checks entry over, if it exists
             ch_fail_score = redis.zscore("failed_checks", old_check)
             failed_checks_to_add[current_check] = ch_fail_score unless ch_fail_score.nil?
@@ -240,6 +254,14 @@ module Flapjack
 
         end
 
+        # TODO all_checks sorted set -- merge/rename entries
+
+        if redis.exists("all_checks:#{current_name}")
+          keys_to_delete << "all_checks:#{old_name}"
+        else
+          keys_to_rename["all_checks:#{old_name}"] = "all_checks:#{current_name}"
+        end
+
         if redis.exists("current_checks:#{current_name}")
           keys_to_delete << "current_checks:#{old_name}"
         else
@@ -339,6 +361,14 @@ module Flapjack
           redis.zunionstore(dest, [ctk, dest], :aggregate => :max)
         end
 
+        all_checks_to_remove.each do |actr|
+          redis.zrem('all_checks', actr)
+        end
+
+        all_checks_to_add.each_pair do |acta, score|
+          redis.zadd('all_checks', score, acta)
+        end
+
         failed_checks_to_remove.each do |fctr|
           redis.zrem('failed_checks', fctr)
         end
@@ -415,31 +445,33 @@ module Flapjack
 
           # if an entity exists with the same name as the incoming data,
           # use its id; failing that allocate a random one
-          entity_id = redis.get("entity_id:#{entity_name}")
+          entity_id = redis.hget('all_entity_ids_by_name', entity_name)
 
           if entity_id.nil? || entity_id.empty?
             entity_id = SecureRandom.uuid
-            redis.set("entity_id:#{entity_name}", entity_id)
-            redis.hset("entity:#{entity_id}", 'name', entity_name)
+            redis.hset('all_entity_ids_by_name', entity_name, entity_id)
+            redis.hset('all_entity_names_by_id', entity_id, entity_name)
           end
         else
           # most likely from API import
-          existing_name = redis.hget("entity:#{entity_id}", 'name')
+          existing_name = redis.hget('all_entity_names_by_id', entity_id)
 
-          if existing_name.nil?
+          # if there's an entity with a matching name, this will change its
+          # id; if no entity exists it creates a new one
 
-            # if there's an entity with a matching name, this will change its
-            # id; if no entity exists it creates a new one
-            redis.set("entity_id:#{entity_name}", entity_id)
-            redis.hset("entity:#{entity_id}", 'name', entity_name)
+          if existing_name.nil?
+            redis.hset('all_entity_ids_by_name', entity_name, entity_id)
+            redis.hset('all_entity_names_by_id', entity_id, entity_name)
 
           elsif existing_name != entity_name
-            if redis.renamenx("entity_id:#{existing_name}", "entity_id:#{entity_name}")
+            if redis.hexists('all_entity_ids_by_name', entity_name)
+              merge(existing_name, entity_name, :redis => redis)
+            else
               rename(existing_name, entity_name, :redis => redis) {
-                redis.hset("entity:#{entity_id}", 'name', entity_name)
+                redis.hdel('all_entity_ids_by_name', existing_name)
+                redis.hset('all_entity_ids_by_name', entity_name, entity_id)
+                redis.hset('all_entity_names_by_id', entity_id, entity_name)
               }
-            else
-              merge(existing_name, entity_name, :redis => redis)
             end
           end
         end
@@ -463,7 +495,7 @@ module Flapjack
 
       def self.find_by_name(entity_name, options = {})
         raise "Redis connection not set" unless redis = options[:redis]
-        entity_id = redis.get("entity_id:#{entity_name}")
+        entity_id = redis.hget("all_entity_ids_by_name", entity_name)
         if entity_id.nil? || entity_id.empty?
           # key doesn't exist
           return unless options[:create]
@@ -476,7 +508,7 @@ module Flapjack
 
       def self.find_by_id(entity_id, options = {})
         raise "Redis connection not set" unless redis = options[:redis]
-        entity_name = redis.hget("entity:#{entity_id}", 'name')
+        entity_name = redis.hget("all_entity_names_by_id", entity_id)
         return if entity_name.nil? || entity_name.empty?
         self.new(:name => entity_name, :id => entity_id, :redis => redis)
       end
@@ -495,21 +527,17 @@ module Flapjack
       # time
       def self.find_all_name_matching(pattern, options = {})
         raise "Redis connection not set" unless redis = options[:redis]
+        regexp = nil
         begin
-          regex = /#{pattern}/
+          regexp = Regexp.new(pattern)
         rescue => e
           if @logger
             @logger.info("Jabber#self.find_all_name_matching - unable to use /#{pattern}/ as a regex pattern: #{e}")
           end
-          return nil
+          regexp = nil
         end
-        redis.keys('entity_id:*').inject([]) {|memo, check|
-          a, entity_name = check.split(':', 2)
-          if (entity_name =~ regex) && !memo.include?(entity_name)
-            memo << entity_name
-          end
-          memo
-        }.sort
+        return if regexp.nil?
+        redis.hkeys('all_entity_ids_by_name').select {|en| regexp === en }.sort
       end
 
       def self.current_names(options = {})
@@ -549,11 +577,12 @@ module Flapjack
         raise "Redis connection not set" unless redis = options[:redis]
 
         entity_ids.inject({}) do |memo, entity_id|
-          entity_name = redis.hget("entity:#{entity_id}", 'name')
-          unless entity_name.nil? || entity_name.empty?
-            checks = Flapjack::Data::EntityCheck.find_all_names_for_entity_name(entity_name, :redis => redis)
-            memo[entity_id] = checks.collect {|check| "#{entity_name}:#{check}" }
-          end
+          entity_name = redis.hget('entity_names_by_id', entity_id)
+          next memo if entity_name.nil? || entity_name.empty?
+          en = Regexp.escape(entity_name)
+          check_names = redis.keys("check:#{entity_name}:*").map {|c| c.sub(/^check:#{en}:/, '') } |
+            Flapjack::Data::EntityCheck.find_current_names_for_entity_name(entity_name, :redis => redis)
+          memo[entity_id] = check_names.map {|cn| "#{entity_name}:#{cn}"}
           memo
         end
       end
@@ -568,13 +597,6 @@ module Flapjack
         checks.length
       end
 
-      # def as_json(*args)
-      #   {
-      #     "id"    => self.id,
-      #     "name"  => self.name,
-      #   }
-      # end
-
       def to_jsonapi(opts = {})
         json_data = {
           "id"        => self.id,