coverband 6.2.2 → 7.0.0.rc.2

data/lib/coverband/collectors/abstract_tracker.rb

@@ -32,6 +32,16 @@ module Coverband
         @keys_to_record = Set.new
       end
 
+      ###
+      # Whether two recordings of the same key combine to the same result.
+      # Presence trackers take the later timestamp, so they do; one accumulating
+      # counters has to say so, since re-applying a sum double counts and the
+      # storage layer picks its layout from this.
+      ###
+      def self.idempotent_merge?
+        true
+      end
+
       def logged_keys
         @logged_keys.to_a
       end
@@ -53,9 +63,12 @@ module Coverband
       end
 
       def used_keys
-        return {} unless redis_store
+        return {} unless storage
 
-        redis_store.hgetall(tracker_key)
+        storage.entries
+      rescue => e
+        logger&.error "Coverband: #{self.class.name} failed to read, error #{e.class.name} info #{e.message}"
+        {}
       end
 
       def all_keys
@@ -76,43 +89,63 @@ module Coverband
       end
 
       def tracking_since
-        return "N/A" unless redis_store
+        return "N/A" unless storage
 
-        if (tracking_time = redis_store.get(tracker_time_key))
-          Time.at(tracking_time.to_i).iso8601
-        else
-          "N/A"
-        end
+        (tracking_time = storage.tracking_since) ? tracking_time.iso8601 : "N/A"
       end
 
+      # local state is dropped by the generation change callback, which only
+      # fires once the new pointer is durable; clearing here first would throw
+      # away unsaved work on a reset that never landed
       def reset_recordings
-        return unless redis_store
+        return unless storage
 
-        redis_store.del(tracker_key)
-        redis_store.del(tracker_time_key)
+        storage.reset
       end
 
       def clear_key!(key)
         return unless key
-        return unless redis_store
+        return unless storage
 
-        puts "#{tracker_key} key #{key}"
-        redis_store.hdel(tracker_key, key)
+        storage.delete_entry(key)
         @logged_keys.delete(key)
+        @keys_to_record.delete(key)
       end
 
+      # loss the storage layer noticed and repaired as best it could, surfaced so
+      # the report can say the numbers are partial rather than imply they are not
+      def data_loss
+        storage&.data_loss
+      end
+
+      # so a tab showing nothing can say why, rather than reading as an app that
+      # used nothing
+      def unwritten
+        storage&.unwritten
+      end
+
+      # exposed so a reporting cycle can batch every pointer read it will make
+      def pointer_session
+        storage&.pointer_session
+      end
+
+      ###
+      # The work is neither stored nor held for us. Keeping our keys against
+      # these is what makes an outage of any length lossless, since this set is
+      # unbounded where the storage queue is capped. Anything else means storage
+      # took it, and a second copy would enqueue the same work twice.
+      ###
+      UNSTORED_STATES = %i[failed unavailable].freeze
+
       def save_report
-        return unless redis_store
+        return unless storage
 
-        redis_store.set(tracker_time_key, Time.now.to_i) unless @one_time_timestamp || tracker_time_key_exists?
-        @one_time_timestamp = true
-        reported_time = Time.now.to_i
-        if @keys_to_record.any?
-          redis_store.hset(tracker_key, @keys_to_record.each_with_object({}) { |key, h| h[key.to_s] = reported_time })
-        end
-        @keys_to_record.clear
+        forget_deleted_keys
+        result = storage.record(delta_to_record)
+        @keys_to_record.clear unless UNSTORED_STATES.include?(result)
+        recover_dropped_keys
       rescue => e
-        # we don't want to raise errors if Coverband can't reach redis.
+        # we don't want to raise errors if Coverband can't reach its store.
         # This is a nice to have not a bring the system down
         logger&.error "Coverband: #{self.class.name} failed to store, error #{e.class.name} info #{e.message}"
       end
@@ -132,6 +165,59 @@ module Coverband
 
       protected
 
+      def delta_to_record
+        return {} if @keys_to_record.empty?
+
+        reported_time = Time.now.to_i
+        @keys_to_record.each_with_object({}) { |key, hash| hash[key.to_s] = reported_time }
+      end
+
+      # a key another process cleared has to leave the dedupe set, or a later
+      # genuine use of it would never be enqueued again
+      def forget_deleted_keys
+        storage.newly_tombstoned.each do |key|
+          @logged_keys.delete_if { |logged| logged.to_s == key }
+          @keys_to_record.delete_if { |pending| pending.to_s == key }
+        end
+      end
+
+      ###
+      # A reset means the operator wants everything gone. An eviction means the
+      # backend lost data this process can partly reconstruct, so the keys it
+      # already knows are queued to be reported once more rather than forgotten.
+      ###
+      def drop_local_state!(change)
+        case change.cause
+        when Storage::GenerationChange::POINTER_EVICTION,
+          Storage::GenerationChange::DOCUMENT_EVICTION
+          @logged_keys.each { |key| @keys_to_record << key if track_key?(key) }
+        when Storage::GenerationChange::OPERATOR_RESET
+          @logged_keys.clear
+          @keys_to_record.clear
+        end
+      end
+
+      ###
+      # Work dropped at the storage caps is gone and the dedupe set has moved on,
+      # so those keys would never be reported again: a used view reading as
+      # unused, the dangerous direction for anything driving deletion. What is
+      # known is queued again, the way an eviction already is.
+      #
+      # Only sound where the merge is idempotent -- re-supplying a summed counter
+      # is the double count :retained exists to prevent. Keyed on the loss
+      # timestamp, so an outage re-supplies once per drop, not once per cycle.
+      ###
+      def recover_dropped_keys
+        return unless self.class.idempotent_merge?
+
+        loss = storage.data_loss
+        return unless loss && loss.kind == :pending_dropped
+        return if @recovered_loss_at == loss.at
+
+        @recovered_loss_at = loss.at
+        @logged_keys.each { |key| @keys_to_record << key if track_key?(key) }
+      end
+
       def newly_seen_key?(key)
         !@logged_keys.include?(key)
       end
@@ -141,44 +227,42 @@ module Coverband
         @ignore_patterns.none? { |pattern| key.match?(pattern) }
       end
 
-      private
-
-      def concrete_target
-        raise "subclass must implement"
-      end
+      ###
+      # Presence merging keeps the later of the two timestamps, so applying the
+      # same delta twice changes nothing.
+      ###
+      def merger
+        @merger ||= lambda do |doc, delta|
+          delta.payload.each do |key, value|
+            next if doc.tombstoned?(key, delta.tombstone_epoch)
 
-      def redis_store
-        @redis_store ||= begin
-          store.raw_store
-        rescue NotImplementedError
-          nil
+            existing = doc.payload[key.to_s]
+            doc.payload[key.to_s] = value.to_s if existing.nil? || existing.to_i < value.to_i
+          end
         end
       end
 
-      def tracker_time_key_exists?
-        return false unless redis_store
-
-        if defined?(redis_store.exists?)
-          redis_store.exists?(tracker_time_key)
-        else
-          redis_store.exists(tracker_time_key)
-        end
+      def storage
+        return @storage if defined?(@storage) && @storage
+
+        factory = store.tracker_storage
+        @storage = factory&.for(
+          class_key,
+          merger: merger,
+          idempotent: self.class.idempotent_merge?,
+          logger: logger,
+          on_generation_change: ->(change) { drop_local_state!(change) }
+        )
       end
 
-      def tracker_key
-        "#{class_key}_tracker"
-      end
+      private
 
-      def tracker_time_key
-        "#{class_key}_tracker_time"
+      def concrete_target
+        raise "subclass must implement"
       end
 
       def class_key
-        @class_key ||= if Coverband.configuration.redis_namespace
-          "#{Coverband.configuration.redis_namespace}_#{self.class.name.split("::").last}"
-        else
-          self.class.name.split("::").last
-        end
+        @class_key ||= self.class.name.split("::").last
       end
     end
   end

data/lib/coverband/collectors/query_burst_tracker.rb

@@ -53,20 +53,18 @@ module Coverband
       end
 
       def used_keys
-        return {} unless redis_store
+        return {} unless storage
 
-        stats_hash = redis_store.hgetall(tracker_key)
-        stats_hash.each_with_object({}) do |(key, stats_json), used|
+        storage.entries.each_with_object({}) do |(key, stats_json), used|
           stats = parse_stats(stats_json)
           used[key] = stats["last_seen"].to_i
         end
       end
 
       def used_key_stats
-        return {} unless redis_store
+        return {} unless storage
 
-        stats_hash = redis_store.hgetall(tracker_key)
-        stats_hash
+        storage.entries
           .transform_values { |stats_json| parse_stats(stats_json) }
           .sort_by { |key, stats| [-stats["threshold_hits"].to_i, -stats["total_sql_time_ms"].to_f, key] }
           .to_h
@@ -91,23 +89,65 @@ module Coverband
         }.to_json
       end
 
+      ###
+      # These stats are cumulative, so a lost update can never be noticed by
+      # reading the total back and re-applying one would double count. The
+      # storage layer's applied sequences are what make the retry safe, which is
+      # also why this tracker can't use the per field Redis hash layout.
+      ###
       def save_report
-        return unless redis_store
-        return if @pending_stats.empty?
+        return unless storage
+        # storage may be holding work from a cycle that could not be written,
+        # and a quiet cycle is the only chance it gets to flush it
+        return if @pending_stats.empty? && storage.pending_size.zero?
+
+        delta = @pending_stats.each_with_object({}) { |(key, stats), h| h[key] = stats.to_json }
+        result = storage.record(delta)
+        @pending_stats.clear unless UNSTORED_STATES.include?(result)
+      rescue => e
+        logger&.error "Coverband: #{self.class.name} failed to store, error #{e.class.name} info #{e.message}"
+      end
 
-        redis_store.set(tracker_time_key, Time.now.to_i) unless @one_time_timestamp || tracker_time_key_exists?
-        @one_time_timestamp = true
+      ###
+      # Counters sum and maxima take the larger value, so this merge is not
+      # idempotent: applying the same delta twice inflates the totals.
+      ###
+      def self.idempotent_merge?
+        false
+      end
 
-        existing_stats = redis_store.hgetall(tracker_key)
-        merged = @pending_stats.each_with_object({}) do |(key, pending), h|
-          existing = parse_stats(existing_stats[key])
-          h[key] = merge_stats(existing, pending).to_json
-        end
+      ###
+      # Unsaved aggregates live here rather than in the key sets the parent
+      # clears, so the lifecycle hooks have to reach them too. Otherwise counters
+      # accumulated before a reset get written into the new generation, and a
+      # cleared key comes back with its old totals.
+      ###
+      protected
 
-        redis_store.hset(tracker_key, merged) if merged.any?
-        @pending_stats.clear
-      rescue => e
-        logger&.error "Coverband: #{self.class.name} failed to store, error #{e.class.name} info #{e.message}"
+      def drop_local_state!(change)
+        super
+        # counters are not reconstructible from local state the way presence
+        # keys are, so an eviction loses them either way
+        @pending_stats&.clear if change.cause == Storage::GenerationChange::OPERATOR_RESET
+      end
+
+      public
+
+      def clear_key!(key)
+        @pending_stats&.delete(key.to_s) if key
+        super
+      end
+
+      def merger
+        @merger ||= lambda do |doc, delta|
+          delta.payload.each do |key, stats_json|
+            next if doc.tombstoned?(key, delta.tombstone_epoch)
+
+            existing = parse_stats(doc.payload[key.to_s])
+            incoming = parse_stats(stats_json)
+            doc.payload[key.to_s] = merge_stats(existing, incoming).to_json
+          end
+        end
       end
 
       private
@@ -222,8 +262,15 @@ module Coverband
         Thread.current[CONTEXT_STACK_KEY] ||= []
       end
 
+      ###
+      # With a database backed cache, Coverband's own storage I/O is SQL. Left
+      # unfiltered it lands on whatever controller action or job triggered the
+      # report, inflating its query count and potentially tripping the very
+      # thresholds this tracker exists to detect.
+      ###
       def ignore_sql_event?(payload)
         return true if payload[:cached]
+        return true if Coverband::Storage::IOGuard.active?
 
         name = payload[:name].to_s
         IGNORED_SQL_NAMES.include?(name)
@@ -266,3 +313,9 @@ module Coverband
     end
   end
 end
+
+Coverband::Collectors::TrackerRegistry.register(
+  :query_burst_tracker,
+  tracker_class: Coverband::Collectors::QueryBurstTracker,
+  enabled: ->(config) { config.track_query_bursts }
+)

data/lib/coverband/collectors/route_tracker.rb

@@ -99,3 +99,9 @@ module Coverband
     end
   end
 end
+
+Coverband::Collectors::TrackerRegistry.register(
+  :route_tracker,
+  tracker_class: Coverband::Collectors::RouteTracker,
+  enabled: ->(config) { config.track_routes }
+)

data/lib/coverband/collectors/tracker_registry.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Coverband
+  module Collectors
+    ###
+    # Registry of the trackers Coverband initializes at Rails boot.
+    #
+    # Bundled trackers register themselves as their file is required. Apps and
+    # gems register their own trackers the same way, which is all Coverband
+    # needs to build them, hook them up, and report on them.
+    ###
+    class TrackerRegistry
+      Entry = Struct.new(:name, :tracker_class, :enabled_proc, keyword_init: true)
+
+      def self.registry
+        @registry ||= {}
+      end
+
+      def self.register(name, tracker_class:, enabled:)
+        name = name.to_sym
+        raise ArgumentError, "Tracker already registered: #{name}" if registry.key?(name)
+
+        # a callable can only be checked once it has built its tracker at boot
+        validate_report_route!(name, tracker_class) unless tracker_class.respond_to?(:call)
+
+        registry[name] = Entry.new(
+          name: name,
+          tracker_class: tracker_class,
+          enabled_proc: enabled
+        )
+      end
+
+      ###
+      # A tracker that inherits AbstractTracker's default REPORT_ROUTE matches
+      # every path in the web reporter, which hides the rest of the report.
+      # Fail loudly rather than let a tracker take over the whole UI.
+      ###
+      def self.validate_report_route!(name, tracker_class)
+        return unless tracker_class.is_a?(Class)
+        return unless tracker_class < Coverband::Collectors::AbstractTracker
+
+        default_route = Coverband::Collectors::AbstractTracker::REPORT_ROUTE
+        return unless default_route == tracker_class::REPORT_ROUTE
+
+        raise ArgumentError, "Tracker #{name} must define its own REPORT_ROUTE, " \
+          "AbstractTracker's default #{default_route.inspect} matches every web report path"
+      end
+
+      def self.each(&block)
+        registry.each_value(&block)
+      end
+
+      def self.names
+        registry.keys
+      end
+
+      # intended for tests, resetting the registry of a booted app is not supported
+      def self.reset!
+        @registry = {}
+      end
+    end
+  end
+end

data/lib/coverband/collectors/translation_tracker.rb

@@ -58,3 +58,9 @@ module Coverband
     end
   end
 end
+
+Coverband::Collectors::TrackerRegistry.register(
+  :translations_tracker,
+  tracker_class: Coverband::Collectors::TranslationTracker,
+  enabled: ->(config) { config.track_translations }
+)

data/lib/coverband/collectors/view_tracker.rb

@@ -68,11 +68,10 @@ module Coverband
       end
 
       def used_keys
-        return {} unless redis_store
+        return {} unless storage
 
-        views = redis_store.hgetall(tracker_key)
         normalized_views = {}
-        views.each_pair do |view, time|
+        storage.entries.each_pair do |view, time|
           normalized_view = normalize_path(view)
           normalized_views[normalized_view] = time
         end
@@ -102,11 +101,12 @@ module Coverband
 
       def clear_key!(filename)
         return unless filename
-        return unless redis_store
+        return unless storage
 
         filename = "#{@project_directory}/#{filename}"
-        redis_store.hdel(tracker_key, filename)
+        storage.delete_entry(filename)
         @logged_keys.delete(filename)
+        @keys_to_record.delete(filename)
       end
 
       private
@@ -144,3 +144,9 @@ module Coverband
     end
   end
 end
+
+Coverband::Collectors::TrackerRegistry.register(
+  :view_tracker,
+  tracker_class: Coverband::Collectors::ViewTracker,
+  enabled: ->(config) { config.track_views }
+)