coverband 6.2.2 → 7.0.0.rc.2

data/lib/coverband/adapters/session_coverage.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require_relative "../storage/session"
+
+module Coverband
+  module Adapters
+    ###
+    # Coverage on top of the merge protocol, shared by the adapters that keep it
+    # in one document per type.
+    #
+    # Counts are additive, so a dropped write cannot be repaired by writing
+    # again: re-applying a sum double counts. The per writer sequences in
+    # Storage::Session are what make the retry safe, which is why both adapters
+    # go through here rather than each merging on their own.
+    #
+    # Including classes provide #storage_target and #coverage_key_base.
+    ###
+    module SessionCoverage
+      ###
+      # Coverage deltas are large, so they get a tighter bound than the trackers
+      # do. The cap counts the queue, not missed cycles: the cycle that recovers
+      # carries its own delta, so it takes a slot too, and the outage a store
+      # can absorb is one less than this.
+      ###
+      COVERAGE_MAX_ENTRIES = 3
+
+      def persistent_coverage?
+        true
+      end
+
+      # the pointers this store will consult, so a cycle can fetch them in one
+      # round trip instead of one small read per document
+      def pointer_sessions
+        Coverband::TYPES.map { |session_type| build_session(session_type) }
+      end
+
+      def prefetch_pointers!(extra_sessions = [])
+        Storage::Session.prefetch_pointers(storage_target, pointer_sessions + Array(extra_sessions))
+      end
+
+      # whether every type actually cleared: a reset whose pointer write did not
+      # land leaves the old generation authoritative, and callers have to be able
+      # to say so rather than report success
+      def clear!
+        results = Coverband::TYPES.map { |type| session_for(type).reset }
+        @cached_file_count = nil
+        results.all?
+      end
+
+      def clear_file!(filename)
+        relative = Utils::RelativeFileConverter.convert(filename)
+        Coverband::TYPES.each do |type|
+          session_for(type).delete_entry(relative)
+          session_for(type).delete_entry(filename)
+        end
+        @cached_file_count = nil
+      end
+
+      def size
+        session_for(type).stored_size
+      end
+
+      def type=(type)
+        super
+        @cached_file_count = nil
+      end
+
+      def coverage(local_type = nil, opts = {})
+        local_type ||= opts.key?(:override_type) ? opts[:override_type] : type
+        data = session_for(local_type).entries.dup
+        unless opts[:skip_hash_check]
+          data.delete_if { |file_path, file_data| file_hash(file_path) != file_data["file_hash"] }
+        end
+        data
+      end
+
+      # expanded here once: re-expanding on a retry would hand an old delta a
+      # fresh timestamp and let it slip past a tombstone recorded in between
+      # returns Session#record's protocol state, so a caller can tell a write
+      # that landed from one that was deferred or refused
+      def save_report(report)
+        result = session_for(type).record(own_expanded_report(report))
+        @cached_file_count = nil
+        result
+      end
+
+      def save_coverage(report, local_type = nil)
+        session_for(local_type || type).record(own_expanded_report(report))
+      end
+
+      def file_count
+        coverage(Coverband::RUNTIME_TYPE, skip_hash_check: true).keys.length
+      end
+
+      def data_loss
+        Coverband::TYPES.map { |type| session_for(type).data_loss }.compact.first
+      end
+
+      # work held because it cannot be written, which for a document that can
+      # never be written is the only signal there is
+      def unwritten
+        Coverband::TYPES.map { |type| session_for(type).unwritten }.compact.first
+      end
+
+      # gives up the repair those held reports would have done, in exchange for
+      # the memory; benchmarks and shutdown only
+      def discard_pending!
+        Coverband::TYPES.each { |type| session_for(type).discard_pending! }
+      end
+
+      private
+
+      ###
+      # A delta has to survive being applied, since re-applying it is how a lost
+      # update is repaired. expand_report hands over the caller's own line arrays
+      # and array_add sums into them in place unless frozen, so an unfrozen
+      # payload is overwritten with the merged document total on its first apply
+      # -- and a repair after that carries the whole document back in, doubling
+      # it once per repair. Freezing selects array_add's copying branch.
+      ###
+      def own_expanded_report(report)
+        expand_report(report).each_value do |entry|
+          entry[Base::DATA_KEY] = entry[Base::DATA_KEY].dup.freeze
+          entry.freeze
+        end
+      end
+
+      def sessions
+        @sessions ||= {}
+      end
+
+      # both types are built together, or reporting one would allocate the
+      # other's keys later at an unpredictable moment
+      def session_for(local_type)
+        local_type ||= type
+        Coverband::TYPES.each { |session_type| build_session(session_type) }
+        build_session(local_type)
+      end
+
+      def build_session(local_type)
+        sessions[local_type] ||= Storage::Session.new(
+          target: storage_target,
+          key_base: coverage_key_base(local_type),
+          merger: coverage_merger,
+          logger: Coverband.configuration.logger,
+          max_entries: COVERAGE_MAX_ENTRIES,
+          grace_seconds: grace_seconds,
+          # Delta has already advanced its baseline by the time a report is
+          # handed over, so nothing else is holding this cycle's work
+          retain_on_failure: true
+        )
+      end
+
+      # line hits sum, so this merge is not idempotent and the applied sequence
+      # guard is what keeps a retry from inflating the counts
+      def coverage_merger
+        @coverage_merger ||= lambda do |doc, delta|
+          incoming = delta.payload.reject do |file, _data|
+            doc.tombstoned?(file, delta.tombstone_epoch)
+          end
+          doc.payload.replace(merge_reports(incoming.dup, doc.payload, skip_expansion: true))
+        end
+      end
+
+      def grace_seconds
+        Coverband.configuration.background_reporting_sleep_seconds * 2
+      rescue
+        1200
+      end
+    end
+  end
+end

data/lib/coverband/adapters/stdout_store.rb

@@ -28,10 +28,6 @@ module Coverband
       def save_report(report)
         $stdout.puts(report.to_json)
       end
-
-      def raw_store
-        raise NotImplementedError, "StdoutStore doesn't support raw_store"
-      end
     end
   end
 end

data/lib/coverband/adapters/tracker_storage/base.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Coverband
+  module Adapters
+    module TrackerStorage
+      ###
+      # What a tracker sees. Physical keys, generations, serialization, and the
+      # applied sequence protocol all live below this line; trackers deal in
+      # keys and timestamps.
+      #
+      # record returns a protocol state, not an outcome: "written" and "proven
+      # durable" differ, and that difference is the point of the watermark.
+      #
+      #   :written_unconfirmed  merged and written, still awaiting confirmation
+      #   :confirmed            an earlier write was proven durable and dropped
+      #   :deferred             enqueued only, nothing flushed this call
+      #   :retained             taken, but not durable yet; still ours to retry
+      #   :failed               refused before we took it, so it is yours again
+      #   :unavailable          the backend cannot be reached at all, same
+      #
+      # Only the last two mean the caller still owns the work. Keeping a copy
+      # against any of the others gives the same delta two owners, and both
+      # replay it.
+      ###
+      class Base
+        ABSTRACT_KEY = "abstract"
+
+        def entries
+          raise ABSTRACT_KEY
+        end
+
+        def record(_delta)
+          raise ABSTRACT_KEY
+        end
+
+        def delete_entry(_key)
+          raise ABSTRACT_KEY
+        end
+
+        def reset
+          raise ABSTRACT_KEY
+        end
+
+        def tracking_since
+          raise ABSTRACT_KEY
+        end
+
+        def generation
+          raise ABSTRACT_KEY
+        end
+
+        def data_loss
+          nil
+        end
+
+        # work held because it could not be written; a layout with no queue of
+        # its own has none to report
+        def unwritten
+          nil
+        end
+
+        def pending_size
+          0
+        end
+
+        # the generation pointer holder, when this repository has one to batch
+        def pointer_session
+          nil
+        end
+
+        # keys another process deleted since the last read, so trackers can drop
+        # them from their permanent local dedupe and record them again if used
+        def newly_tombstoned
+          []
+        end
+      end
+    end
+  end
+end

data/lib/coverband/adapters/tracker_storage/document_repository.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require_relative "base"
+require_relative "../../storage/session"
+
+module Coverband
+  module Adapters
+    module TrackerStorage
+      ###
+      # A tracker stored as one document: metadata and payload written together
+      # and reverted together. Every cache backed tracker, plus the additive ones
+      # on Redis. The tracker supplies the merge, since only it knows whether two
+      # values combine by later timestamp or by summing.
+      ###
+      class DocumentRepository < Base
+        extend Forwardable
+
+        # the session owns the protocol; this decides what a tracker may ask for
+        def_delegators :@session, :record, :delete_entry, :reset, :tracking_since,
+          :data_loss, :unwritten, :pending_size, :newly_tombstoned
+        def_delegator :@session, :generation_token, :generation
+
+        def initialize(target:, key_base:, merger:, logger: nil, on_generation_change: nil, session_options: {})
+          @session = Storage::Session.new(
+            target: target,
+            key_base: key_base,
+            merger: merger,
+            logger: logger,
+            on_generation_change: on_generation_change,
+            **session_options
+          )
+        end
+
+        def entries
+          # Redis hgetall hands back strings and JSON parsing does not, so the
+          # two layouts would otherwise not be interchangeable for callers
+          @session.entries.each_with_object({}) do |(key, value), normalized|
+            normalized[key.to_s] = value.is_a?(String) ? value : value.to_s
+          end
+        end
+
+        def pointer_session
+          @session
+        end
+      end
+    end
+  end
+end

data/lib/coverband/adapters/tracker_storage/factory.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require_relative "document_repository"
+require_relative "redis_hash_repository"
+require_relative "../../storage/redis_target"
+
+module Coverband
+  module Adapters
+    module TrackerStorage
+      ###
+      # Builds tracker repositories, choosing the layout by merge semantics
+      # rather than by backend.
+      #
+      # Idempotent presence trackers keep native Redis hashes where the backend
+      # has them: a field written by one process is never reverted by another's
+      # write, which no whole document protocol can match.
+      #
+      # Everything else is a document. Additive trackers can never use the hash,
+      # because a hash co-writes metadata without co-reverting it: a stale
+      # multi-field HSET leaves another writer's field in place while wiping the
+      # metadata that recorded it, so that writer applies its delta twice. On a
+      # cache, idempotent trackers are documents too, and still need applied
+      # sequences -- not for retry safety, but to notice a stale write erased a
+      # key, which a permanent local dedupe would otherwise never re-enqueue.
+      ###
+      class Factory
+        def self.redis(redis:, namespace:, format_version:)
+          new(target: Storage::RedisTarget.new(redis), namespace: namespace,
+            format_version: format_version, native_hashes: true)
+        end
+
+        def self.cache(target:, namespace:, format_version:)
+          new(target: target, namespace: namespace, format_version: format_version)
+        end
+
+        def initialize(target:, namespace:, format_version:, native_hashes: false)
+          @target = target
+          @namespace = namespace
+          @format_version = format_version
+          @native_hashes = native_hashes
+        end
+
+        def for(name, merger:, idempotent: true, logger: nil, on_generation_change: nil)
+          key_base = [@format_version, @namespace, "tracker", name].compact.join(".")
+
+          if @native_hashes && idempotent
+            RedisHashRepository.new(target: @target, key_base: key_base,
+              logger: logger, on_generation_change: on_generation_change)
+          else
+            DocumentRepository.new(target: @target, key_base: key_base, merger: merger,
+              logger: logger, on_generation_change: on_generation_change)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/coverband/adapters/tracker_storage/redis_hash_repository.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require_relative "../../storage/generation_coordinator"
+require_relative "../../storage/read_fallback"
+
+module Coverband
+  module Adapters
+    module TrackerStorage
+      ###
+      # Presence trackers on Redis, kept on native hash operations.
+      #
+      # Their merge is max(timestamp), so re-applying a field is a no-op and a
+      # field written by one process is never reverted by another's write --
+      # strictly stronger than any whole document protocol, so these keep HSET
+      # and HDEL. What HDEL cannot do is stop a delete being undone by an HSET
+      # queued before it, which is why single key clears are best effort.
+      ###
+      class RedisHashRepository < Base
+        include Storage::ReadFallback
+
+        STARTED_AT_FIELD = "__coverband_started_at__"
+
+        def initialize(target:, key_base:, logger: nil, grace_seconds: 1200, on_generation_change: nil)
+          @target = target
+          @logger = logger
+          @on_generation_change = on_generation_change
+          @generation_coordinator = Storage::GenerationCoordinator.new(
+            target: target,
+            key_base: key_base,
+            grace_seconds: grace_seconds,
+            on_change: ->(change) { handle_generation_change(change) }
+          )
+        end
+
+        def pointer_session
+          self
+        end
+
+        # no pending deltas here, but a primed pointer is still cycle state
+        def discard_pending!
+          @generation_coordinator.discard_primed_pointer!
+        end
+
+        def pointer_key
+          @generation_coordinator.pointer_key
+        end
+
+        def prime_pointer(pointer)
+          @generation_coordinator.prime_pointer(pointer)
+        end
+
+        def generation
+          safely { @generation_coordinator.token }
+        end
+
+        def entries
+          safely({}) do
+            operation do
+              @target.hgetall(data_key).reject { |field, _value| field == STARTED_AT_FIELD }
+            end
+          end
+        end
+
+        def record(delta)
+          return :deferred if delta.empty?
+
+          operation do
+            values = delta.each_with_object({}) { |(key, value), hash| hash[key.to_s] = value.to_s }
+            values[STARTED_AT_FIELD] = Time.now.to_i.to_s unless started_at_set?
+            @target.hset(data_key, values)
+            @started_at_set = true
+            :written_unconfirmed
+          end
+        rescue => e
+          @logger&.error("Coverband: #{self.class.name} failed to store, error #{e.class.name} info #{e.message}")
+          :failed
+        end
+
+        # best effort: a key observed by another process just before the delete
+        # can come back with no post-delete observation behind it
+        def delete_entry(key)
+          operation do
+            @target.hdel(data_key, key.to_s)
+            :written_unconfirmed
+          end
+        end
+
+        def reset
+          @generation_coordinator.reset
+        end
+
+        def tracking_since
+          safely do
+            operation do
+              value = @target.hgetall(data_key)[STARTED_AT_FIELD]
+              value ? Time.at(value.to_i) : nil
+            end
+          end
+        end
+
+        private
+
+        def started_at_set?
+          @started_at_set ||= !@target.hgetall(data_key)[STARTED_AT_FIELD].nil?
+        end
+
+        def handle_generation_change(change)
+          return unless [Storage::GenerationChange::OPERATOR_RESET,
+            Storage::GenerationChange::POINTER_EVICTION,
+            Storage::GenerationChange::INITIALIZATION_RACE].include?(change.cause)
+
+          @started_at_set = false
+          @on_generation_change&.call(change)
+        end
+
+        def operation(&block)
+          @generation_coordinator.operation(&block)
+        end
+
+        def data_key
+          @generation_coordinator.data_key
+        end
+      end
+    end
+  end
+end