igniter 0.4.5 → 0.5.1

data/packages/igniter-cluster/lib/igniter/cluster/mesh/router.rb

@@ -0,0 +1,306 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Cluster
+    module Mesh
+    require_relative "../replication/capability_query"
+
+    # Thread-safe capability router with a short-lived health cache.
+    #
+    # Resolves the URL to call for a given routing mode:
+    #   - :capability  → round-robin among alive peers that advertise the capability;
+    #                    raises DeferredCapabilityError when no alive peer is found.
+    #   - :pinned      → asserts the named peer is alive and returns its URL;
+    #                    raises IncidentError when the peer is unknown or unreachable.
+    #
+    # Peer pool = static peers (Config#peers) + dynamic peers (Config#peer_registry).
+    # Static peers take precedence when a name appears in both sets.
+    class Router
+      HEALTH_CACHE_TTL = 5 # seconds
+
+      def initialize(config)
+        @config       = config
+        @health_cache = {}
+        @mutex        = Mutex.new
+        @round_robin  = Hash.new(0)
+      end
+
+      # Find an alive peer advertising +capability+.
+      # Returns the peer URL. Raises DeferredCapabilityError when none are alive.
+      def find_peer_for(capability, deferred_result)
+        find_peer_for_query({ all_of: [capability.to_sym] }, deferred_result, round_robin_key: capability.to_sym)
+      end
+
+      # Return a structured explanation of how capability routing would resolve.
+      # Does not mutate round-robin state.
+      def explain_peer_for(capability)
+        explain_peer_for_query({ all_of: [capability.to_sym] }, round_robin_key: capability.to_sym)
+      end
+
+      # Find an alive peer matching a full capability query.
+      # Returns the peer URL. Raises DeferredCapabilityError when none are alive.
+      def find_peer_for_query(query, deferred_result, round_robin_key: nil)
+        normalized = Igniter::Cluster::Replication::CapabilityQuery.normalize(query)
+        candidates = all_matching_peers(normalized).select { |peer| alive?(peer) }
+
+        if candidates.empty?
+          raise DeferredCapabilityError.new(
+            normalized.name || normalized.all_of.first,
+            deferred_result,
+            query: normalized.to_h,
+            explanation: explain_peer_for_query(normalized, round_robin_key: round_robin_key)
+          )
+        end
+
+        url_for_round_robin(round_robin_key || normalized.to_h, top_ranked_candidates(normalized, candidates))
+      end
+
+      # Return a structured explanation of how a full capability query resolves.
+      # Does not mutate round-robin state.
+      def explain_peer_for_query(query, round_robin_key: nil)
+        normalized = Igniter::Cluster::Replication::CapabilityQuery.normalize(query)
+        peers = all_known_peers
+        evaluations = peers.map { |peer| evaluate_peer(normalized, peer) }
+        eligible_evaluations = evaluations.select { |entry| entry[:eligible] }
+        top_tier = top_ranked_candidates(normalized, eligible_evaluations.map { |entry| entry[:peer] })
+        top_tier_names = top_tier.map(&:name)
+        selected_peer, round_robin_index = preview_round_robin_candidate(round_robin_key || normalized.to_h, top_tier)
+
+        {
+          routing_mode: :capability,
+          query: normalized.to_h,
+          peer_count: peers.size,
+          matched_count: evaluations.count { |entry| entry[:matched] },
+          eligible_count: eligible_evaluations.size,
+          top_tier_count: top_tier.size,
+          selected_url: selected_peer&.url,
+          selected_peer: selected_peer&.name,
+          round_robin_index: round_robin_index,
+          peers: evaluations.map do |entry|
+            peer = entry[:peer]
+            {
+              name: peer.name,
+              url: peer.url,
+              matched: entry[:matched],
+              alive: entry[:alive],
+              eligible: entry[:eligible],
+              match_details: entry[:match_details],
+              top_tier: top_tier_names.include?(peer.name),
+              selected: selected_peer&.name == peer.name,
+              ranking_fingerprint: entry[:ranking_fingerprint],
+              reasons: evaluation_reasons(entry, top_tier_names, selected_peer)
+            }
+          end
+        }
+      end
+
+      # Resolve the URL of a pinned peer by name.
+      # Raises IncidentError if the peer is unknown or unreachable.
+      def resolve_pinned(peer_name)
+        peer = find_named_peer(peer_name)
+
+        unless peer
+          raise IncidentError.new(
+            peer_name,
+            "Pinned peer '#{peer_name}' is not registered in Igniter::Cluster::Mesh",
+            context: { routing_trace: explain_pinned(peer_name) }
+          )
+        end
+
+        unless alive?(peer)
+          raise IncidentError.new(
+            peer_name,
+            nil,
+            context: { routing_trace: explain_pinned(peer_name) }
+          )
+        end
+
+        peer.url
+      end
+
+      # Return a structured explanation of pinned routing resolution.
+      def explain_pinned(peer_name)
+        peer = find_named_peer(peer_name)
+
+        {
+          routing_mode: :pinned,
+          peer_name: peer_name.to_s,
+          known: !peer.nil?,
+          selected_url: peer&.url,
+          reachable: peer ? alive?(peer) : false,
+          reasons: pinned_reasons(peer)
+        }
+      end
+
+      # Expire a peer's cached health status (e.g., after a successful or failed request).
+      def invalidate_health!(url)
+        @mutex.synchronize { @health_cache.delete(url) }
+      end
+
+      private
+
+      # Combined static + dynamic peer pool for capability-query lookup.
+      # Static peers take precedence over same-named dynamic peers.
+      def all_matching_peers(query)
+        merge_peers(
+          @config.peers_matching_query(query),
+          @config.peer_registry.peers_matching_query(query)
+        )
+      end
+
+      def all_known_peers
+        merge_peers(@config.peers, @config.peer_registry.all)
+      end
+
+      # Lookup peer by name across static and dynamic pools.
+      def find_named_peer(name)
+        @config.peer_named(name) || @config.peer_registry.peer_named(name)
+      end
+
+      # Merge static and dynamic peer lists; static names win on collision.
+      def merge_peers(static, dynamic)
+        seen = static.each_with_object({}) { |p, h| h[p.name] = true }
+        static + dynamic.reject { |p| seen[p.name] }
+      end
+
+      def url_for_round_robin(capability, candidates)
+        peer, = pick_round_robin_candidate(capability, candidates)
+        peer.url
+      end
+
+      def top_ranked_candidates(query, candidates)
+        return candidates if candidates.empty?
+        return trust_ranked_candidates(candidates) unless query.ordered? || query.decisioned?
+
+        ranked = candidates.sort do |left, right|
+          comparison = query.compare_profiles(left.profile, right.profile)
+          next comparison unless comparison.zero?
+
+          compare_trust_profiles(left.profile, right.profile)
+        end
+        best_fingerprint = routing_fingerprint(query, ranked.first.profile)
+        ranked.select { |peer| routing_fingerprint(query, peer.profile) == best_fingerprint }
+      end
+
+      def evaluate_peer(query, peer)
+        match_details = query.explain_profile(peer.profile)
+        matched = match_details[:matched]
+        alive = matched ? alive?(peer) : nil
+        eligible = matched && alive
+
+        {
+          peer: peer,
+          matched: matched,
+          alive: alive,
+          eligible: eligible,
+          match_details: match_details,
+          ranking_fingerprint: eligible ? routing_fingerprint(query, peer.profile) : nil
+        }
+      end
+
+      def routing_fingerprint(query, profile)
+        Array(query.ranking_fingerprint(profile)) + trust_fingerprint(profile)
+      end
+
+      def trust_ranked_candidates(candidates)
+        ranked = candidates.sort do |left, right|
+          compare_trust_profiles(left.profile, right.profile)
+        end
+        best_fingerprint = trust_fingerprint(ranked.first.profile)
+        ranked.select { |peer| trust_fingerprint(peer.profile) == best_fingerprint }
+      end
+
+      def compare_trust_profiles(left, right)
+        left_fingerprint = trust_fingerprint(left)
+        right_fingerprint = trust_fingerprint(right)
+
+        left_fingerprint <=> right_fingerprint
+      end
+
+      def trust_fingerprint(profile)
+        metadata = profile.metadata || {}
+        identity_trust = metadata.dig(:mesh_trust, :status)
+        attestation_trust = metadata.dig(:mesh_capabilities, :trust, :status)
+        attestation_freshness = metadata.dig(:mesh_capabilities, :freshness_seconds)
+        governance_trust = metadata.dig(:mesh_governance, :trust, :status)
+        governance_freshness = metadata.dig(:mesh_governance, :freshness_seconds)
+        governance_blocked_events = metadata.dig(:mesh_governance, :blocked_events)
+        governance_applied_events = metadata.dig(:mesh_governance, :applied_events)
+
+        [
+          trust_priority(identity_trust, missing_rank: 1, invalid_rank: 2),
+          trust_priority(attestation_trust, missing_rank: 2, invalid_rank: 3),
+          attestation_freshness.nil? ? 1_000_000_000 : attestation_freshness.to_i,
+          trust_priority(governance_trust, missing_rank: 2, invalid_rank: 3),
+          governance_freshness.nil? ? 1_000_000_000 : governance_freshness.to_i,
+          governance_blocked_events.nil? ? 1_000_000_000 : governance_blocked_events.to_i,
+          governance_applied_events.nil? ? 1_000_000_000 : -governance_applied_events.to_i
+        ]
+      end
+
+      def trust_priority(status, missing_rank:, invalid_rank:)
+        case status&.to_sym
+        when :trusted
+          0
+        when nil, :unknown
+          missing_rank
+        else
+          invalid_rank
+        end
+      end
+
+      def evaluation_reasons(entry, top_tier_names, selected_peer)
+        return [:query_mismatch] unless entry[:matched]
+        return [:unreachable] unless entry[:alive]
+        return [:lower_ranked] if entry[:eligible] && !top_tier_names.include?(entry[:peer].name)
+        return [:selected] if selected_peer&.name == entry[:peer].name
+
+        [:not_selected_this_turn]
+      end
+
+      def pick_round_robin_candidate(key, candidates)
+        idx = @mutex.synchronize do
+          i = @round_robin[key] % candidates.size
+          @round_robin[key] = i + 1
+          i
+        end
+        [candidates[idx], idx]
+      end
+
+      def preview_round_robin_candidate(key, candidates)
+        return [nil, nil] if candidates.empty?
+
+        idx = @mutex.synchronize { @round_robin[key] % candidates.size }
+        [candidates[idx], idx]
+      end
+
+      def pinned_reasons(peer)
+        return [:unknown_peer] unless peer
+        return [:unreachable] unless alive?(peer)
+
+        [:selected]
+      end
+
+      def alive?(peer) # rubocop:disable Metrics/MethodLength
+        @mutex.synchronize do
+          entry = @health_cache[peer.url]
+          return entry[:alive] if entry && (Time.now.utc - entry[:checked_at]) < HEALTH_CACHE_TTL
+        end
+
+        alive = begin
+          Igniter::Server::Client.new(peer.url, timeout: 3).health
+          true
+        rescue Igniter::Server::Client::ConnectionError
+          false
+        end
+
+        @mutex.synchronize do
+          @health_cache[peer.url] = { alive: alive, checked_at: Time.now.utc }
+        end
+
+        alive
+      end
+    end
+    end
+  end
+end

data/packages/igniter-cluster/lib/igniter/cluster/mesh/workload_signal.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Igniter
+  module Cluster
+    module Mesh
+      # A single recorded runtime event for one peer-capability interaction.
+      # Immutable — produced by WorkloadTracker#record.
+      WorkloadSignal = ::Data.define(
+        :peer_name,
+        :capability,
+        :success,
+        :duration_ms,
+        :error_class,
+        :recorded_at
+      ) do
+        def self.build(peer_name:, capability:, success:, duration_ms: nil, error: nil, recorded_at: Time.now.utc.iso8601)
+          new(
+            peer_name:   peer_name.to_s,
+            capability:  capability&.to_sym,
+            success:     success ? true : false,
+            duration_ms: duration_ms&.to_f,
+            error_class: error ? error.class.name : nil,
+            recorded_at: recorded_at.to_s
+          )
+        end
+
+        def failure?
+          !success
+        end
+
+        def to_h
+          {
+            peer_name:   peer_name,
+            capability:  capability,
+            success:     success,
+            duration_ms: duration_ms,
+            error_class: error_class,
+            recorded_at: recorded_at
+          }.compact
+        end
+      end
+    end
+  end
+end

data/packages/igniter-cluster/lib/igniter/cluster/mesh/workload_tracker.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Cluster
+    module Mesh
+      # Thread-safe accumulator of runtime workload signals across all peers.
+      #
+      # Each peer-capability pair maintains a sliding window of the most recent
+      # `window_size` WorkloadSignal records. Older signals are dropped when the
+      # window overflows, keeping memory bounded.
+      #
+      # Usage:
+      #   tracker = WorkloadTracker.new(window_size: 100)
+      #   tracker.record("node-a", :database, success: true, duration_ms: 42)
+      #   tracker.record("node-a", :database, success: false, error: e)
+      #
+      #   report = tracker.report_for("node-a")
+      #   report.failure_rate   # => 0.5
+      #   report.degraded?      # => false (threshold is 0.3 by default)
+      #
+      #   tracker.degraded_peers   # => ["node-a"] when any peer exceeds threshold
+      #   tracker.reset_peer!("node-a")  # clear signals for a specific peer
+      class WorkloadTracker
+        DEFAULT_WINDOW_SIZE          = 200
+        DEFAULT_DEGRADED_THRESHOLD   = 0.3   # failure rate above which peer is degraded
+        DEFAULT_OVERLOAD_THRESHOLD_MS = 1000  # avg ms above which peer is overloaded
+
+        attr_reader :window_size, :degraded_threshold, :overload_threshold_ms
+
+        def initialize(window_size: DEFAULT_WINDOW_SIZE,
+                       degraded_threshold: DEFAULT_DEGRADED_THRESHOLD,
+                       overload_threshold_ms: DEFAULT_OVERLOAD_THRESHOLD_MS)
+          @window_size           = window_size
+          @degraded_threshold    = degraded_threshold.to_f
+          @overload_threshold_ms = overload_threshold_ms.to_f
+          @windows               = Hash.new { |h, k| h[k] = [] }
+          @mutex                 = Mutex.new
+        end
+
+        # Record a single workload signal for a peer-capability pair.
+        #
+        # @param peer_name   [String]
+        # @param capability  [Symbol, nil]
+        # @param success     [Boolean]
+        # @param duration_ms [Numeric, nil]
+        # @param error       [Exception, nil]
+        # @return [WorkloadSignal]
+        def record(peer_name, capability = nil, success:, duration_ms: nil, error: nil)
+          signal = WorkloadSignal.build(
+            peer_name:   peer_name,
+            capability:  capability,
+            success:     success,
+            duration_ms: duration_ms,
+            error:       error
+          )
+          @mutex.synchronize do
+            key = window_key(peer_name, capability)
+            window = @windows[key]
+            window << signal
+            window.shift if window.size > @window_size
+          end
+          signal
+        end
+
+        # Compute and return a capacity report for a single peer (all capabilities combined).
+        #
+        # @param peer_name [String]
+        # @return [PeerCapacityReport]
+        def report_for(peer_name)
+          signals   = signals_for_peer(peer_name)
+          caps      = capabilities_seen_for(peer_name)
+          build_report(peer_name, signals, capabilities: caps)
+        end
+
+        # All signals recorded for a peer across all its capabilities.
+        #
+        # @return [Array<WorkloadSignal>]
+        def signals_for_peer(peer_name)
+          @mutex.synchronize do
+            @windows
+              .select { |k, _| k.start_with?("#{peer_name}:") || k == peer_name.to_s }
+              .values.flatten
+          end
+        end
+
+        # Capacity report for a specific peer+capability pair.
+        #
+        # @return [PeerCapacityReport]
+        def report_for_capability(peer_name, capability)
+          key     = window_key(peer_name, capability)
+          signals = @mutex.synchronize { @windows[key].dup }
+          build_report(peer_name, signals, capabilities: [capability&.to_sym].compact)
+        end
+
+        # Aggregated reports for every peer seen so far (snapshot).
+        #
+        # @return [Hash<String, PeerCapacityReport>]
+        def all_reports
+          peer_names = @mutex.synchronize do
+            @windows.keys.map { |k| k.split(":").first }.uniq
+          end
+          peer_names.each_with_object({}) { |name, h| h[name] = report_for(name) }
+        end
+
+        # Peers whose failure_rate exceeds `degraded_threshold`.
+        #
+        # @param threshold [Float, nil]  override for this call
+        # @return [Array<String>]
+        def degraded_peers(threshold: nil)
+          t = (threshold || @degraded_threshold).to_f
+          all_reports.select { |_, r| r.failure_rate >= t }.keys
+        end
+
+        # Peers whose avg_duration_ms exceeds `overload_threshold_ms`.
+        #
+        # @param threshold_ms [Numeric, nil]  override for this call
+        # @return [Array<String>]
+        def overloaded_peers(threshold_ms: nil)
+          t = (threshold_ms || @overload_threshold_ms).to_f
+          all_reports.select { |_, r| r.avg_duration_ms && r.avg_duration_ms >= t }.keys
+        end
+
+        # All peer names with recorded signals.
+        #
+        # @return [Array<String>]
+        def known_peers
+          @mutex.synchronize do
+            @windows.keys.map { |k| k.split(":").first }.uniq
+          end
+        end
+
+        # Clear signals for a specific peer (e.g. after repair).
+        #
+        # @param peer_name [String]
+        # @return [self]
+        def reset_peer!(peer_name)
+          @mutex.synchronize do
+            @windows.reject! { |k, _| k.start_with?("#{peer_name}:") || k == peer_name.to_s }
+          end
+          self
+        end
+
+        # Clear all signals.
+        #
+        # @return [self]
+        def reset!
+          @mutex.synchronize { @windows.clear }
+          self
+        end
+
+        def total_signals
+          @mutex.synchronize { @windows.values.sum(&:size) }
+        end
+
+        # Build the mesh_workload metadata hash for a peer, suitable for merging
+        # into a NodeObservation's metadata. Returns nil if no signals recorded.
+        #
+        # @param peer_name [String]
+        # @return [Hash, nil]
+        def to_metadata_for(peer_name)
+          report = report_for(peer_name)
+          return nil if report.total.zero?
+
+          {
+            mesh_workload: {
+              failure_rate:    report.failure_rate,
+              avg_duration_ms: report.avg_duration_ms,
+              total:           report.total,
+              degraded:        report.degraded?,
+              overloaded:      report.overloaded?
+            }.compact
+          }
+        end
+
+        private
+
+        def window_key(peer_name, capability)
+          capability ? "#{peer_name}:#{capability}" : peer_name.to_s
+        end
+
+        def capabilities_seen_for(peer_name)
+          @mutex.synchronize do
+            @windows
+              .keys
+              .select { |k| k.start_with?("#{peer_name}:") }
+              .map    { |k| k.split(":", 2).last.to_sym }
+          end
+        end
+
+        def build_report(peer_name, signals, capabilities:)
+          total     = signals.size
+          successes = signals.count(&:success)
+          failures  = total - successes
+
+          failure_rate = total.zero? ? 0.0 : failures.to_f / total
+
+          durations    = signals.filter_map(&:duration_ms)
+          avg_ms       = durations.empty? ? nil : durations.sum / durations.size
+
+          PeerCapacityReport.new(
+            peer_name:       peer_name.to_s,
+            total:           total,
+            successes:       successes,
+            failures:        failures,
+            failure_rate:    failure_rate,
+            avg_duration_ms: avg_ms,
+            degraded:        failure_rate >= @degraded_threshold,
+            overloaded:      avg_ms && avg_ms >= @overload_threshold_ms,
+            capabilities:    capabilities
+          )
+        end
+      end
+    end
+  end
+end