igniter 0.4.5 → 0.5.1

data/packages/igniter-cluster/lib/igniter/cluster/mesh/poller.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Cluster
+    module Mesh
+    # Background thread that periodically fetches the peer list from every seed
+    # and populates the local PeerRegistry with newly discovered peers.
+    #
+    # - Errors from individual seeds are swallowed (seed may be temporarily down).
+    # - Does not remove peers from the registry on failure — the health cache in
+    #   Router handles routing around dead peers without purging known topology.
+    # - Thread is non-daemon: call #stop explicitly on shutdown (Discovery does this).
+    class Poller
+      def initialize(config)
+        @config  = config
+        @running = false
+        @thread  = nil
+        @mutex   = Mutex.new
+      end
+
+      # Start the background polling thread. Idempotent.
+      def start
+        @mutex.synchronize do
+          return if @running
+
+          @running = true
+          @thread  = Thread.new { run_loop }
+          @thread.abort_on_exception = false
+        end
+      end
+
+      # Stop the background thread. Idempotent.
+      def stop
+        @mutex.synchronize do
+          @running = false
+          @thread&.kill
+          @thread = nil
+        end
+      end
+
+      def running?
+        @mutex.synchronize { @running }
+      end
+
+      # Fetch peers from all seeds, then run a gossip round with random registry
+      # peers (Phase 3). Synchronous — used at startup and inside the background loop.
+      def poll_once
+        @config.seeds.each { |url| fetch_peers_from(url) }
+        GossipRound.new(@config).run if @config.gossip_fanout.positive?
+      end
+
+      private
+
+      def run_loop
+        loop do
+          sleep(@config.discovery_interval)
+          break unless @running
+
+          poll_once
+        end
+      end
+
+      def fetch_peers_from(seed_url)
+        peers = Igniter::Server::Client.new(seed_url, timeout: 5).list_peers
+        peers.each do |pd|
+          attributes = PeerIdentityEnvelope.build(
+            source: pd,
+            trust_store: @config.trust_store,
+            relayed_by: seed_url
+          )
+          next if attributes[:name].nil? || attributes[:url].nil?
+          next if attributes[:name] == @config.peer_name
+
+          @config.peer_registry.register(
+            Peer.new(
+              name: attributes[:name],
+              url: attributes[:url],
+              capabilities: attributes[:capabilities] || [],
+              tags: attributes[:tags] || [],
+              metadata: attributes[:metadata] || {}
+            )
+          )
+
+          CheckpointGossip.sync(attributes[:metadata] || {}, config: @config, source: :poller)
+        end
+      rescue Igniter::Server::Client::ConnectionError
+        nil
+      end
+    end
+    end
+  end
+end

data/packages/igniter-cluster/lib/igniter/cluster/mesh/rebalance_plan.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Cluster
+    module Mesh
+      # Immutable result of a RebalancePlanner#plan call.
+      #
+      # Carries a list of ownership transfer recommendations ordered from
+      # highest-impact to lowest. Each transfer hash has the shape:
+      #
+      #   {
+      #     action:      :transfer_ownership,
+      #     entity_type: String,
+      #     entity_id:   String,
+      #     from_owner:  String,
+      #     to_owner:    String
+      #   }
+      #
+      # Call #to_routing_plans to convert transfers into routing plan hashes
+      # compatible with RoutingPlanExecutor.
+      class RebalancePlan
+        attr_reader :transfers, :rationale, :skew
+
+        # @param transfers [Array<Hash>]   ordered transfer recommendations
+        # @param rationale [String]        human-readable explanation
+        # @param skew      [Integer]       entity count imbalance (max - min across eligible nodes)
+        def initialize(transfers:, rationale:, skew:)
+          @transfers = Array(transfers).map(&:freeze).freeze
+          @rationale = rationale.to_s
+          @skew      = skew.to_i
+          freeze
+        end
+
+        def balanced?
+          @transfers.empty?
+        end
+
+        def size
+          @transfers.size
+        end
+
+        # Convert transfers to routing plan hashes consumable by RoutingPlanExecutor.
+        def to_routing_plans
+          @transfers.map do |t|
+            {
+              action:            :transfer_ownership,
+              scope:             :ownership_placement,
+              automated:         true,
+              requires_approval: false,
+              params: {
+                entity_type: t[:entity_type],
+                entity_id:   t[:entity_id],
+                from_owner:  t[:from_owner],
+                to_owner:    t[:to_owner]
+              }
+            }
+          end
+        end
+
+        def to_h
+          { transfers: @transfers, rationale: @rationale, skew: @skew }
+        end
+      end
+    end
+  end
+end

data/packages/igniter-cluster/lib/igniter/cluster/mesh/rebalance_planner.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Cluster
+    module Mesh
+      # Analyses entity ownership distribution across cluster nodes and recommends
+      # ownership transfers to balance load across eligible peers.
+      #
+      # Algorithm:
+      #   1. Collect all Ownership::Claim objects from the registry.
+      #   2. Identify eligible target nodes via ObservationQuery (healthy + capable).
+      #   3. Count claims per owner. Owners absent from the eligible set are treated
+      #      as transfer sources only ("orphaned" owners).
+      #   4. target = floor(total_claims / eligible_node_count)
+      #   5. Source priority: ineligible owners first, then most-overloaded eligible.
+      #   6. Destination: eligible node with the fewest current claims.
+      #   7. Greedily generate transfers until no source/destination pair remains.
+      #
+      # skew_threshold: minimum (max - min) count difference to trigger rebalancing.
+      # Default is 2 — a skew of 1 is acceptable and avoids churn.
+      #
+      # Entry points:
+      #   RebalancePlanner.new(ownership_registry: r, observations: obs).plan
+      #   Igniter::Cluster::Mesh.rebalance(ownership_registry, capabilities: [:database])
+      class RebalancePlanner
+        DEFAULT_SKEW_THRESHOLD = 2
+        MAX_ITERATIONS         = 10_000
+
+        # @param ownership_registry [Ownership::Registry]    claim store
+        # @param observations       [Array<NodeObservation>] live peer snapshot
+        # @param capabilities       [Array<Symbol>, nil]     restrict eligible nodes to these caps
+        # @param skew_threshold     [Integer]
+        def initialize(ownership_registry:, observations:, capabilities: nil, skew_threshold: DEFAULT_SKEW_THRESHOLD)
+          @registry       = ownership_registry
+          @observations   = observations
+          @capabilities   = capabilities ? Array(capabilities).map(&:to_sym) : nil
+          @skew_threshold = skew_threshold
+        end
+
+        # @return [RebalancePlan]
+        def plan
+          eligible = eligible_node_names
+          claims   = @registry.all
+
+          if eligible.empty?
+            return RebalancePlan.new(transfers: [], rationale: "no eligible nodes found", skew: 0)
+          end
+
+          if claims.empty?
+            return RebalancePlan.new(transfers: [], rationale: "no claims to rebalance", skew: 0)
+          end
+
+          counts = build_counts(claims, eligible)
+          skew   = counts.values.max - counts.values.min
+
+          if skew <= @skew_threshold
+            return RebalancePlan.new(
+              transfers: [],
+              rationale: "skew #{skew} is within threshold (#{@skew_threshold})",
+              skew: skew
+            )
+          end
+
+          transfers = compute_transfers(claims, counts, eligible)
+          rationale = "skew #{skew} > threshold #{@skew_threshold}; #{transfers.size} transfer(s) planned"
+
+          RebalancePlan.new(transfers: transfers, rationale: rationale, skew: skew)
+        end
+
+        private
+
+        def eligible_node_names
+          q = ObservationQuery.new(@observations).healthy
+          q = q.with(*@capabilities) if @capabilities
+          q.to_a.map(&:name).to_set
+        end
+
+        # Build a count map: every eligible node starts at 0, then add actual claims.
+        def build_counts(claims, eligible)
+          counts = eligible.each_with_object({}) { |name, h| h[name] = 0 }
+          claims.each do |claim|
+            counts[claim.owner] ||= 0
+            counts[claim.owner] += 1
+          end
+          counts
+        end
+
+        def compute_transfers(all_claims, counts, eligible)
+          target    = all_claims.size / eligible.size
+          transfers = []
+          remaining = all_claims.dup
+          working   = counts.dup
+          iterations = 0
+
+          loop do
+            iterations += 1
+            break if iterations > MAX_ITERATIONS
+
+            from_owner = pick_source(working, target, eligible)
+            break unless from_owner
+
+            to_owner = pick_destination(working, eligible)
+            break unless to_owner
+            break if from_owner == to_owner
+
+            claim = remaining.find { |c| c.owner == from_owner }
+            break unless claim
+
+            transfers << {
+              action:      :transfer_ownership,
+              entity_type: claim.entity_type,
+              entity_id:   claim.entity_id,
+              from_owner:  from_owner,
+              to_owner:    to_owner
+            }
+
+            remaining.delete(claim)
+            working[from_owner] -= 1
+            working[to_owner]   = (working[to_owner] || 0) + 1
+          end
+
+          transfers
+        end
+
+        # Highest-priority transfer source.
+        # Ineligible owners (orphaned claims) take precedence over overloaded eligible nodes.
+        # Eligible sources: stop when all eligible node counts are within 1 of each other.
+        def pick_source(counts, _target, eligible)
+          ineligible = counts
+            .reject { |owner, _| eligible.include?(owner) }
+            .select { |_, count| count > 0 }
+
+          return ineligible.max_by { |_, c| c }.first if ineligible.any?
+
+          eligible_counts = counts.select { |owner, _| eligible.include?(owner) }
+          max_count = eligible_counts.values.max || 0
+          min_count = eligible_counts.values.min || 0
+          return nil if max_count - min_count <= 1
+
+          eligible_counts.select { |_, c| c == max_count }.keys.min
+        end
+
+        # Eligible node with the fewest current claims (greedy destination).
+        # Breaks ties by name for deterministic ordering.
+        def pick_destination(counts, eligible)
+          eligible_counts = counts.select { |owner, _| eligible.include?(owner) }
+          min_count = eligible_counts.values.min
+          eligible_counts.select { |_, c| c == min_count }.keys.min
+        end
+      end
+    end
+  end
+end

data/packages/igniter-cluster/lib/igniter/cluster/mesh/repair_loop.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Cluster
+    module Mesh
+    # Background loop that periodically executes automated routing remediation
+    # plans from two signal sources:
+    #
+    #   1. Routing diagnostics — plans extracted from the configured report provider
+    #      (same as before; governs routing plan executor actions).
+    #
+    #   2. Workload signals — plans generated from WorkloadTracker degraded/overloaded
+    #      peers (fires only when config.workload_tracker is set).
+    #
+    # Each tick calls heal_once which runs both sources in sequence and records
+    # separate governance trail events for each: :routing_self_heal_tick and
+    # :workload_self_heal_tick.
+    #
+    # heal_once is fully backward-compatible — it returns the RoutingPlanResult
+    # from the routing repair path (or idle_result when no report is available).
+    # Workload repair is a side-effect of the same tick.
+    class RepairLoop
+      def initialize(config)
+        @config  = config
+        @running = false
+        @thread  = nil
+        @mutex   = Mutex.new
+      end
+
+      def start
+        @mutex.synchronize do
+          return if @running
+
+          @running = true
+          @thread  = Thread.new { run_loop }
+          @thread.abort_on_exception = false
+        end
+      end
+
+      def stop
+        @mutex.synchronize do
+          @running = false
+          @thread&.kill
+          @thread = nil
+        end
+      end
+
+      def running?
+        @mutex.synchronize { @running }
+      end
+
+      # One unified repair tick: routing plans + workload signals.
+      #
+      # Returns the RoutingPlanResult from the routing path (backward-compatible).
+      # Workload repair runs as a side-effect and records :workload_self_heal_tick
+      # in the governance trail when problem peers are found.
+      def heal_once
+        routing_result = execute_routing_heal
+        execute_workload_heal if @config.workload_tracker
+        routing_result
+      end
+
+      private
+
+      def run_loop
+        loop do
+          sleep(@config.self_heal_interval)
+          break unless @running
+
+          heal_once
+        end
+      end
+
+      def execute_routing_heal
+        report = @config.current_routing_report
+        return idle_result(:no_report) unless report
+
+        result = Igniter::Cluster::RoutingPlanExecutor.new(config: @config).run_many(
+          Array(report.dig(:routing, :plans)),
+          automated_only: true,
+          limit: @config.self_heal_limit
+        )
+
+        @config.governance_trail&.record(
+          :routing_self_heal_tick,
+          source: :repair_loop,
+          payload: {
+            plans:   Array(report.dig(:routing, :plans)).size,
+            applied: result.applied.size,
+            blocked: result.blocked.size,
+            skipped: result.skipped.size
+          }
+        )
+
+        result
+      rescue StandardError
+        idle_result(:loop_error)
+      end
+
+      def execute_workload_heal
+        tracker       = @config.workload_tracker
+        degraded      = tracker.degraded_peers
+        overloaded    = tracker.overloaded_peers
+        problem_peers = (degraded + overloaded).uniq
+
+        return if problem_peers.empty?
+
+        now          = Time.now.utc
+        registry_obs = @config.peer_registry.observations(now: now, workload_tracker: tracker)
+        static_obs   = @config.peers.map { |p| p.to_observation(now: now, workload_tracker: tracker) }
+        all_obs      = (registry_obs + static_obs).uniq(&:name)
+
+        plans = problem_peers.flat_map do |peer_name|
+          obs = all_obs.find { |o| o.name == peer_name }
+          next [] unless obs
+
+          [{
+            id:                "workload-repair-#{peer_name}",
+            action:            :refresh_capabilities,
+            scope:             :mesh_router,
+            automated:         true,
+            requires_approval: false,
+            params:            { peer_name: peer_name, reason: :workload_degradation }
+          }]
+        end
+
+        return if plans.empty?
+
+        result = Igniter::Cluster::RoutingPlanExecutor.new(config: @config).run_many(
+          plans,
+          automated_only: true,
+          limit:          @config.self_heal_limit
+        )
+
+        @config.governance_trail&.record(
+          :workload_self_heal_tick,
+          source:  :repair_loop,
+          payload: {
+            degraded:   degraded,
+            overloaded: overloaded,
+            plans:      plans.size,
+            applied:    result.applied.size,
+            blocked:    result.blocked.size
+          }
+        )
+      rescue StandardError
+        nil
+      end
+
+      def idle_result(reason)
+        Igniter::Cluster::RoutingPlanResult.new(
+          applied: [],
+          blocked: [],
+          skipped: [],
+          summary: {
+            status:         :idle,
+            reason:         reason,
+            total:          0,
+            applied:        0,
+            blocked:        0,
+            skipped:        0,
+            automated_only: true
+          }
+        )
+      end
+    end
+    end
+  end
+end