igniter 0.4.5 → 0.5.1

data/packages/igniter-cluster/lib/igniter/cluster/rag/ranker.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Cluster
+    module RAG
+      # Merges and re-ranks RetrievalResult collections from multiple shards.
+      #
+      # Ranking is by composite_score, which weights raw text relevance by the
+      # source peer's trust status and observation confidence. This means a
+      # result from a trusted, high-confidence peer will beat an equally-
+      # relevant result from an unknown peer.
+      #
+      # Deduplication is id-based: when the same Chunk id appears in multiple
+      # shards, the highest-composite-score copy is kept.
+      #
+      # Entry point:
+      #   ranker = Ranker.new
+      #   merged = ranker.merge(local_results, remote_results_a, remote_results_b, limit: 10)
+      class Ranker
+        # Merge one or more RetrievalResult arrays into a single ranked list.
+        #
+        # @param result_arrays [Array<Array<RetrievalResult>>]
+        # @param limit         [Integer, nil]   max results; nil = no cap
+        # @param deduplicate   [Boolean]        remove duplicate chunk ids
+        # @return [Array<RetrievalResult>]
+        def merge(*result_arrays, limit: nil, deduplicate: true)
+          all    = result_arrays.flatten
+          all    = deduplicate(all) if deduplicate
+          sorted = all.sort_by { |r| -r.composite_score }
+          limit ? sorted.first(limit) : sorted
+        end
+
+        private
+
+        def deduplicate(results)
+          best = {}
+          results.each do |r|
+            existing = best[r.id]
+            best[r.id] = r if existing.nil? || r.composite_score > existing.composite_score
+          end
+          best.values
+        end
+      end
+    end
+  end
+end

data/packages/igniter-cluster/lib/igniter/cluster/rag/retrieval_query.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Cluster
+    module RAG
+      # Typed specification for a knowledge retrieval request.
+      #
+      # Accepted by KnowledgeShard#search and Mesh.retrieve.
+      class RetrievalQuery
+        attr_reader :text, :tags, :limit, :min_score
+
+        # @param text      [String]           free-text query
+        # @param tags      [Array<Symbol>]    restrict to chunks that have ALL of these tags
+        # @param limit     [Integer]          max results to return
+        # @param min_score [Float]            minimum text relevance score (0..1)
+        def initialize(text:, tags: [], limit: 10, min_score: 0.0)
+          @text      = text.to_s.freeze
+          @tags      = Array(tags).map(&:to_sym).freeze
+          @limit     = [limit.to_i, 1].max
+          @min_score = [min_score.to_f, 0.0].max
+          freeze
+        end
+
+        def to_h
+          { text: @text, tags: @tags, limit: @limit, min_score: @min_score }
+        end
+      end
+    end
+  end
+end

data/packages/igniter-cluster/lib/igniter/cluster/rag/retrieval_result.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Cluster
+    module RAG
+      # A single result returned by KnowledgeShard#search or the Ranker.
+      #
+      # Carries the matched Chunk together with retrieval provenance:
+      # text relevance score, source shard name, and (optionally) the
+      # NodeObservation of the peer that holds the shard.
+      #
+      # composite_score weights the raw text score by the peer's trust
+      # status and observation confidence — results from trusted, high-
+      # confidence peers rank above equally-relevant but untrusted ones.
+      class RetrievalResult
+        attr_reader :chunk, :score, :source, :observation
+
+        # @param chunk       [Chunk]            the matched knowledge unit
+        # @param score       [Float]            raw text relevance score (0..1)
+        # @param source      [String]           shard name / peer name
+        # @param observation [NodeObservation, nil]  live peer snapshot (nil = local)
+        def initialize(chunk:, score:, source:, observation: nil)
+          @chunk       = chunk
+          @score       = score.to_f
+          @source      = source.to_s.freeze
+          @observation = observation
+          freeze
+        end
+
+        # Chunk attribute shortcuts
+        def id;         @chunk.id; end
+        def content;    @chunk.content; end
+        def tags;       @chunk.tags; end
+        def metadata;   @chunk.metadata; end
+        def indexed_at; @chunk.indexed_at; end
+
+        # Provenance accessors
+        def trusted?
+          @observation ? @observation.trusted? : true
+        end
+
+        def confidence
+          @observation&.confidence || 1.0
+        end
+
+        # Composite score = raw_score × trust_factor × confidence
+        #
+        # trust_factor:
+        #   trusted peer     → 1.00
+        #   unknown peer     → 0.85
+        #   untrusted peer   → 0.70
+        def composite_score
+          trust_factor = case @observation&.trust_status
+                         when :trusted then 1.00
+                         when nil      then 1.00  # local / no observation
+                         else
+                           @observation.trusted? ? 1.00 : 0.85
+                         end
+          (@score * trust_factor * confidence).round(6)
+        end
+
+        def to_h
+          {
+            id:              id,
+            content:         content,
+            score:           @score,
+            composite_score: composite_score,
+            source:          @source,
+            tags:            tags,
+            trusted:         trusted?,
+            confidence:      confidence
+          }
+        end
+      end
+    end
+  end
+end

data/packages/igniter-cluster/lib/igniter/cluster/rag.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "rag/chunk"
+require_relative "rag/retrieval_query"
+require_relative "rag/retrieval_result"
+require_relative "rag/knowledge_shard"
+require_relative "rag/ranker"
+require_relative "rag/net_http_adapter"
+require_relative "rag/fanout_retriever"
+
+module Igniter
+  module Cluster
+    # Phase 5+7: Decentralized Knowledge Plane.
+    #
+    # Each cluster node can maintain a local KnowledgeShard — a content-addressed
+    # store of text chunks that can be searched via keyword relevance.
+    #
+    # v1: local shard retrieval with trust-aware scoring.
+    # v2: distributed fan-out to remote `:rag`-capable peers via FanoutRetriever.
+    #
+    # Cluster integration:
+    #   Igniter::Cluster::Mesh.configure { |c| c.local_capabilities << :rag }
+    #   Igniter::Cluster::Mesh.shard.add("Ruby closures bind at creation time", tags: [:ruby])
+    #
+    #   # Local only (default):
+    #   results = Igniter::Cluster::Mesh.retrieve("how closures work", limit: 5)
+    #
+    #   # Distributed fan-out:
+    #   results = Igniter::Cluster::Mesh.retrieve("how closures work", distributed: true)
+    #
+    # Standalone:
+    #   shard = Igniter::Cluster::RAG::KnowledgeShard.new(name: "my-shard")
+    #   shard.add("...", tags: [:topic])
+    #   results = shard.search("query text", tags: [:topic], limit: 10)
+    module RAG
+    end
+  end
+end

data/packages/igniter-cluster/lib/igniter/cluster/remote_adapter.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Cluster
+    # Cluster-aware remote adapter.
+    #
+    # Builds on top of the plain server transport and adds capability/pinned
+    # peer resolution via mesh routing.
+    class RemoteAdapter < Igniter::Server::RemoteAdapter
+      def call(node:, inputs:, execution: nil)
+        super
+      rescue Igniter::Cluster::Mesh::DeferredCapabilityError => e
+        publish_pending_routing_trace(node, e, execution)
+        raise
+      rescue Igniter::Cluster::Mesh::IncidentError => e
+        publish_failed_routing_trace(node, e, execution)
+        raise
+      end
+
+      private
+
+      def resolve_url(node)
+        case node.routing_mode
+        when :capability
+          resolve_capability_url(node)
+        when :pinned
+          resolve_pinned_url(node)
+        else
+          super
+        end
+      end
+
+      def resolve_capability_url(node)
+        query = node.capability_query || { all_of: [node.capability] }
+        deferred = Igniter::Runtime::DeferredResult.build(
+          payload: { capability: node.capability, query: query },
+          source_node: node.name,
+          waiting_on: node.name
+        )
+        if node.capability_query
+          Igniter::Cluster::Mesh.router.find_peer_for_query(node.capability_query, deferred)
+        else
+          Igniter::Cluster::Mesh.router.find_peer_for(node.capability, deferred)
+        end
+      end
+
+      def resolve_pinned_url(node)
+        Igniter::Cluster::Mesh.router.resolve_pinned(node.pinned_to)
+      end
+
+      def publish_pending_routing_trace(node, error, execution)
+        report = Igniter::Cluster::Diagnostics::RoutingContributor.report_for_trace(
+          node_name: node.name,
+          path: node.path,
+          status: :pending,
+          routing_trace: error.explanation,
+          token: error.deferred_result.token,
+          waiting_on: error.deferred_result.waiting_on || node.name,
+          source_node: error.deferred_result.source_node || node.name,
+          latest_event_type: :node_pending
+        )
+
+        enrich_and_record_report!(report, execution)
+      end
+
+      def publish_failed_routing_trace(node, error, execution)
+        routing_trace = error.context[:routing_trace]
+        return unless routing_trace
+
+        report = Igniter::Cluster::Diagnostics::RoutingContributor.report_for_trace(
+          node_name: node.name,
+          path: node.path,
+          status: :failed,
+          routing_trace: routing_trace,
+          error: {
+            type: error.class.name,
+            message: error.message
+          },
+          latest_event_type: :node_failed
+        )
+
+        enrich_and_record_report!(report, execution)
+      end
+
+      def enrich_and_record_report!(report, execution)
+        return unless report.is_a?(Hash)
+
+        if execution
+          report = report.merge(
+            graph: execution.compiled_graph.name,
+            execution_id: execution.events.execution_id
+          )
+        end
+
+        Igniter::Cluster::Mesh.config.record_routing_report!(report)
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/packages/igniter-cluster/lib/igniter/cluster/replication/bootstrapper.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Cluster
+    module Replication
+    # Abstract base class for deployment bootstrappers.
+    #
+    # Subclasses implement three lifecycle methods:
+    #   install — copy/install the application on the remote
+    #   start   — launch the application process
+    #   verify  — confirm the application is running (default impl provided)
+    class Bootstrapper
+      BootstrapError = Class.new(Igniter::Error)
+
+      # Install the application on the remote server.
+      #
+      # @param session [SSHSession] active SSH session to the remote host
+      # @param manifest [Manifest] self-description of the running instance
+      # @param env [Hash] environment variables to write on remote
+      # @param target_path [String] base directory on the remote server
+      def install(session:, manifest:, env: {}, target_path: "/opt/igniter")
+        raise NotImplementedError, "#{self.class}#install not implemented"
+      end
+
+      # Start the application on the remote server.
+      #
+      # @param session [SSHSession] active SSH session to the remote host
+      # @param manifest [Manifest] self-description of the running instance
+      # @param target_path [String] base directory on the remote server
+      def start(session:, manifest:, target_path: "/opt/igniter")
+        raise NotImplementedError, "#{self.class}#start not implemented"
+      end
+
+      # Verify the application is running on the remote server.
+      # Returns true if Igniter can be loaded, false otherwise.
+      #
+      # @param session [SSHSession] active SSH session to the remote host
+      # @param target_path [String] base directory on the remote server (unused by default impl)
+      def verify(session:, target_path: "/opt/igniter") # rubocop:disable Lint/UnusedMethodArgument
+        result = session.exec("ruby -e 'require \"igniter\"; puts Igniter::VERSION' 2>/dev/null")
+        result[:success] && !result[:stdout].strip.empty?
+      rescue SSHSession::SSHError
+        false
+      end
+
+      protected
+
+      # Write environment variables to a shell-sourceable file on the remote.
+      #
+      # @param session [SSHSession] active SSH session
+      # @param env [Hash] key-value pairs to export
+      # @param path [String] remote path for the env file
+      def write_env_file(session:, env:, path:)
+        return if env.nil? || env.empty?
+
+        require "shellwords"
+        lines = env.map { |k, v| "export #{k}=#{v.to_s.shellescape}" }.join("\n")
+        session.exec!("printf '%s\\n' #{lines.shellescape} > #{path}")
+      end
+    end
+    end
+  end
+end

data/packages/igniter-cluster/lib/igniter/cluster/replication/bootstrappers/gem.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "shellwords"
+
+module Igniter
+  module Cluster
+    module Replication
+    module Bootstrappers
+      # Deploys by installing the igniter gem from RubyGems on the remote host.
+      #
+      # Optionally pins a specific gem version. The startup script defaults to
+      # the +igniter-stack+ executable installed with the gem.
+      class Gem < Bootstrapper
+        def initialize(version: nil, startup_script: nil)
+          super()
+          @version        = version
+          @startup_script = startup_script
+        end
+
+        def install(session:, manifest:, env: {}, target_path: "/opt/igniter") # rubocop:disable Lint/UnusedMethodArgument
+          ver_flag = @version ? " -v #{@version}" : ""
+          session.exec!("gem install igniter#{ver_flag} --no-document")
+          session.exec!("mkdir -p #{target_path}")
+          write_env_file(session: session, env: env, path: "#{target_path}/.env")
+        end
+
+        def start(session:, manifest:, target_path: "/opt/igniter") # rubocop:disable Lint/UnusedMethodArgument
+          log_path = "#{target_path}/igniter.log"
+          pid_path = "#{target_path}/igniter.pid"
+          script   = @startup_script || "igniter-stack"
+          session.exec!(
+            "sh -lc 'nohup #{script} >> #{Shellwords.escape(log_path)} 2>&1 & echo $! > #{Shellwords.escape(pid_path)}'"
+          )
+        end
+      end
+    end
+    end
+  end
+end

data/packages/igniter-cluster/lib/igniter/cluster/replication/bootstrappers/git.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "shellwords"
+
+module Igniter
+  module Cluster
+    module Replication
+    module Bootstrappers
+      # Deploys by cloning a git repository on the remote host.
+      #
+      # Requires git to be installed on the remote. After cloning, installs
+      # bundler and runs bundle install.
+      class Git < Bootstrapper
+        DEFAULT_BRANCH         = "main"
+        DEFAULT_BUNDLE_OPTIONS = "--without development test"
+
+        def initialize(repo_url:, branch: DEFAULT_BRANCH, bundle_options: DEFAULT_BUNDLE_OPTIONS)
+          super()
+          @repo_url       = repo_url
+          @branch         = branch
+          @bundle_options = bundle_options
+        end
+
+        def install(session:, manifest:, env: {}, target_path: "/opt/igniter") # rubocop:disable Lint/UnusedMethodArgument
+          app_path = "#{target_path}/app"
+          session.exec!("mkdir -p #{target_path}")
+          session.exec!("git clone --branch #{@branch} --depth 1 #{@repo_url} #{app_path}")
+          session.exec!("cd #{app_path} && gem install bundler --no-document")
+          session.exec!("cd #{app_path} && bundle install #{@bundle_options}")
+          write_env_file(session: session, env: env, path: "#{target_path}/.env")
+        end
+
+        def start(session:, manifest:, target_path: "/opt/igniter")
+          app_path = "#{target_path}/app"
+          log_path = "#{target_path}/igniter.log"
+          pid_path = "#{target_path}/igniter.pid"
+          cmd      = manifest.startup_command
+          session.exec!(
+            "sh -lc 'cd #{Shellwords.escape(app_path)} && nohup ruby #{cmd} >> #{Shellwords.escape(log_path)} 2>&1 & echo $! > #{Shellwords.escape(pid_path)}'"
+          )
+        end
+      end
+    end
+    end
+  end
+end

data/packages/igniter-cluster/lib/igniter/cluster/replication/bootstrappers/tarball.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+require "fileutils"
+require "shellwords"
+
+module Igniter
+  module Cluster
+    module Replication
+    module Bootstrappers
+      # Deploys by creating a gzipped tarball of the local source tree,
+      # uploading it via SCP, and extracting it on the remote host.
+      #
+      # This strategy works without git or internet access on the remote,
+      # making it suitable for air-gapped or private environments.
+      class Tarball < Bootstrapper
+        def install(session:, manifest:, env: {}, target_path: "/opt/igniter")
+          tarball = create_tarball(manifest)
+          begin
+            remote_tar = "/tmp/igniter_replication_#{Process.pid}.tar.gz"
+            session.upload!(tarball, remote_tar)
+            extract_and_install(session, remote_tar, target_path)
+            write_env_file(session: session, env: env, path: "#{target_path}/.env")
+          ensure
+            FileUtils.rm_f(tarball)
+          end
+        end
+
+        def start(session:, manifest:, target_path: "/opt/igniter")
+          app_path = "#{target_path}/app"
+          log_path = "#{target_path}/igniter.log"
+          pid_path = "#{target_path}/igniter.pid"
+          cmd      = File.basename(manifest.startup_command)
+          session.exec!(
+            "sh -lc 'cd #{Shellwords.escape(app_path)} && nohup ruby #{cmd} >> #{Shellwords.escape(log_path)} 2>&1 & echo $! > #{Shellwords.escape(pid_path)}'"
+          )
+        end
+
+        private
+
+        def extract_and_install(session, remote_tar, target_path)
+          app_path = "#{target_path}/app"
+          session.exec!("mkdir -p #{app_path}")
+          session.exec!("tar -xzf #{remote_tar} -C #{app_path} --strip-components=1")
+          session.exec!("cd #{app_path} && gem install bundler --no-document")
+          session.exec!("cd #{app_path} && bundle install --without development test")
+          session.exec!("rm -f #{remote_tar}")
+        end
+
+        def create_tarball(manifest)
+          source  = manifest.source_path
+          tmpfile = File.join(Dir.tmpdir, "igniter_replication_#{Process.pid}.tar.gz")
+          parent  = File.dirname(source)
+          name    = File.basename(source)
+          system("tar", "-czf", tmpfile, "-C", parent, name)
+          tmpfile
+        end
+      end
+    end
+    end
+  end
+end