legion-settings 1.3.26 → 1.4.0

data/lib/legion/settings/extensions.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require_relative 'extensions/store'
+require_relative 'extensions/filter'
+require_relative 'extensions/normalizer'
+
+module Legion
+  module Settings
+    # Thread-safe runtime registry for extensions, runners, and tools.
+    #
+    # Used by the LegionIO boot pipeline to register discovered extensions,
+    # their runner modules, and individual tools. Consumers (legion-mcp,
+    # legion-llm, legion-rbac, API) read from this registry at runtime.
+    #
+    # Each store is a Concurrent::Map-backed Store instance. Read operations
+    # return frozen duplicates so callers cannot mutate registry internals.
+    module Extensions
+      @extension_store = Store.new
+      @runner_store = Store.new
+      @tool_store = Store.new
+
+      class << self
+        # ----------------------------------------------------------------
+        # Registration (called during LegionIO boot pipeline)
+        # ----------------------------------------------------------------
+
+        def register_extension(name, metadata = {})
+          normalized = Normalizer.normalize_extension(name, metadata)
+          @extension_store.register(name, normalized)
+        end
+
+        def register_runner(name, metadata = {})
+          normalized = Normalizer.normalize_runner(name, metadata)
+          @runner_store.register(name, normalized)
+        end
+
+        def register_tool(name, metadata = {})
+          normalized = Normalizer.normalize_tool(name, metadata)
+          @tool_store.register(name, normalized)
+        end
+
+        def transition(name, state, **extra)
+          @extension_store.update(name, state: state, transitioned_at: Time.now, **extra)
+        end
+
+        # ----------------------------------------------------------------
+        # Query (called by legion-mcp, legion-llm, legion-rbac, API)
+        # ----------------------------------------------------------------
+
+        def extensions
+          @extension_store.all
+        end
+
+        def runners
+          @runner_store.all
+        end
+
+        def tools
+          @tool_store.all
+        end
+
+        def find_extension(name)
+          @extension_store.find(name)
+        end
+
+        def find_runner(name)
+          @runner_store.find(name)
+        end
+
+        def find_tool(name)
+          @tool_store.find(name)
+        end
+
+        def filter_tools(**criteria)
+          entries = @tool_store.all.map(&:dup)
+          result = Filter.apply_tool_filters(entries, criteria, extension_store: @extension_store)
+          result.each(&:freeze)
+          result.freeze
+        end
+
+        def filter_extensions(**criteria)
+          entries = @extension_store.all.map(&:dup)
+          result = Filter.apply_extension_filters(entries, criteria)
+          result.each(&:freeze)
+          result.freeze
+        end
+
+        # ----------------------------------------------------------------
+        # Lifecycle
+        # ----------------------------------------------------------------
+
+        def unregister_extension(name)
+          removed = @extension_store.delete(name)
+          return nil unless removed
+
+          key = name.to_s
+          @runner_store.delete_where { |v| v[:extension].to_s == key }
+          @tool_store.delete_where { |v| v[:extension].to_s == key }
+          removed
+        end
+
+        def unregister_tool(name)
+          @tool_store.delete(name)
+        end
+
+        def reset!
+          @extension_store.clear
+          @runner_store.clear
+          @tool_store.clear
+        end
+
+        # ----------------------------------------------------------------
+        # Counts
+        # ----------------------------------------------------------------
+
+        def extension_count
+          @extension_store.size
+        end
+
+        def runner_count
+          @runner_store.size
+        end
+
+        def tool_count
+          @tool_store.size
+        end
+      end
+    end
+  end
+end

data/lib/legion/settings/helper.rb

@@ -1,41 +1,119 @@
 # frozen_string_literal: true
 
+require 'concurrent/hash'
+
 module Legion
   module Settings
     module Helper
+      # Namespace boundary words — segment extraction stops at these.
+      # Matches LegionIO's Extensions::Helpers::Base::NAMESPACE_BOUNDARIES.
+      NAMESPACE_BOUNDARIES = %w[Actor Actors Runners Helpers Transport Data].freeze
+
+      # Returns the gem-level settings hash for this extension.
+      # Sub-modules (ConceptualBlending inside Agentic::Language) get
+      # the SAME hash as the root — they access their section via key:
+      #   settings[:conceptual_blending]
+      #
+      # Path resolution uses segments derived from the class namespace:
+      #   Legion::Extensions::Github            → Settings[:extensions][:github]
+      #   Legion::Extensions::Agentic::Learning → Settings[:extensions][:agentic][:learning]
+      #   Legion::Extensions::MicrosoftTeams    → Settings[:extensions][:microsoft_teams]
+      #   Legion::Extensions::Llm::Openai       → Settings[:extensions][:llm][:openai]
       def settings
-        ext_key = derive_settings_key
-        if Legion::Settings[:extensions]&.key?(ext_key)
-          Legion::Settings[:extensions][ext_key]
-        else
-          {}
-        end
+        segments = derive_settings_segments
+        dig_or_create(Legion::Settings[:extensions], segments)
       end
 
       private
 
-      def derive_settings_key
-        if respond_to?(:lex_filename)
-          fname = lex_filename
-          (fname.is_a?(Array) ? fname.first : fname).to_sym
-        else
-          derive_settings_key_from_class
-        end
+      # Derives the gem-level segments from the class namespace.
+      # Stops at NAMESPACE_BOUNDARIES so sub-modules (Runners, Actors, etc.)
+      # resolve to their parent extension, not deeper.
+      #
+      # Legion::Extensions::Agentic::Learning::ConceptualBlending::Runners::Blend
+      #   → ['agentic', 'learning'] (stops at ConceptualBlending because next is Runners)
+      #
+      # Legion::Extensions::Agentic::Learning::ConceptualBlending
+      #   → ['agentic', 'learning'] (stops at ConceptualBlending — it's a sub-module, not a segment)
+      #
+      # Wait — ConceptualBlending IS a namespace part, not a boundary word.
+      # The gem is lex-agentic-learning → segments are ['agentic', 'learning'].
+      # ConceptualBlending is INSIDE the gem, not part of the gem name.
+      # So we need to know the gem's segment count to stop there.
+      #
+      # Strategy: if the caller responds to :segments (LegionIO's Base mixin),
+      # use those directly. Otherwise derive from namespace, stopping at
+      # boundary words or after 2 levels (covers most lex-X-Y patterns).
+      def derive_settings_segments
+        # Prefer explicit segments from LegionIO's Helpers::Base
+        return segments.map { |s| s.to_s.to_sym } if respond_to?(:segments)
+
+        derive_segments_from_class
       end
 
-      def derive_settings_key_from_class
+      def derive_segments_from_class
         name = respond_to?(:ancestors) ? ancestors.first.to_s : self.class.to_s
         parts = name.split('::')
         ext_idx = parts.index('Extensions')
-        target = if ext_idx && parts[ext_idx + 1]
-                   parts[ext_idx + 1]
-                 else
-                   parts.last
-                 end
-        target.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
-              .gsub(/([a-z\d])([A-Z])/, '\1_\2')
-              .downcase
-              .to_sym
+        return [camelize_to_snake(parts.last).to_sym] unless ext_idx
+
+        segment_parts = []
+        ((ext_idx + 1)...parts.length).each do |i|
+          break if NAMESPACE_BOUNDARIES.include?(parts[i])
+
+          segment_parts << camelize_to_snake(parts[i]).to_sym
+        end
+
+        # The gem-level segments are the parts between Extensions:: and
+        # the first sub-module that isn't part of the gem name.
+        # For lex-agentic-learning, gem segments = [:agentic, :learning].
+        # ConceptualBlending is a sub-module INSIDE the gem.
+        # We use the registered extension entry to find the correct depth,
+        # falling back to all segments if no registry entry exists.
+        resolve_gem_segments(segment_parts)
+      end
+
+      def resolve_gem_segments(all_segments)
+        return all_segments if all_segments.length <= 1
+
+        # Check if Settings::Extensions has this extension registered
+        # with known segments — use those as the authoritative gem boundary.
+        if defined?(Legion::Settings::Extensions)
+          # Try progressively shorter segment paths to find the registered gem
+          all_segments.length.downto(1) do |len|
+            candidate = all_segments[0, len]
+            gem_name = "lex-#{candidate.join('-')}"
+            entry = Legion::Settings::Extensions.find_extension(gem_name)
+            return candidate if entry
+          end
+        end
+
+        all_segments
+      end
+
+      # Digs into a nested hash using segments as keys, creating
+      # Concurrent::Hash at each level if missing.
+      def dig_or_create(root, segments)
+        return Concurrent::Hash.new unless root.is_a?(Hash)
+
+        segments.reduce(root) do |current, key|
+          if current.is_a?(Hash) && current.key?(key)
+            current[key]
+          elsif current.is_a?(Hash)
+            empty = Concurrent::Hash.new
+            current[key] = empty
+            empty
+          else
+            return Concurrent::Hash.new
+          end
+        end
+      end
+
+      def camelize_to_snake(str)
+        str.to_s
+           .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+           .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+           .downcase
       end
     end
   end

data/lib/legion/settings/loader.rb

@@ -4,9 +4,11 @@ require 'resolv'
 require 'socket'
 require 'digest'
 require 'tmpdir'
+require 'concurrent/hash'
 require 'legion/logging'
 require 'legion/settings/os'
 require_relative 'dns_bootstrap'
+require_relative 'deep_merge'
 
 module Legion
   module Settings
@@ -15,7 +17,7 @@ module Legion
       include Legion::Logging::Helper
 
       class Error < RuntimeError; end
-      attr_reader :warnings, :errors, :loaded_files, :settings
+      attr_reader :warnings, :errors, :loaded_files, :settings, :merged_modules
 
       def self.default_directories
         env_dirs = ENV.fetch('LEGION_SETTINGS_DIRS', nil)
@@ -40,13 +42,14 @@ module Legion
         @settings = default_settings
         @indifferent_access = false
         @loaded_files = []
+        @merged_modules = {}
         log.debug('Initialized Legion::Settings::Loader with default settings')
       end
 
       def dns_defaults
         resolv_config = read_resolv_config
         {
-          fqdn:           detect_fqdn,
+          fqdn:           nil, # lazy — resolved on first access via resolve_fqdn!
           default_domain: resolv_config[:search_domains]&.first,
           search_domains: resolv_config[:search_domains] || [],
           nameservers:    resolv_config[:nameservers] || [],
@@ -54,6 +57,15 @@ module Legion
         }
       end
 
+      # Lazily resolve the FQDN on first access instead of blocking at init.
+      #
+      # @return [String, nil] the fully qualified domain name, or nil
+      def resolve_fqdn!
+        return @settings[:dns][:fqdn] if @settings.dig(:dns, :fqdn)
+
+        @settings[:dns][:fqdn] = detect_fqdn
+      end
+
       def client_defaults
         {
           hostname: system_hostname,
@@ -63,64 +75,25 @@ module Legion
         }
       end
 
-      def logging_defaults
-        {
-          level:       'info',
-          format:      'text',
-          log_file:    './legionio/logs/legion.log',
-          log_stdout:  true,
-          trace:       true,
-          async:       true,
-          include_pid: false,
-          transport:   {
-            enabled:            true,
-            forward_logs:       true,
-            forward_exceptions: true
-          }
-        }
-      end
-
-      def absorbers_defaults
-        {
-          enabled:   true,
-          max_depth: 5,
-          sources:   {
-            meetings:    {
-              enabled:          true,
-              include_chat:     true,
-              include_files:    true,
-              retention_days:   90,
-              min_duration_min: 5
-            },
-            email_inbox: {
-              enabled:      false,
-              folder:       'inbox',
-              max_age_days: 30
-            },
-            github:      {
-              enabled: true,
-              events:  %w[pull_request issues]
-            },
-            files:       {
-              enabled:    true,
-              watch_dirs: [],
-              extensions: %w[pdf docx txt md pptx rtf]
-            }
-          }
-        }
-      end
+      # No more per-module defaults methods in the Loader.
+      # Tier 1 deps (json, logging) are called directly in default_settings.
+      # Tier 2 libraries (transport, cache, etc.) self-register via
+      # Legion::Settings.register_library when they load.
 
       def default_settings
         {
+          # --- Tier 1: gemspec dependencies (always installed with legion-settings) ---
+          # legion-logging: always available, has Settings.default
+          logging:                    Legion::Logging::Settings.default,
+          # legion-json: always available, no Settings module yet — stub
+          # until legion-json adds Legion::JSON::Settings.default
+          json:                       Concurrent::Hash.new,
+
+          # --- Structural: owned by legion-settings itself ---
           client:                     client_defaults,
-          cluster:                    { public_keys: {} },
-          crypt:                      {
-            cluster_secret:         nil,
-            cluster_secret_timeout: 5,
-            vault:                  { connected: false }
-          },
-          cache:                      { enabled: true, connected: false, driver: 'dalli' },
-          extensions:                 {
+          cluster:                    Concurrent::Hash.new,
+          dns:                        dns_defaults,
+          extensions:                 Concurrent::Hash[
             core:               %w[
               lex-node lex-tasker lex-scheduler lex-health lex-ping
               lex-telemetry lex-metering lex-log lex-audit
@@ -139,20 +112,25 @@ module Legion
             reserved_words:     %w[transport cache crypt data settings json logging llm rbac legion],
             agentic:            { allowed: nil, blocked: [] },
             parallel_pool_size: 24
-          },
+          ],
           reload:                     false,
           reloading:                  false,
           auto_install_missing_lex:   true,
           default_extension_settings: {},
-          logging:                    logging_defaults,
-          absorbers:                  absorbers_defaults,
-          transport:                  { connected: false },
-          data:                       { connected: false },
           role:                       { profile: nil, extensions: [] },
           region:                     { current: nil, primary: nil, failover: nil, peers: [],
                                         default_affinity: 'any', data_residency: {} },
           process:                    { role: 'full' },
-          dns:                        dns_defaults
+
+          # --- Tier 2: stubs for libraries that self-register via register_library ---
+          # These ensure Settings[:key] returns a hash (not nil) before
+          # the owning library loads. The library replaces these with its
+          # full defaults when it calls Legion::Settings.register_library.
+          absorbers:                  Concurrent::Hash.new,
+          cache:                      Concurrent::Hash.new,
+          crypt:                      Concurrent::Hash.new,
+          data:                       Concurrent::Hash.new,
+          transport:                  Concurrent::Hash.new
         }
       end
 
@@ -164,12 +142,26 @@ module Legion
         @settings
       end
 
+      # Direct key lookup — does NOT trigger indifferent_access! rebuild.
+      # This is the hot path called by every Settings[:key] access.
+      # Supports both symbol and string keys without converting the whole tree.
       def [](key)
-        to_hash[key]
+        result = @settings[key]
+        return result unless result.nil? && key.is_a?(String)
+
+        @settings[key.to_sym]
       end
 
+      # Direct nested lookup — does NOT trigger indifferent_access! rebuild.
       def dig(*keys)
-        to_hash.dig(*keys)
+        keys.reduce(self) do |current, key|
+          return nil unless current.respond_to?(:[])
+
+          value = current.is_a?(Loader) ? current[key] : (current[key] || current[key.to_s])
+          return nil if value.nil? && !current.is_a?(Loader)
+
+          value
+        end
       end
 
       def []=(key, value)
@@ -221,20 +213,21 @@ module Legion
 
       def load_module_settings(config)
         mod_name = config.keys.first
-        log_debug("Loading module settings: #{mod_name}")
+        log.debug("Loading module settings: #{mod_name}")
+        @merged_modules = deep_merge(@merged_modules, config)
         @settings = deep_merge(config, @settings)
         mark_dirty!
       end
 
       def load_module_default(config)
         mod_name = config.keys.first
-        log_debug("Loading module defaults: #{mod_name}")
+        log.debug("Loading module defaults: #{mod_name}")
         @settings = deep_merge(config, @settings)
         mark_dirty!
       end
 
       def load_file(file)
-        log_debug("Trying to load file #{file}")
+        log.debug("Trying to load file #{file}")
         if File.file?(file) && File.readable?(file)
           begin
             contents = read_config_file(file)
@@ -244,11 +237,11 @@ module Legion
             @loaded_files << file
             log.debug("Loaded settings file #{file}")
           rescue Legion::JSON::ParseError => e
-            log_error("config file must be valid json: #{file}")
-            log_error("  parse error: #{e.message}")
+            log.error("config file must be valid json: #{file}")
+            log.error("  parse error: #{e.message}")
           end
         else
-          log_warn("Config file does not exist or is not readable file:#{file}")
+          log.warn("Config file does not exist or is not readable file:#{file}")
         end
       end
 
@@ -257,7 +250,7 @@ module Legion
         if File.readable?(path) && File.executable?(path)
           files = Dir.glob(File.join(path, '**', '*.json'))
           files.each { |file| load_file(file) }
-          log_info("Settings: loaded directory #{path} (#{files.size} files)")
+          log.info("Settings: loaded directory #{path} (#{files.size} files)")
         else
           load_error('insufficient permissions for loading', directory: directory)
         end
@@ -270,7 +263,7 @@ module Legion
           @settings[:client][:subscriptions].uniq!
           mark_dirty!
         else
-          log_warn('unable to apply legion client overrides, reason: client subscriptions is not an array')
+          log.warn('unable to apply legion client overrides, reason: client subscriptions is not an array')
         end
       end
 
@@ -302,7 +295,7 @@ module Legion
       end
 
       def load_dns_first_boot(bootstrap)
-        log_debug("DNS bootstrap: first boot, fetching from #{bootstrap.url}")
+        log.debug("DNS bootstrap: first boot, fetching from #{bootstrap.url}")
         config = bootstrap.fetch
         bootstrap.write_cache(config) if config
         config
@@ -324,7 +317,7 @@ module Legion
           fresh = bootstrap.fetch
           bootstrap.write_cache(fresh) if fresh
         rescue StandardError => e
-          log_warn("DNS background refresh failed: #{e.message}")
+          log.warn("DNS background refresh failed: #{e.message}")
         end
       end
 
@@ -361,7 +354,7 @@ module Legion
 
         @settings[:api] ||= {}
         @settings[:api][:port] = ENV['LEGION_API_PORT'].to_i
-        log_warn("using api port environment variable, api: #{@settings[:api]}")
+        log.warn("using api port environment variable, api: #{@settings[:api]}")
         mark_dirty!
       end
 
@@ -374,29 +367,15 @@ module Legion
 
       def read_config_file(file)
         contents = File.read(file).dup
-        if contents.respond_to?(:force_encoding)
-          encoding = ::Encoding::ASCII_8BIT
-          contents = contents.force_encoding(encoding)
-          bom = (+"\xEF\xBB\xBF").force_encoding(encoding)
-          contents.sub!(bom, '')
-        else
-          contents.sub!(/^\357\273\277/, '')
-        end
+        encoding = ::Encoding::ASCII_8BIT
+        contents = contents.force_encoding(encoding)
+        bom = (+"\xEF\xBB\xBF").force_encoding(encoding)
+        contents.sub!(bom, '')
         contents.strip
       end
 
       def deep_merge(hash_one, hash_two)
-        merged = hash_one.dup
-        hash_two.each do |key, value|
-          merged[key] = if hash_one[key].is_a?(Hash) && value.is_a?(Hash)
-                          deep_merge(hash_one[key], value)
-                        elsif hash_one[key].is_a?(Array) && value.is_a?(Array)
-                          hash_one[key].concat(value).uniq
-                        else
-                          value
-                        end
-        end
-        merged
+        DeepMerge.deep_merge(hash_one, hash_two)
       end
 
       def create_loaded_tempfile!
@@ -423,7 +402,7 @@ module Legion
       def system_hostname
         Socket.gethostname
       rescue StandardError => e
-        log_debug("Legion::Settings::Loader#system_hostname failed: #{e.message}")
+        log.debug("Legion::Settings::Loader#system_hostname failed: #{e.message}")
         'unknown'
       end
 
@@ -432,7 +411,7 @@ module Legion
         preferred = addresses.find { |a| rfc1918?(a.ip_address) }
         (preferred || addresses.first)&.ip_address || 'unknown'
       rescue StandardError => e
-        log_debug("Legion::Settings::Loader#system_address failed: #{e.message}")
+        log.debug("Legion::Settings::Loader#system_address failed: #{e.message}")
         'unknown'
       end
 
@@ -442,34 +421,18 @@ module Legion
           ip.start_with?('192.168.')
       end
 
-      def log_info(message)
-        log.info(message)
-      end
-
-      def log_debug(message)
-        log.debug(message)
-      end
-
-      def log_warn(message)
-        log.warn(message)
-      end
-
-      def log_error(message)
-        log.error(message)
-      end
-
       def warning(message, data = {})
         @warnings << {
           message: message
         }.merge(data)
-        log_warn(message)
+        log.warn(message)
       end
 
       def load_error(message, data = {})
         @errors << {
           message: message
         }.merge(data)
-        log_error(message)
+        log.error(message)
         raise(Error, message)
       end
 
@@ -480,7 +443,7 @@ module Legion
           nameservers:    config[:nameserver]&.map(&:to_s)&.uniq
         }
       rescue StandardError => e
-        log_warn("Failed to read resolv config: #{e.message}")
+        log.warn("Failed to read resolv config: #{e.message}")
         { search_domains: [], nameservers: [] }
       end
 
@@ -491,10 +454,10 @@ module Legion
 
         fqdn.include?('.') ? fqdn : nil
       rescue Timeout::Error
-        log_debug('FQDN detection skipped (DNS timeout)')
+        log.debug('FQDN detection skipped (DNS timeout)')
         nil
       rescue StandardError => e
-        log_debug("FQDN detection skipped (#{e.message.split(':').first})")
+        log.debug("FQDN detection skipped (#{e.message.split(':').first})")
         nil
       end
     end

data/lib/legion/settings/overlay.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'legion/settings/deep_merge'
+
 module Legion
   module Settings
     # Thread-local request-scoped settings overlay.
@@ -38,6 +40,13 @@ module Legion
           Thread.current[THREAD_KEY]
         end
 
+        # Returns true when a thread-local overlay is active.
+        #
+        # @return [Boolean]
+        def active?
+          !Thread.current[THREAD_KEY].nil?
+        end
+
         # Clear the thread-local overlay for the current thread.
         def clear_overlay!
           Thread.current[THREAD_KEY] = nil
@@ -61,16 +70,7 @@ module Legion
         private
 
         def deep_merge(base, overrides)
-          result = base.dup
-          overrides.each do |key, value|
-            existing = result[key]
-            result[key] = if existing.is_a?(Hash) && value.is_a?(Hash)
-                            deep_merge(existing, value)
-                          else
-                            value
-                          end
-          end
-          result
+          DeepMerge.deep_merge(base, overrides)
         end
       end
     end