swarm_sdk 2.1.3 → 2.2.0

data/lib/swarm_sdk/swarm_registry.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  # Registry for managing sub-swarms in composable swarms
+  #
+  # SwarmRegistry handles lazy loading, caching, and lifecycle management
+  # of child swarms registered via the `swarms` DSL block.
+  #
+  # ## Features
+  # - Lazy loading: Sub-swarms are only loaded when first accessed
+  # - Caching: Loaded swarms are cached for reuse
+  # - Hierarchical IDs: Sub-swarms get IDs based on parent + registration name
+  # - Context control: keep_context determines if swarm state persists
+  # - Lifecycle management: Cleanup cascades through all sub-swarms
+  #
+  # ## Example
+  #
+  #   registry = SwarmRegistry.new(parent_swarm_id: "main_app")
+  #   registry.register("code_review", file: "./swarms/code_review.rb", keep_context: true)
+  #
+  #   # Lazy load on first access
+  #   swarm = registry.load_swarm("code_review")
+  #   # => Swarm with swarm_id = "main_app/code_review"
+  #
+  #   # Reset if keep_context: false
+  #   registry.reset_if_needed("code_review")
+  #
+  class SwarmRegistry
+    # Initialize a new swarm registry
+    #
+    # @param parent_swarm_id [String] ID of the parent swarm
+    def initialize(parent_swarm_id:)
+      @parent_swarm_id = parent_swarm_id
+      @registered_swarms = {}
+      # Format: { "code_review" => { file: "...", keep_context: true, instance: nil } }
+    end
+
+    # Register a sub-swarm for lazy loading
+    #
+    # @param name [String] Registration name for the swarm
+    # @param source [Hash] Source specification with :type and :value
+    #   - { type: :file, value: "./path/to/swarm.rb" }
+    #   - { type: :yaml, value: "version: 2\n..." }
+    #   - { type: :block, value: Proc }
+    # @param keep_context [Boolean] Whether to preserve conversation state between calls (default: true)
+    # @return [void]
+    # @raise [ArgumentError] If swarm with same name already registered
+    def register(name, source:, keep_context: true)
+      raise ArgumentError, "Swarm '#{name}' already registered" if @registered_swarms.key?(name)
+
+      @registered_swarms[name] = {
+        source: source,
+        keep_context: keep_context,
+        instance: nil, # Lazy load
+      }
+    end
+
+    # Check if a swarm is registered
+    #
+    # @param name [String] Swarm registration name
+    # @return [Boolean] True if swarm is registered
+    def registered?(name)
+      @registered_swarms.key?(name)
+    end
+
+    # Load a registered swarm (lazy load + cache)
+    #
+    # Loads the swarm from its source (file, yaml, or block) on first access, then caches it.
+    # Sets hierarchical swarm_id based on parent_swarm_id + registration name.
+    #
+    # @param name [String] Swarm registration name
+    # @return [Swarm] Loaded swarm instance
+    # @raise [ConfigurationError] If swarm not registered
+    def load_swarm(name)
+      entry = @registered_swarms[name]
+      raise ConfigurationError, "Swarm '#{name}' not registered" unless entry
+
+      # Return cached instance if exists
+      return entry[:instance] if entry[:instance]
+
+      # Load from appropriate source
+      swarm_id = "#{@parent_swarm_id}/#{name}" # Hierarchical
+      source = entry[:source]
+
+      swarm = case source[:type]
+      when :file
+        SwarmLoader.load_from_file(
+          source[:value],
+          swarm_id: swarm_id,
+          parent_swarm_id: @parent_swarm_id,
+        )
+      when :yaml
+        SwarmLoader.load_from_yaml_string(
+          source[:value],
+          swarm_id: swarm_id,
+          parent_swarm_id: @parent_swarm_id,
+        )
+      when :block
+        SwarmLoader.load_from_block(
+          source[:value],
+          swarm_id: swarm_id,
+          parent_swarm_id: @parent_swarm_id,
+        )
+      else
+        raise ConfigurationError, "Unknown source type: #{source[:type]}"
+      end
+
+      entry[:instance] = swarm
+      swarm
+    end
+
+    # Reset swarm context if keep_context: false
+    #
+    # @param name [String] Swarm registration name
+    # @return [void]
+    def reset_if_needed(name)
+      entry = @registered_swarms[name]
+      return if entry[:keep_context]
+
+      entry[:instance]&.reset_context!
+    end
+
+    # Cleanup all registered swarms
+    #
+    # Stops all loaded swarm instances and clears the registry.
+    # Should be called when parent swarm is done.
+    #
+    # @return [void]
+    def shutdown_all
+      @registered_swarms.each_value do |entry|
+        entry[:instance]&.cleanup
+      end
+      @registered_swarms.clear
+    end
+  end
+end

data/lib/swarm_sdk/tools/delegate.rb

@@ -14,10 +14,12 @@ module SwarmSDK
       #
       # @param delegate_name [String] Name of the delegate agent (e.g., "backend")
       # @param delegate_description [String] Description of the delegate agent
-      # @param delegate_chat [AgentChat] The chat instance for the delegate agent
+      # @param delegate_chat [AgentChat, nil] The chat instance for the delegate agent (nil if delegating to swarm)
       # @param agent_name [Symbol, String] Name of the agent using this tool
       # @param swarm [Swarm] The swarm instance
       # @param hook_registry [Hooks::Registry] Registry for callbacks
+      # @param call_stack [Array] Delegation call stack for circular dependency detection
+      # @param swarm_registry [SwarmRegistry, nil] Registry for sub-swarms (nil if not using composable swarms)
       # @param delegating_chat [Agent::Chat, nil] The chat instance of the agent doing the delegating (for accessing hooks)
       def initialize(
         delegate_name:,
@@ -26,6 +28,8 @@ module SwarmSDK
         agent_name:,
         swarm:,
         hook_registry:,
+        call_stack:,
+        swarm_registry: nil,
         delegating_chat: nil
       )
         super()
@@ -36,6 +40,8 @@ module SwarmSDK
         @agent_name = agent_name
         @swarm = swarm
         @hook_registry = hook_registry
+        @call_stack = call_stack
+        @swarm_registry = swarm_registry
         @delegating_chat = delegating_chat
 
         # Generate tool name in the expected format: DelegateTaskTo[AgentName]
@@ -63,6 +69,13 @@ module SwarmSDK
       # @param task [String] Task to delegate
       # @return [String] Result from delegate agent or error message
       def execute(task:)
+        # Check for circular dependency
+        if @call_stack.include?(@delegate_target)
+          emit_circular_warning
+          return "Error: Circular delegation detected: #{@call_stack.join(" -> ")} -> #{@delegate_target}. " \
+            "Please restructure your delegation to avoid infinite loops."
+        end
+
         # Get agent-specific hooks from the delegating chat instance
         agent_hooks = if @delegating_chat&.respond_to?(:hook_agent_hooks)
           @delegating_chat.hook_agent_hooks || {}
@@ -94,9 +107,16 @@ module SwarmSDK
           return result.value
         end
 
-        # Proceed with delegation
-        response = @delegate_chat.ask(task)
-        delegation_result = response.content
+        # Determine delegation type and proceed
+        delegation_result = if @delegate_chat
+          # Delegate to agent
+          delegate_to_agent(task)
+        elsif @swarm_registry&.registered?(@delegate_target)
+          # Delegate to registered swarm
+          delegate_to_swarm(task)
+        else
+          raise ConfigurationError, "Unknown delegation target: #{@delegate_target}"
+        end
 
         # Trigger post_delegation callback
         post_context = Hooks::Context.new(
@@ -127,10 +147,12 @@ module SwarmSDK
         LogStream.emit(
           type: "delegation_error",
           agent: @agent_name,
+          swarm_id: @swarm.swarm_id,
+          parent_swarm_id: @swarm.parent_swarm_id,
           delegate_to: @tool_name,
           error_class: e.class.name,
           error_message: "Request timed out",
-          backtrace: e.backtrace&.first(5) || [],
+          error_backtrace: e.backtrace&.first(5) || [],
         )
         "Error: Request to #{@tool_name} timed out. The agent may be overloaded or the LLM service is not responding. Please try again or simplify the task."
       rescue Faraday::Error => e
@@ -138,10 +160,12 @@ module SwarmSDK
         LogStream.emit(
           type: "delegation_error",
           agent: @agent_name,
+          swarm_id: @swarm.swarm_id,
+          parent_swarm_id: @swarm.parent_swarm_id,
           delegate_to: @tool_name,
           error_class: e.class.name,
           error_message: e.message,
-          backtrace: e.backtrace&.first(5) || [],
+          error_backtrace: e.backtrace&.first(5) || [],
         )
         "Error: Network error communicating with #{@tool_name}: #{e.class.name}. Please check connectivity and try again."
       rescue StandardError => e
@@ -150,15 +174,76 @@ module SwarmSDK
         LogStream.emit(
           type: "delegation_error",
           agent: @agent_name,
+          swarm_id: @swarm.swarm_id,
+          parent_swarm_id: @swarm.parent_swarm_id,
           delegate_to: @tool_name,
           error_class: e.class.name,
           error_message: e.message,
-          backtrace: backtrace_array,
+          error_backtrace: backtrace_array,
         )
         # Return error string for LLM
         backtrace_str = backtrace_array.join("\n  ")
         "Error: #{@tool_name} encountered an error: #{e.class.name}: #{e.message}\nBacktrace:\n  #{backtrace_str}"
       end
+
+      private
+
+      # Delegate to an agent
+      #
+      # @param task [String] Task to delegate
+      # @return [String] Result from agent
+      def delegate_to_agent(task)
+        # Push delegate target onto call stack to track delegation chain
+        @call_stack.push(@delegate_target)
+        begin
+          response = @delegate_chat.ask(task)
+          response.content
+        ensure
+          # Always pop from stack, even if delegation fails
+          @call_stack.pop
+        end
+      end
+
+      # Delegate to a registered swarm
+      #
+      # @param task [String] Task to delegate
+      # @return [String] Result from swarm's lead agent
+      def delegate_to_swarm(task)
+        # Load sub-swarm (lazy load + cache)
+        subswarm = @swarm_registry.load_swarm(@delegate_target)
+
+        # Push delegate target onto call stack to track delegation chain
+        @call_stack.push(@delegate_target)
+        begin
+          # Execute sub-swarm's lead agent
+          lead_agent = subswarm.agents[subswarm.lead_agent]
+          response = lead_agent.ask(task)
+          result = response.content
+
+          # Reset if keep_context: false
+          @swarm_registry.reset_if_needed(@delegate_target)
+
+          result
+        ensure
+          # Always pop from stack, even if delegation fails
+          @call_stack.pop
+        end
+      end
+
+      # Emit circular dependency warning event
+      #
+      # @return [void]
+      def emit_circular_warning
+        LogStream.emit(
+          type: "delegation_circular_dependency",
+          agent: @agent_name,
+          swarm_id: @swarm.swarm_id,
+          parent_swarm_id: @swarm.parent_swarm_id,
+          target: @delegate_target,
+          call_stack: @call_stack,
+          timestamp: Time.now.utc.iso8601,
+        )
+      end
     end
   end
 end

data/lib/swarm_sdk/tools/read.rb

@@ -92,25 +92,37 @@ module SwarmSDK
           return validation_error("Path is a directory, not a file. Use Bash with ls to read directories.")
         end
 
-        # Register this read in the tracker (use resolved path)
-        Stores::ReadTracker.register_read(@agent_name, resolved_path)
-
         # Check if it's a document and try to convert it
         converter = find_converter_for_file(resolved_path)
         if converter
           result = converter.new.convert(resolved_path)
+          # For document files, register the converted text content
+          # Extract text from result (may be wrapped in system-reminder tags)
+          if result.is_a?(String)
+            # Remove system-reminder wrapper if present to get clean text for digest
+            text_content = result.gsub(%r{<system-reminder>.*?</system-reminder>}m, "").strip
+            Stores::ReadTracker.register_read(@agent_name, resolved_path, text_content)
+          end
           return result
         end
 
         # Try to read as text, handle binary files separately
         content = read_file_content(resolved_path)
 
-        # If content is a Content object (binary file), return it directly
-        return content if content.is_a?(RubyLLM::Content)
+        # If content is a Content object (binary file), track with binary digest and return
+        if content.is_a?(RubyLLM::Content)
+          # For binary files, read raw bytes for digest
+          binary_content = File.binread(resolved_path)
+          Stores::ReadTracker.register_read(@agent_name, resolved_path, binary_content)
+          return content
+        end
 
         # Return early if we got an error message or system reminder
         return content if content.is_a?(String) && (content.start_with?("Error:") || content.start_with?("<system-reminder>"))
 
+        # At this point, we have valid text content - register the read with digest
+        Stores::ReadTracker.register_read(@agent_name, resolved_path, content)
+
         # Check if file is empty
         if content.empty?
           return format_with_reminder(

data/lib/swarm_sdk/tools/stores/read_tracker.rb

@@ -3,39 +3,74 @@
 module SwarmSDK
   module Tools
     module Stores
-      # ReadTracker manages read-file tracking for all agents
+      # ReadTracker manages read-file tracking for all agents with content digest verification
       #
       # This module maintains a global registry of which files each agent has read
-      # during their conversation. This enables enforcement of the "read-before-write"
-      # and "read-before-edit" rules that ensure agents have context before modifying files.
+      # during their conversation along with SHA256 digests of the content. This enables
+      # enforcement of the "read-before-write" and "read-before-edit" rules that ensure
+      # agents have context before modifying files, AND prevents editing files that have
+      # changed externally since being read.
       #
-      # Each agent maintains an independent set of read files, keyed by agent identifier.
+      # Each agent maintains an independent map of read files to content digests.
       module ReadTracker
-        @read_files = {}
+        @read_files = {} # { agent_id => { file_path => sha256_digest } }
         @mutex = Mutex.new
 
         class << self
-          # Register that an agent has read a file
+          # Register that an agent has read a file with content digest
           #
           # @param agent_id [Symbol] The agent identifier
           # @param file_path [String] The absolute path to the file
-          def register_read(agent_id, file_path)
+          # @param content [String] File content (for digest calculation)
+          # @return [String] The calculated SHA256 digest
+          def register_read(agent_id, file_path, content)
             @mutex.synchronize do
-              @read_files[agent_id] ||= Set.new
-              @read_files[agent_id] << File.expand_path(file_path)
+              @read_files[agent_id] ||= {}
+              digest = Digest::SHA256.hexdigest(content)
+              @read_files[agent_id][File.expand_path(file_path)] = digest
+              digest
             end
           end
 
-          # Check if an agent has read a file
+          # Check if an agent has read a file AND content hasn't changed
           #
           # @param agent_id [Symbol] The agent identifier
           # @param file_path [String] The absolute path to the file
-          # @return [Boolean] true if the agent has read this file
+          # @return [Boolean] true if agent read file and content matches
           def file_read?(agent_id, file_path)
             @mutex.synchronize do
               return false unless @read_files[agent_id]
 
-              @read_files[agent_id].include?(File.expand_path(file_path))
+              expanded_path = File.expand_path(file_path)
+              stored_digest = @read_files[agent_id][expanded_path]
+              return false unless stored_digest
+
+              # Check if file still exists and matches stored digest
+              return false unless File.exist?(expanded_path)
+
+              current_digest = Digest::SHA256.hexdigest(File.read(expanded_path))
+              current_digest == stored_digest
+            end
+          end
+
+          # Get all read files with digests for snapshot
+          #
+          # @param agent_id [Symbol] The agent identifier
+          # @return [Hash] { file_path => digest }
+          def get_read_files(agent_id)
+            @mutex.synchronize do
+              @read_files[agent_id]&.dup || {}
+            end
+          end
+
+          # Restore read files with digests from snapshot
+          #
+          # @param agent_id [Symbol] The agent identifier
+          # @param files_with_digests [Hash] { file_path => digest }
+          # @return [void]
+          def restore_read_files(agent_id, files_with_digests)
+            @mutex.synchronize do
+              @read_files[agent_id] = files_with_digests.dup
             end
           end
 

data/lib/swarm_sdk/tools/stores/scratchpad_storage.rb

@@ -218,6 +218,51 @@ module SwarmSDK
         def size
           @entries.size
         end
+
+        # Get all entries with content for snapshot
+        #
+        # Thread-safe method that returns a copy of all entries.
+        # Used by snapshot/restore functionality.
+        #
+        # @return [Hash] { path => Entry }
+        def all_entries
+          @mutex.synchronize do
+            @entries.dup
+          end
+        end
+
+        # Restore entries from snapshot
+        #
+        # Restores entries directly without using write() to preserve timestamps.
+        # This ensures entry ordering and metadata accuracy after restore.
+        #
+        # @param entries_data [Hash] { path => { content:, title:, updated_at:, size: } }
+        # @return [void]
+        def restore_entries(entries_data)
+          @mutex.synchronize do
+            entries_data.each do |path, data|
+              # Handle both symbol and string keys from JSON
+              content = data[:content] || data["content"]
+              title = data[:title] || data["title"]
+              updated_at_str = data[:updated_at] || data["updated_at"]
+
+              # Parse timestamp from ISO8601 string
+              updated_at = Time.parse(updated_at_str)
+
+              # Create entry with preserved timestamp
+              entry = Entry.new(
+                content: content,
+                title: title,
+                updated_at: updated_at,
+                size: content.bytesize,
+              )
+
+              # Update storage
+              @entries[path] = entry
+              @total_size += entry.size
+            end
+          end
+        end
       end
     end
   end

data/lib/swarm_sdk/utils.rb

@@ -45,6 +45,24 @@ module SwarmSDK
           obj
         end
       end
+
+      # Convert hash to YAML string
+      #
+      # Converts a Ruby hash to a YAML string. Useful for creating inline
+      # swarm definitions from hash configurations.
+      #
+      # @param hash [Hash] Hash to convert
+      # @return [String] YAML string representation
+      #
+      # @example
+      #   config = { version: 2, swarm: { name: "Test" } }
+      #   Utils.hash_to_yaml(config)
+      #   # => "---\nversion: 2\nswarm:\n  name: Test\n"
+      def hash_to_yaml(hash)
+        # Convert symbols to strings for valid YAML
+        stringified = stringify_keys(hash)
+        stringified.to_yaml
+      end
     end
   end
 end

data/lib/swarm_sdk/validation_result.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  # Internal result object for validation phase during snapshot restore
+  #
+  # Used during restore to track which agents can be restored and which
+  # need to be skipped due to configuration mismatches.
+  #
+  # @api private
+  class ValidationResult
+    attr_reader :warnings,
+      :skipped_agents,
+      :restorable_agents,
+      :skipped_delegations,
+      :restorable_delegations
+
+    # Initialize validation result
+    #
+    # @param warnings [Array<Hash>] Warning messages with details
+    # @param skipped_agents [Array<Symbol>] Names of agents that can't be restored
+    # @param restorable_agents [Array<Symbol>] Names of agents that can be restored
+    # @param skipped_delegations [Array<String>] Names of delegations that can't be restored
+    # @param restorable_delegations [Array<String>] Names of delegations that can be restored
+    def initialize(warnings:, skipped_agents:, restorable_agents:,
+      skipped_delegations:, restorable_delegations:)
+      @warnings = warnings
+      @skipped_agents = skipped_agents
+      @restorable_agents = restorable_agents
+      @skipped_delegations = skipped_delegations
+      @restorable_delegations = restorable_delegations
+    end
+  end
+end

data/lib/swarm_sdk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SwarmSDK
-  VERSION = "2.1.3"
+  VERSION = "2.2.0"
 end

data/lib/swarm_sdk.rb

@@ -15,6 +15,7 @@ require "yaml"
 require "async"
 require "async/semaphore"
 require "ruby_llm"
+require "ruby_llm/mcp"
 
 require_relative "swarm_sdk/version"
 
@@ -25,6 +26,8 @@ loader.push_dir("#{__dir__}/swarm_sdk", namespace: SwarmSDK)
 loader.inflector = Zeitwerk::GemInflector.new(__FILE__)
 loader.inflector.inflect(
   "cli" => "CLI",
+  "llm_instrumentation_middleware" => "LLMInstrumentationMiddleware",
+  "mcp" => "MCP",
   "openai_with_responses" => "OpenAIWithResponses",
 )
 loader.setup
@@ -43,8 +46,8 @@ module SwarmSDK
     attr_accessor :settings
 
     # Main entry point for DSL
-    def build(&block)
-      Swarm::Builder.build(&block)
+    def build(allow_filesystem_tools: nil, &block)
+      Swarm::Builder.build(allow_filesystem_tools: allow_filesystem_tools, &block)
     end
 
     # Validate YAML configuration without creating a swarm
@@ -84,6 +87,10 @@ module SwarmSDK
       begin
         config = Configuration.new(yaml_content, base_dir: base_dir)
         config.load_and_validate
+
+        # Build swarm to trigger DSL validation
+        # This catches errors from Agent::Definition, Builder, etc.
+        config.to_swarm
       rescue ConfigurationError, CircularDependencyError => e
         errors << parse_configuration_error(e)
       rescue StandardError => e
@@ -164,10 +171,10 @@ module SwarmSDK
     # @example Load with default base_dir (Dir.pwd)
     #   yaml = File.read("config.yml")
     #   swarm = SwarmSDK.load(yaml)  # base_dir defaults to Dir.pwd
-    def load(yaml_content, base_dir: Dir.pwd)
+    def load(yaml_content, base_dir: Dir.pwd, allow_filesystem_tools: nil)
       config = Configuration.new(yaml_content, base_dir: base_dir)
       config.load_and_validate
-      swarm = config.to_swarm
+      swarm = config.to_swarm(allow_filesystem_tools: allow_filesystem_tools)
 
       # Apply hooks if any are configured (YAML-only feature)
       if hooks_configured?(config)
@@ -196,9 +203,9 @@ module SwarmSDK
     #
     # @example With absolute path
     #   swarm = SwarmSDK.load_file("/absolute/path/config.yml")
-    def load_file(path)
+    def load_file(path, allow_filesystem_tools: nil)
       config = Configuration.load_file(path)
-      swarm = config.to_swarm
+      swarm = config.to_swarm(allow_filesystem_tools: allow_filesystem_tools)
 
       # Apply hooks if any are configured (YAML-only feature)
       if hooks_configured?(config)
@@ -235,7 +242,7 @@ module SwarmSDK
     def hooks_configured?(config)
       config.swarm_hooks.any? ||
         config.all_agents_hooks.any? ||
-        config.agents.any? { |_, agent_def| agent_def.hooks&.any? }
+        config.agents.any? { |_, agent_config| agent_config[:hooks]&.any? }
     end
 
     # Parse configuration error and extract structured information
@@ -323,7 +330,7 @@ module SwarmSDK
           field: "swarm.lead",
         )
 
-      # Unknown agent in connections
+      # Unknown agent in connections (old format)
       when /Agent '([^']+)' has connection to unknown agent '([^']+)'/i
         agent_name = Regexp.last_match(1)
         error_hash.merge!(
@@ -332,6 +339,15 @@ module SwarmSDK
           agent: agent_name,
         )
 
+      # Unknown agent in connections (new format with composable swarms)
+      when /Agent '([^']+)' delegates to unknown target '([^']+)'/i
+        agent_name = Regexp.last_match(1)
+        error_hash.merge!(
+          type: :invalid_reference,
+          field: "swarm.agents.#{agent_name}.delegates_to",
+          agent: agent_name,
+        )
+
       # Circular dependency
       when /Circular dependency detected/i
         error_hash.merge!(
@@ -397,17 +413,33 @@ module SwarmSDK
     # WebFetch tool LLM processing configuration
     attr_accessor :webfetch_provider, :webfetch_model, :webfetch_base_url, :webfetch_max_tokens
 
+    # Filesystem tools control
+    attr_accessor :allow_filesystem_tools
+
     def initialize
       @webfetch_provider = nil
       @webfetch_model = nil
       @webfetch_base_url = nil
       @webfetch_max_tokens = 4096
+      @allow_filesystem_tools = parse_env_bool("SWARM_SDK_ALLOW_FILESYSTEM_TOOLS", default: true)
     end
 
     # Check if WebFetch LLM processing is enabled
     def webfetch_llm_enabled?
       !@webfetch_provider.nil? && !@webfetch_model.nil?
     end
+
+    private
+
+    def parse_env_bool(key, default:)
+      return default unless ENV.key?(key)
+
+      value = ENV[key].to_s.downcase
+      return true if ["true", "yes", "1", "on", "enabled"].include?(value)
+      return false if ["false", "no", "0", "off", "disabled"].include?(value)
+
+      default
+    end
   end
 
   # Initialize default settings