swarm_memory 2.1.2 → 2.1.4

data/lib/swarm_sdk/swarm.rb

@@ -4,25 +4,10 @@ module SwarmSDK
   # Swarm orchestrates multiple AI agents with shared rate limiting and coordination.
   #
   # This is the main user-facing API for SwarmSDK. Users create swarms using:
-  # - Direct API: Create Agent::Definition objects and add to swarm
-  # - Ruby DSL: Use Swarm::Builder for fluent configuration
-  # - YAML: Load from configuration files
-  #
-  # ## Direct API
-  #
-  #   swarm = Swarm.new(name: "Development Team")
-  #
-  #   backend_agent = Agent::Definition.new(:backend, {
-  #     description: "Backend developer",
-  #     model: "gpt-5",
-  #     system_prompt: "You build APIs and databases...",
-  #     tools: [:Read, :Edit, :Bash],
-  #     delegates_to: [:database]
-  #   })
-  #   swarm.add_agent(backend_agent)
-  #
-  #   swarm.lead = :backend
-  #   result = swarm.execute("Build authentication")
+  # - Ruby DSL: SwarmSDK.build { ... } (Recommended)
+  # - YAML String: SwarmSDK.load(yaml, base_dir:)
+  # - YAML File: SwarmSDK.load_file(path)
+  # - Direct API: Swarm.new + add_agent (Advanced)
   #
   # ## Ruby DSL (Recommended)
   #
@@ -39,14 +24,36 @@ module SwarmSDK
   #   end
   #   result = swarm.execute("Build authentication")
   #
-  # ## YAML API
+  # ## YAML String API
+  #
+  #   yaml = File.read("swarm.yml")
+  #   swarm = SwarmSDK.load(yaml, base_dir: "/path/to/project")
+  #   result = swarm.execute("Build authentication")
+  #
+  # ## YAML File API (Convenience)
+  #
+  #   swarm = SwarmSDK.load_file("swarm.yml")
+  #   result = swarm.execute("Build authentication")
+  #
+  # ## Direct API (Advanced)
+  #
+  #   swarm = Swarm.new(name: "Development Team")
+  #
+  #   backend_agent = Agent::Definition.new(:backend, {
+  #     description: "Backend developer",
+  #     model: "gpt-5",
+  #     system_prompt: "You build APIs and databases...",
+  #     tools: [:Read, :Edit, :Bash],
+  #     delegates_to: [:database]
+  #   })
+  #   swarm.add_agent(backend_agent)
   #
-  #   swarm = Swarm.load("swarm.yml")
+  #   swarm.lead = :backend
   #   result = swarm.execute("Build authentication")
   #
   # ## Architecture
   #
-  # All three APIs converge on Agent::Definition for validation.
+  # All APIs converge on Agent::Definition for validation.
   # Swarm delegates to specialized concerns:
   # - Agent::Definition: Validates configuration, builds system prompts
   # - AgentInitializer: Complex 5-pass agent setup
@@ -54,23 +61,42 @@ module SwarmSDK
   # - McpConfigurator: MCP client management (via AgentInitializer)
   #
   class Swarm
-    DEFAULT_GLOBAL_CONCURRENCY = 50
-    DEFAULT_LOCAL_CONCURRENCY = 10
-    DEFAULT_MCP_LOG_LEVEL = Logger::WARN
+    include Concerns::Snapshotable
+    include Concerns::Validatable
+    include Concerns::Cleanupable
+    include LoggingCallbacks
+    include HookTriggers
+
+    # Backward compatibility aliases - use Defaults module for new code
+    DEFAULT_MCP_LOG_LEVEL = Defaults::Logging::MCP_LOG_LEVEL
 
     # Default tools available to all agents
     DEFAULT_TOOLS = ToolConfigurator::DEFAULT_TOOLS
 
-    attr_reader :name, :agents, :lead_agent, :mcp_clients
+    attr_reader :name, :agents, :lead_agent, :mcp_clients, :delegation_instances, :agent_definitions, :swarm_id, :parent_swarm_id, :swarm_registry, :scratchpad_storage, :allow_filesystem_tools, :hook_registry, :global_semaphore, :plugin_storages, :config_for_hooks, :observer_configs
+    attr_accessor :delegation_call_stack
 
     # Check if scratchpad tools are enabled
     #
     # @return [Boolean]
     def scratchpad_enabled?
-      @scratchpad_enabled
+      @scratchpad_mode == :enabled
     end
     attr_writer :config_for_hooks
 
+    # Check if first message has been sent (for system reminder injection)
+    #
+    # @return [Boolean]
+    def first_message_sent?
+      @first_message_sent
+    end
+
+    # Set first message sent flag (used by snapshot/restore)
+    #
+    # @param value [Boolean] New value
+    # @return [void]
+    attr_writer :first_message_sent
+
     # Class-level MCP log level configuration
     @mcp_log_level = DEFAULT_MCP_LOG_LEVEL
     @mcp_logging_configured = false
@@ -102,54 +128,54 @@ module SwarmSDK
 
         @mcp_logging_configured = true
       end
-
-      # Load swarm from YAML configuration file
-      #
-      # @param config_path [String] Path to YAML configuration file
-      # @return [Swarm] Configured swarm instance
-      def load(config_path)
-        config = Configuration.load(config_path)
-        swarm = config.to_swarm
-
-        # Apply hooks if any are configured (YAML-only feature)
-        if hooks_configured?(config)
-          Hooks::Adapter.apply_hooks(swarm, config)
-        end
-
-        # Store config reference for agent hooks (applied during initialize_agents)
-        swarm.config_for_hooks = config
-
-        swarm
-      end
-
-      private
-
-      def hooks_configured?(config)
-        config.swarm_hooks.any? ||
-          config.all_agents_hooks.any? ||
-          config.agents.any? { |_, agent_def| agent_def.hooks&.any? }
-      end
     end
 
     # Initialize a new Swarm
     #
     # @param name [String] Human-readable swarm name
+    # @param swarm_id [String, nil] Optional swarm ID (auto-generated if not provided)
+    # @param parent_swarm_id [String, nil] Optional parent swarm ID (nil for root swarms)
     # @param global_concurrency [Integer] Max concurrent LLM calls across entire swarm
     # @param default_local_concurrency [Integer] Default max concurrent tool calls per agent
-    # @param scratchpad [Tools::Stores::Scratchpad, nil] Optional scratchpad instance (for testing)
-    # @param scratchpad_enabled [Boolean] Whether to enable scratchpad tools (default: true)
-    def initialize(name:, global_concurrency: DEFAULT_GLOBAL_CONCURRENCY, default_local_concurrency: DEFAULT_LOCAL_CONCURRENCY, scratchpad: nil, scratchpad_enabled: true)
+    # @param scratchpad [Tools::Stores::Scratchpad, nil] Optional scratchpad instance (for testing/internal use)
+    # @param scratchpad_mode [Symbol, String] Scratchpad mode (:enabled or :disabled). :per_node not allowed for non-node swarms.
+    # @param allow_filesystem_tools [Boolean, nil] Whether to allow filesystem tools (nil uses global setting)
+    def initialize(name:, swarm_id: nil, parent_swarm_id: nil, global_concurrency: Defaults::Concurrency::GLOBAL_LIMIT, default_local_concurrency: Defaults::Concurrency::LOCAL_LIMIT, scratchpad: nil, scratchpad_mode: :enabled, allow_filesystem_tools: nil)
       @name = name
+      @swarm_id = swarm_id || generate_swarm_id(name)
+      @parent_swarm_id = parent_swarm_id
       @global_concurrency = global_concurrency
       @default_local_concurrency = default_local_concurrency
-      @scratchpad_enabled = scratchpad_enabled
+
+      # Handle scratchpad_mode parameter
+      # For Swarm: :enabled or :disabled (not :per_node - that's for nodes)
+      @scratchpad_mode = validate_swarm_scratchpad_mode(scratchpad_mode)
+
+      # Resolve allow_filesystem_tools with priority:
+      # 1. Explicit parameter (if not nil)
+      # 2. Global settings
+      @allow_filesystem_tools = if allow_filesystem_tools.nil?
+        SwarmSDK.settings.allow_filesystem_tools
+      else
+        allow_filesystem_tools
+      end
+
+      # Swarm registry for managing sub-swarms (initialized later if needed)
+      @swarm_registry = nil
+
+      # Delegation call stack for circular dependency detection
+      @delegation_call_stack = []
 
       # Shared semaphore for all agents
       @global_semaphore = Async::Semaphore.new(@global_concurrency)
 
       # Shared scratchpad storage for all agents (volatile)
-      # Use provided scratchpad storage (for testing) or create volatile one
-      @scratchpad_storage = scratchpad || Tools::Stores::ScratchpadStorage.new
+      # Use provided scratchpad storage (for testing) or create volatile one based on mode
+      @scratchpad_storage = if scratchpad
+        scratchpad # Testing/internal use - explicit instance provided
+      elsif @scratchpad_mode == :enabled
+        Tools::Stores::ScratchpadStorage.new
+      end
 
       # Per-agent plugin storages (persistent)
       # Format: { plugin_name => { agent_name => storage } }
@@ -165,6 +191,7 @@ module SwarmSDK
       # Agent definitions and instances
       @agent_definitions = {}
       @agents = {}
+      @delegation_instances = {} # { "delegate@delegator" => Agent::Chat }
       @agents_initialized = false
       @agent_contexts = {}
 
@@ -175,6 +202,14 @@ module SwarmSDK
 
       # Track if first message has been sent
       @first_message_sent = false
+
+      # Track if agent_start events have been emitted
+      # This prevents duplicate emissions and ensures events are emitted when logging is ready
+      @agent_start_events_emitted = false
+
+      # Observer agent configurations
+      @observer_configs = []
+      @observer_manager = nil
     end
 
     # Add an agent to the swarm
@@ -230,36 +265,53 @@ module SwarmSDK
     # and the entire swarm coordinates with shared rate limiting.
     # Supports reprompting via swarm_stop hooks.
     #
+    # By default, this method blocks until execution completes. Set wait: false
+    # to return an Async::Task immediately, enabling cancellation via task.stop.
+    #
     # @param prompt [String] Task to execute
+    # @param wait [Boolean] If true (default), blocks until execution completes.
+    #   If false, returns Async::Task immediately for non-blocking execution.
     # @yield [Hash] Log entry if block given (for streaming)
-    # @return [Result] Execution result
-    def execute(prompt, &block)
+    # @return [Result, Async::Task] Result if wait: true, Async::Task if wait: false
+    #
+    # @example Blocking execution (default)
+    #   result = swarm.execute("Build auth")
+    #   puts result.content
+    #
+    # @example Non-blocking execution with cancellation
+    #   task = swarm.execute("Build auth", wait: false) { |event| puts event }
+    #   # ... do other work ...
+    #   task.stop  # Cancel anytime
+    #   result = task.wait  # Returns nil for cancelled tasks
+    def execute(prompt, wait: true, &block)
       raise ConfigurationError, "No lead agent set. Set lead= first." unless @lead_agent
 
-      start_time = Time.now
       logs = []
       current_prompt = prompt
+      has_logging = block_given?
+
+      # Save original Fiber storage for restoration (preserves parent context for nested swarms)
+      original_fiber_storage = {
+        execution_id: Fiber[:execution_id],
+        swarm_id: Fiber[:swarm_id],
+        parent_swarm_id: Fiber[:parent_swarm_id],
+      }
+
+      # Set fiber-local execution context
+      # Use ||= to inherit parent's execution_id if one exists (for mini-swarms)
+      Fiber[:execution_id] ||= generate_execution_id
+      Fiber[:swarm_id] = @swarm_id
+      Fiber[:parent_swarm_id] = @parent_swarm_id
 
       # Setup logging FIRST if block given (so swarm_start event can be emitted)
-      if block_given?
-        # Register callback to collect logs and forward to user's block
-        LogCollector.on_log do |entry|
-          logs << entry
-          block.call(entry)
-        end
+      setup_logging(logs, &block) if has_logging
 
-        # Set LogStream to use LogCollector as emitter
-        LogStream.emitter = LogCollector
-      end
+      # Setup observer execution if any observers configured
+      # MUST happen AFTER setup_logging (which clears Fiber[:log_subscriptions])
+      setup_observer_execution if @observer_configs.any?
 
       # Trigger swarm_start hooks (before any execution)
-      # Hook can append stdout to prompt (exit code 0)
-      # Default callback emits swarm_start event to LogStream
-      swarm_start_result = trigger_swarm_start(current_prompt)
-      if swarm_start_result&.replace?
-        # Hook provided stdout to append to prompt
-        current_prompt = "#{current_prompt}\n\n<hook-context>\n#{swarm_start_result.value}\n</hook-context>"
-      end
+      current_prompt = apply_swarm_start_hooks(current_prompt)
 
       # Trigger first_message hooks on first execution
       unless @first_message_sent
@@ -270,128 +322,18 @@ module SwarmSDK
       # Lazy initialization of agents (with optional logging)
       initialize_agents unless @agents_initialized
 
-      # Execution loop (supports reprompting)
-      result = nil
-      swarm_stop_triggered = false
-
-      loop do
-        # Execute within Async reactor to enable fiber scheduler for parallel execution
-        # This sets Fiber.scheduler, making Faraday fiber-aware so HTTP requests yield during I/O
-        # Use finished: false to suppress warnings for expected task failures
-        lead = @agents[@lead_agent]
-        response = Async(finished: false) do
-          lead.ask(current_prompt)
-        end.wait
-
-        # Check if swarm was finished by a hook (finish_swarm)
-        if response.is_a?(Hash) && response[:__finish_swarm__]
-          result = Result.new(
-            content: response[:message],
-            agent: @lead_agent.to_s,
-            logs: logs,
-            duration: Time.now - start_time,
-          )
-
-          # Trigger swarm_stop hooks for event emission
-          trigger_swarm_stop(result)
-          swarm_stop_triggered = true
-
-          # Break immediately - don't allow reprompting when swarm is finished by hook
-          break
-        end
+      # Emit agent_start events if agents were initialized before logging was set up
+      emit_retroactive_agent_start_events if has_logging
 
-        result = Result.new(
-          content: response.content,
-          agent: @lead_agent.to_s,
-          logs: logs,
-          duration: Time.now - start_time,
-        )
-
-        # Trigger swarm_stop hooks (for reprompt check and event emission)
-        hook_result = trigger_swarm_stop(result)
-        swarm_stop_triggered = true
-
-        # Check if hook requests reprompting
-        if hook_result&.reprompt?
-          current_prompt = hook_result.value
-          swarm_stop_triggered = false # Will trigger again in next iteration
-          # Continue loop with new prompt
-        else
-          # Exit loop - execution complete
-          break
-        end
-      end
-
-      result
-    rescue ConfigurationError, AgentNotFoundError
-      # Re-raise configuration errors - these should be fixed, not caught
-      raise
-    rescue TypeError => e
-      # Catch the specific "String does not have #dig method" error
-      if e.message.include?("does not have #dig method")
-        agent_definition = @agent_definitions[@lead_agent]
-        error_msg = if agent_definition.base_url
-          "LLM API request failed: The proxy/server at '#{agent_definition.base_url}' returned an invalid response. " \
-            "This usually means the proxy is unreachable, requires authentication, or returned an error in non-JSON format. " \
-            "Original error: #{e.message}"
-        else
-          "LLM API request failed with unexpected response format. Original error: #{e.message}"
-        end
-
-        result = Result.new(
-          content: nil,
-          agent: @lead_agent.to_s,
-          error: LLMError.new(error_msg),
-          logs: logs,
-          duration: Time.now - start_time,
-        )
-      else
-        result = Result.new(
-          content: nil,
-          agent: @lead_agent.to_s,
-          error: e,
-          logs: logs,
-          duration: Time.now - start_time,
-        )
-      end
-      result
-    rescue StandardError => e
-      result = Result.new(
-        content: nil,
-        agent: @lead_agent&.to_s || "unknown",
-        error: e,
+      # Delegate to Executor for actual execution
+      executor = Executor.new(self)
+      @current_task = executor.run(
+        current_prompt,
+        wait: wait,
         logs: logs,
-        duration: Time.now - start_time,
+        has_logging: has_logging,
+        original_fiber_storage: original_fiber_storage,
       )
-      result
-    ensure
-      # Trigger swarm_stop if not already triggered (handles error cases)
-      unless swarm_stop_triggered
-        trigger_swarm_stop_final(result, start_time, logs)
-      end
-
-      # Cleanup MCP clients after execution
-      cleanup
-
-      # Reset logging state for next execution if we set it up
-      #
-      # IMPORTANT: Only reset if we set up logging (block_given? == true).
-      # When this swarm is a mini-swarm within a NodeOrchestrator workflow,
-      # the orchestrator manages LogCollector and we don't set up logging.
-      #
-      # Flow in NodeOrchestrator:
-      # 1. NodeOrchestrator sets up LogCollector + LogStream (no block given to mini-swarms)
-      # 2. Each mini-swarm executes without logging block (block_given? == false)
-      # 3. Each mini-swarm skips reset (didn't set up logging)
-      # 4. NodeOrchestrator resets once at the very end
-      #
-      # Flow in standalone swarm / interactive REPL:
-      # 1. Swarm.execute sets up LogCollector + LogStream (block given)
-      # 2. Swarm.execute resets in ensure block (cleanup for next call)
-      if block_given?
-        LogCollector.reset!
-        LogStream.reset!
-      end
     end
 
     # Get an agent chat instance by name
@@ -425,77 +367,17 @@ module SwarmSDK
       @agent_definitions.keys
     end
 
-    # Validate swarm configuration and return warnings
-    #
-    # This performs lightweight validation checks without creating agents.
-    # Useful for displaying configuration warnings before execution.
-    #
-    # @return [Array<Hash>] Array of warning hashes from all agent definitions
-    #
-    # @example
-    #   swarm = Swarm.load("config.yml")
-    #   warnings = swarm.validate
-    #   warnings.each do |warning|
-    #     puts "⚠️  #{warning[:agent]}: #{warning[:model]} not found"
-    #   end
-    def validate
-      @agent_definitions.flat_map { |_name, definition| definition.validate }
+    # Implement Snapshotable interface
+    def primary_agents
+      @agents
     end
 
-    # Emit validation warnings as log events
-    #
-    # This validates all agent definitions and emits any warnings as
-    # model_lookup_warning events through LogStream. Useful for emitting
-    # warnings before execution starts (e.g., in REPL after welcome screen).
-    #
-    # Requires LogStream.emitter to be set.
-    #
-    # @return [Array<Hash>] The validation warnings that were emitted
-    #
-    # @example
-    #   LogCollector.on_log { |event| puts event }
-    #   LogStream.emitter = LogCollector
-    #   swarm.emit_validation_warnings
-    def emit_validation_warnings
-      warnings = validate
-
-      warnings.each do |warning|
-        case warning[:type]
-        when :model_not_found
-          LogStream.emit(
-            type: "model_lookup_warning",
-            agent: warning[:agent],
-            model: warning[:model],
-            error_message: warning[:error_message],
-            suggestions: warning[:suggestions],
-            timestamp: Time.now.utc.iso8601,
-          )
-        end
-      end
-
-      warnings
+    def delegation_instances_hash
+      @delegation_instances
     end
 
-    # Cleanup all MCP clients
-    #
-    # Stops all MCP client connections gracefully.
-    # Should be called when the swarm is no longer needed.
-    #
-    # @return [void]
-    def cleanup
-      return if @mcp_clients.empty?
-
-      @mcp_clients.each do |agent_name, clients|
-        clients.each do |client|
-          client.stop if client.alive?
-          RubyLLM.logger.debug("SwarmSDK: Stopped MCP client '#{client.name}' for agent #{agent_name}")
-        rescue StandardError => e
-          RubyLLM.logger.error("SwarmSDK: Error stopping MCP client '#{client.name}' for agent #{agent_name}: #{e.message}")
-        end
-      end
-
-      @mcp_clients.clear
-    end
+    # NOTE: validate() and emit_validation_warnings() are provided by Concerns::Validatable
+    # Note: cleanup() is provided by Concerns::Cleanupable
 
     # Register a named hook that can be referenced in agent configurations
     #
@@ -515,28 +397,229 @@ module SwarmSDK
       self
     end
 
-    # Add a swarm-level default hook that applies to all agents
+    # Reset context for all agents
     #
-    # Default hooks are inherited by all agents unless overridden at agent level.
-    # Useful for swarm-wide policies like logging, validation, or monitoring.
+    # Clears conversation history for all agents. This is used by composable swarms
+    # to reset sub-swarm context when keep_context: false is specified.
     #
-    # @param event [Symbol] Event type (e.g., :pre_tool_use, :post_tool_use)
-    # @param matcher [String, Regexp, nil] Optional regex pattern for tool names
-    # @param priority [Integer] Execution priority (higher = earlier)
-    # @param block [Proc] Hook implementation
-    # @return [self]
+    # @return [void]
+    def reset_context!
+      @agents.each_value do |agent_chat|
+        agent_chat.clear_conversation if agent_chat.respond_to?(:clear_conversation)
+      end
+    end
+
+    # Add observer configuration
+    #
+    # Called by Swarm::Builder to register observer agent configurations.
+    # Validates that the referenced agent exists.
+    #
+    # @param config [Observer::Config] Observer configuration
+    # @return [void]
+    def add_observer_config(config)
+      validate_observer_agent(config.agent_name)
+      @observer_configs << config
+    end
+
+    # Wait for all observer tasks to complete
+    #
+    # Called by Executor to wait for observer agents before cleanup.
+    # Safe to call even if no observers are configured.
     #
-    # @example Add logging for all tool calls
-    #   swarm.add_default_callback(:pre_tool_use) do |context|
-    #     puts "[#{context.agent_name}] Calling #{context.tool_call.name}"
+    # @return [void]
+    def wait_for_observers
+      @observer_manager&.wait_for_completion
+    end
+
+    # Cleanup observer subscriptions
+    #
+    # Called by Executor.cleanup_after_execution to unsubscribe observers.
+    # Matches the MCP cleanup pattern.
+    #
+    # @return [void]
+    def cleanup_observers
+      @observer_manager&.cleanup
+      @observer_manager = nil
+    end
+
+    # Create snapshot of current conversation state
+    #
+    # Returns a Snapshot object containing:
+    # - All agent conversations (@messages arrays)
+    # - Agent context state (warnings, compression, TodoWrite tracking, skills)
+    # - Delegation instance conversations
+    # - Scratchpad contents (volatile shared storage)
+    # - Read tracking state (which files each agent has read with digests)
+    # - Memory read tracking state (which memory entries each agent has read with digests)
+    #
+    # Configuration (agent definitions, tools, prompts) stays in your YAML/DSL
+    # and is NOT included in snapshots.
+    #
+    # @return [Snapshot] Snapshot object with convenient serialization methods
+    #
+    # @example Save snapshot to JSON file
+    #   snapshot = swarm.snapshot
+    #   snapshot.write_to_file("session.json")
+    #
+    # @example Convert to hash or JSON string
+    #   snapshot = swarm.snapshot
+    #   hash = snapshot.to_hash
+    #   json_string = snapshot.to_json
+    def snapshot
+      StateSnapshot.new(self).snapshot
+    end
+
+    # Restore conversation state from snapshot
+    #
+    # Accepts a Snapshot object, hash, or JSON string. Validates compatibility
+    # between snapshot and current swarm configuration, restores agent conversations,
+    # context state, scratchpad, and read tracking. Returns RestoreResult with
+    # warnings about any agents that couldn't be restored due to configuration
+    # mismatches.
+    #
+    # The swarm must be created with the SAME configuration (agent definitions,
+    # tools, prompts) as when the snapshot was created. Only conversation state
+    # is restored from the snapshot.
+    #
+    # @param snapshot [Snapshot, Hash, String] Snapshot object, hash, or JSON string
+    # @return [RestoreResult] Result with warnings about skipped agents
+    #
+    # @example Restore from Snapshot object
+    #   swarm = SwarmSDK.build { ... }  # Same config as snapshot
+    #   snapshot = Snapshot.from_file("session.json")
+    #   result = swarm.restore(snapshot)
+    #   if result.success?
+    #     puts "All agents restored"
+    #   else
+    #     puts result.summary
+    #     result.warnings.each { |w| puts "  - #{w[:message]}" }
     #   end
-    def add_default_callback(event, matcher: nil, priority: 0, &block)
-      @hook_registry.add_default(event, matcher: matcher, priority: priority, &block)
-      self
+    #
+    # Restore swarm state from snapshot
+    #
+    # By default, uses current system prompts from agent definitions (YAML + SDK defaults + plugin injections).
+    # Set preserve_system_prompts: true to use historical prompts from snapshot.
+    #
+    # @param snapshot [Snapshot, Hash, String] Snapshot object, hash, or JSON string
+    # @param preserve_system_prompts [Boolean] Use historical system prompts instead of current config (default: false)
+    # @return [RestoreResult] Result with warnings about partial restores
+    def restore(snapshot, preserve_system_prompts: false)
+      StateRestorer.new(self, snapshot, preserve_system_prompts: preserve_system_prompts).restore
+    end
+
+    # Override swarm IDs for composable swarms
+    #
+    # Used by SwarmLoader to set hierarchical IDs when loading sub-swarms.
+    # This is called after the swarm is built to ensure proper parent/child relationships.
+    #
+    # @param swarm_id [String] New swarm ID
+    # @param parent_swarm_id [String] New parent swarm ID
+    # @return [void]
+    def override_swarm_ids(swarm_id:, parent_swarm_id:)
+      @swarm_id = swarm_id
+      @parent_swarm_id = parent_swarm_id
     end
 
+    # Set swarm registry for composable swarms
+    #
+    # Used by Builder to set the registry after swarm creation.
+    # This must be called before agent initialization to enable swarm delegation.
+    #
+    # @param registry [SwarmRegistry] Configured swarm registry
+    # @return [void]
+    attr_writer :swarm_registry
+
+    # --- Internal API (for Executor use only) ---
+    # Hook triggers for swarm lifecycle events are provided by HookTriggers module
+
     private
 
+    # Apply swarm_start hooks to prompt
+    #
+    # @param prompt [String] Original prompt
+    # @return [String] Modified prompt (possibly with hook context appended)
+    def apply_swarm_start_hooks(prompt)
+      swarm_start_result = trigger_swarm_start(prompt)
+      if swarm_start_result&.replace?
+        "#{prompt}\n\n<hook-context>\n#{swarm_start_result.value}\n</hook-context>"
+      else
+        prompt
+      end
+    end
+
+    # Validate that observer agent exists
+    #
+    # @param agent_name [Symbol] Name of the observer agent
+    # @raise [ConfigurationError] If agent not found
+    # @return [void]
+    def validate_observer_agent(agent_name)
+      return if @agent_definitions.key?(agent_name)
+
+      raise ConfigurationError,
+        "Observer agent '#{agent_name}' not found. " \
+          "Define the agent first with `agent :#{agent_name} do ... end`"
+    end
+
+    # Setup observer manager and subscriptions
+    #
+    # Creates Observer::Manager and registers event subscriptions.
+    # Must be called AFTER setup_logging (which clears Fiber[:log_subscriptions]).
+    #
+    # @return [void]
+    def setup_observer_execution
+      @observer_manager = Observer::Manager.new(self)
+      @observer_configs.each { |c| @observer_manager.add_config(c) }
+      @observer_manager.setup
+    end
+
+    # Validate and normalize scratchpad mode for Swarm
+    #
+    # Regular Swarms support :enabled or :disabled.
+    # Rejects :per_node since it only makes sense for Workflow with multiple nodes.
+    #
+    # @param value [Symbol, String] Scratchpad mode (strings from YAML converted to symbols)
+    # @return [Symbol] :enabled or :disabled
+    # @raise [ArgumentError] If :per_node used, or invalid value
+    def validate_swarm_scratchpad_mode(value)
+      # Convert strings from YAML to symbols
+      value = value.to_sym if value.is_a?(String)
+
+      case value
+      when :enabled, :disabled
+        value
+      when :per_node
+        raise ArgumentError,
+          "scratchpad: :per_node is only valid for Workflow with nodes. " \
+            "For regular Swarms, use :enabled or :disabled."
+      else
+        raise ArgumentError,
+          "Invalid scratchpad mode for Swarm: #{value.inspect}. " \
+            "Use :enabled or :disabled."
+      end
+    end
+
+    # Generate a unique swarm ID from name
+    #
+    # Creates a swarm ID by sanitizing the name and appending a random suffix.
+    # Used when swarm_id is not explicitly provided.
+    #
+    # @param name [String] Swarm name
+    # @return [String] Generated swarm ID (e.g., "dev_team_a3f2b1c8")
+    def generate_swarm_id(name)
+      sanitized = name.to_s.gsub(/[^a-z0-9_-]/i, "_").downcase
+      "#{sanitized}_#{SecureRandom.hex(4)}"
+    end
+
+    # Generate a unique execution ID
+    #
+    # Creates an execution ID that uniquely identifies a single swarm.execute() call.
+    # Format: "exec_{swarm_id}_{random_hex}"
+    #
+    # @return [String] Generated execution ID (e.g., "exec_main_a3f2b1c8")
+    def generate_execution_id
+      "exec_#{@swarm_id}_#{SecureRandom.hex(8)}"
+    end
+
     # Initialize all agents using AgentInitializer
     #
     # This is called automatically (lazy initialization) by execute() and agent().
@@ -546,58 +629,14 @@ module SwarmSDK
     def initialize_agents
       return if @agents_initialized
 
-      initializer = AgentInitializer.new(
-        self,
-        @agent_definitions,
-        @global_semaphore,
-        @hook_registry,
-        @scratchpad_storage,
-        @plugin_storages,
-        config_for_hooks: @config_for_hooks,
-      )
+      initializer = AgentInitializer.new(self)
 
       @agents = initializer.initialize_all
       @agent_contexts = initializer.agent_contexts
       @agents_initialized = true
 
-      # Emit agent_start events for all agents
-      emit_agent_start_events
-    end
-
-    # Emit agent_start events for all initialized agents
-    def emit_agent_start_events
-      # Only emit if LogStream is enabled
-      return unless LogStream.emitter
-
-      @agents.each do |agent_name, chat|
-        agent_def = @agent_definitions[agent_name]
-
-        # Build plugin storage info for logging
-        plugin_storage_info = {}
-        @plugin_storages.each do |plugin_name, agent_storages|
-          next unless agent_storages.key?(agent_name)
-
-          plugin_storage_info[plugin_name] = {
-            enabled: true,
-            # Get additional info from agent definition if available
-            config: agent_def.respond_to?(plugin_name) ? extract_plugin_config_info(agent_def.public_send(plugin_name)) : nil,
-          }
-        end
-
-        LogStream.emit(
-          type: "agent_start",
-          agent: agent_name,
-          swarm_name: @name,
-          model: agent_def.model,
-          provider: agent_def.provider || "openai",
-          directory: agent_def.directory,
-          system_prompt: agent_def.system_prompt,
-          tools: chat.tools.keys,
-          delegates_to: agent_def.delegates_to,
-          plugin_storages: plugin_storage_info,
-          timestamp: Time.now.utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
-        )
-      end
+      # NOTE: agent_start events are emitted in execute() when logging is set up
+      # This ensures events are never lost, even if agents are initialized early (e.g., by restore())
     end
 
     # Normalize tools to internal format (kept for add_agent)
@@ -647,7 +686,7 @@ module SwarmSDK
 
     # Create delegation tool (delegates to AgentInitializer)
     def create_delegation_tool(name:, description:, delegate_chat:, agent_name:)
-      AgentInitializer.new(self, @agent_definitions, @global_semaphore, @hook_registry, @scratchpad_storage, @plugin_storages)
+      AgentInitializer.new(self)
         .create_delegation_tool(name: name, description: description, delegate_chat: delegate_chat, agent_name: agent_name)
     end
 
@@ -674,309 +713,5 @@ module SwarmSDK
       # Unknown config type
       nil
     end
-
-    # Register default logging hooks that emit LogStream events
-    #
-    # These hooks implement the standard SwarmSDK logging behavior.
-    # Users can override or extend them by registering their own hooks.
-    #
-    # @return [void]
-    def register_default_logging_callbacks
-      # Log swarm start
-      add_default_callback(:swarm_start, priority: -100) do |context|
-        # Only log if LogStream emitter is set (logging enabled)
-        next unless LogStream.emitter
-
-        LogStream.emit(
-          type: "swarm_start",
-          agent: context.metadata[:lead_agent], # Include agent for consistency
-          swarm_name: context.metadata[:swarm_name],
-          lead_agent: context.metadata[:lead_agent],
-          prompt: context.metadata[:prompt],
-          timestamp: context.metadata[:timestamp],
-        )
-      end
-
-      # Log swarm stop
-      add_default_callback(:swarm_stop, priority: -100) do |context|
-        # Only log if LogStream emitter is set (logging enabled)
-        next unless LogStream.emitter
-
-        LogStream.emit(
-          type: "swarm_stop",
-          swarm_name: context.metadata[:swarm_name],
-          lead_agent: context.metadata[:lead_agent],
-          last_agent: context.metadata[:last_agent], # Agent that produced final response
-          content: context.metadata[:content], # Final response content
-          success: context.metadata[:success],
-          duration: context.metadata[:duration],
-          total_cost: context.metadata[:total_cost],
-          total_tokens: context.metadata[:total_tokens],
-          agents_involved: context.metadata[:agents_involved],
-          timestamp: context.metadata[:timestamp],
-        )
-      end
-
-      # Log user requests
-      add_default_callback(:user_prompt, priority: -100) do |context|
-        # Only log if LogStream emitter is set (logging enabled)
-        next unless LogStream.emitter
-
-        LogStream.emit(
-          type: "user_prompt",
-          agent: context.agent_name,
-          model: context.metadata[:model] || "unknown",
-          provider: context.metadata[:provider] || "unknown",
-          message_count: context.metadata[:message_count] || 0,
-          tools: context.metadata[:tools] || [],
-          delegates_to: context.metadata[:delegates_to] || [],
-          metadata: context.metadata,
-        )
-      end
-
-      # Log intermediate agent responses with tool calls
-      add_default_callback(:agent_step, priority: -100) do |context|
-        # Only log if LogStream emitter is set (logging enabled)
-        next unless LogStream.emitter
-
-        # Extract top-level fields and remove from metadata to avoid duplication
-        metadata_without_duplicates = context.metadata.except(:model, :content, :tool_calls, :finish_reason, :usage, :tool_executions)
-
-        LogStream.emit(
-          type: "agent_step",
-          agent: context.agent_name,
-          model: context.metadata[:model],
-          content: context.metadata[:content],
-          tool_calls: context.metadata[:tool_calls],
-          finish_reason: context.metadata[:finish_reason],
-          usage: context.metadata[:usage],
-          tool_executions: context.metadata[:tool_executions],
-          metadata: metadata_without_duplicates,
-        )
-      end
-
-      # Log final agent responses
-      add_default_callback(:agent_stop, priority: -100) do |context|
-        # Only log if LogStream emitter is set (logging enabled)
-        next unless LogStream.emitter
-
-        # Extract top-level fields and remove from metadata to avoid duplication
-        metadata_without_duplicates = context.metadata.except(:model, :content, :tool_calls, :finish_reason, :usage, :tool_executions)
-
-        LogStream.emit(
-          type: "agent_stop",
-          agent: context.agent_name,
-          model: context.metadata[:model],
-          content: context.metadata[:content],
-          tool_calls: context.metadata[:tool_calls],
-          finish_reason: context.metadata[:finish_reason],
-          usage: context.metadata[:usage],
-          tool_executions: context.metadata[:tool_executions],
-          metadata: metadata_without_duplicates,
-        )
-      end
-
-      # Log tool calls (pre_tool_use)
-      add_default_callback(:pre_tool_use, priority: -100) do |context|
-        # Only log if LogStream emitter is set (logging enabled)
-        next unless LogStream.emitter
-
-        # Delegation tracking is handled separately in AgentChat
-        # Just log the tool call - delegation info will be in metadata if needed
-        LogStream.emit(
-          type: "tool_call",
-          agent: context.agent_name,
-          tool_call_id: context.tool_call.id,
-          tool: context.tool_call.name,
-          arguments: context.tool_call.parameters,
-          metadata: context.metadata,
-        )
-      end
-
-      # Log tool results (post_tool_use)
-      add_default_callback(:post_tool_use, priority: -100) do |context|
-        # Only log if LogStream emitter is set (logging enabled)
-        next unless LogStream.emitter
-
-        # Delegation tracking is handled separately in AgentChat
-        # Usage tracking is handled in agent_step/agent_stop events
-        LogStream.emit(
-          type: "tool_result",
-          agent: context.agent_name,
-          tool_call_id: context.tool_result.tool_call_id,
-          tool: context.tool_result.tool_name,
-          result: context.tool_result.content,
-          metadata: context.metadata,
-        )
-      end
-
-      # Log context warnings
-      add_default_callback(:context_warning, priority: -100) do |context|
-        # Only log if LogStream emitter is set (logging enabled)
-        next unless LogStream.emitter
-
-        LogStream.emit(
-          type: "context_limit_warning",
-          agent: context.agent_name,
-          model: context.metadata[:model] || "unknown",
-          threshold: "#{context.metadata[:threshold]}%",
-          current_usage: "#{context.metadata[:percentage]}%",
-          tokens_used: context.metadata[:tokens_used],
-          tokens_remaining: context.metadata[:tokens_remaining],
-          context_limit: context.metadata[:context_limit],
-          metadata: context.metadata,
-        )
-      end
-    end
-
-    # Trigger swarm_start hooks when swarm execution begins
-    #
-    # This is a swarm-level event that fires when Swarm.execute is called
-    # (before first user message is sent). Hooks can halt execution or append stdout to prompt.
-    # Default callback emits to LogStream for logging.
-    #
-    # @param prompt [String] The user's task prompt
-    # @return [Hooks::Result, nil] Result with stdout to append (if exit 0) or nil
-    # @raise [Hooks::Error] If hook halts execution
-    def trigger_swarm_start(prompt)
-      context = Hooks::Context.new(
-        event: :swarm_start,
-        agent_name: @lead_agent.to_s,
-        swarm: self,
-        metadata: {
-          swarm_name: @name,
-          lead_agent: @lead_agent,
-          prompt: prompt,
-          timestamp: Time.now.utc.iso8601,
-        },
-      )
-
-      executor = Hooks::Executor.new(@hook_registry, logger: RubyLLM.logger)
-      result = executor.execute_safe(event: :swarm_start, context: context, callbacks: [])
-
-      # Halt execution if hook requests it
-      raise Hooks::Error, "Swarm start halted by hook: #{result.value}" if result.halt?
-
-      # Return result so caller can check for replace (stdout injection)
-      result
-    rescue StandardError => e
-      RubyLLM.logger.error("SwarmSDK: Error in swarm_start hook: #{e.message}")
-      raise
-    end
-
-    # Trigger swarm_stop for final event emission (called in ensure block)
-    #
-    # This ALWAYS emits the swarm_stop event, even if there was an error.
-    # It does NOT check for reprompt (that's done in trigger_swarm_stop_for_reprompt_check).
-    #
-    # @param result [Result, nil] Execution result (may be nil if exception before result created)
-    # @param start_time [Time] Execution start time
-    # @param logs [Array] Collected logs
-    # @return [void]
-    def trigger_swarm_stop_final(result, start_time, logs)
-      # Create a minimal result if one doesn't exist (exception before result created)
-      result ||= Result.new(
-        content: nil,
-        agent: @lead_agent&.to_s || "unknown",
-        logs: logs,
-        duration: Time.now - start_time,
-        error: StandardError.new("Unknown error"),
-      )
-
-      context = Hooks::Context.new(
-        event: :swarm_stop,
-        agent_name: @lead_agent.to_s,
-        swarm: self,
-        metadata: {
-          swarm_name: @name,
-          lead_agent: @lead_agent,
-          last_agent: result.agent, # Agent that produced the final response
-          content: result.content, # Final response content
-          success: result.success?,
-          duration: result.duration,
-          total_cost: result.total_cost,
-          total_tokens: result.total_tokens,
-          agents_involved: result.agents_involved,
-          result: result,
-          timestamp: Time.now.utc.iso8601,
-        },
-      )
-
-      executor = Hooks::Executor.new(@hook_registry, logger: RubyLLM.logger)
-      executor.execute_safe(event: :swarm_stop, context: context, callbacks: [])
-    rescue StandardError => e
-      # Don't let swarm_stop errors break the ensure block
-      RubyLLM.logger.error("SwarmSDK: Error in swarm_stop final emission: #{e.message}")
-    end
-
-    # Trigger swarm_stop hooks for reprompt check and event emission
-    #
-    # This is called in the normal execution flow to check if hooks request reprompting.
-    # The default callback also emits the swarm_stop event to LogStream.
-    #
-    # @param result [Result] The execution result
-    # @return [Hooks::Result, nil] Hook result (reprompt action if applicable)
-    def trigger_swarm_stop(result)
-      context = Hooks::Context.new(
-        event: :swarm_stop,
-        agent_name: @lead_agent.to_s,
-        swarm: self,
-        metadata: {
-          swarm_name: @name,
-          lead_agent: @lead_agent,
-          last_agent: result.agent, # Agent that produced the final response
-          content: result.content, # Final response content
-          success: result.success?,
-          duration: result.duration,
-          total_cost: result.total_cost,
-          total_tokens: result.total_tokens,
-          agents_involved: result.agents_involved,
-          result: result, # Include full result for hook access
-          timestamp: Time.now.utc.iso8601,
-        },
-      )
-
-      executor = Hooks::Executor.new(@hook_registry, logger: RubyLLM.logger)
-      hook_result = executor.execute_safe(event: :swarm_stop, context: context, callbacks: [])
-
-      # Return hook result so caller can handle reprompt
-      hook_result
-    rescue StandardError => e
-      RubyLLM.logger.error("SwarmSDK: Error in swarm_stop hook: #{e.message}")
-      nil
-    end
-
-    # Trigger first_message hooks when first user message is sent
-    #
-    # This is a swarm-level event that fires once on the first call to execute().
-    # Hooks can halt execution before the first message is sent.
-    #
-    # @param prompt [String] The first user message
-    # @return [void]
-    # @raise [Hooks::Error] If hook halts execution
-    def trigger_first_message(prompt)
-      return if @hook_registry.get_defaults(:first_message).empty?
-
-      context = Hooks::Context.new(
-        event: :first_message,
-        agent_name: @lead_agent.to_s,
-        swarm: self,
-        metadata: {
-          swarm_name: @name,
-          lead_agent: @lead_agent,
-          prompt: prompt,
-          timestamp: Time.now.utc.iso8601,
-        },
-      )
-
-      executor = Hooks::Executor.new(@hook_registry, logger: RubyLLM.logger)
-      result = executor.execute_safe(event: :first_message, context: context, callbacks: [])
-
-      # Halt execution if hook requests it
-      raise Hooks::Error, "First message halted by hook: #{result.value}" if result.halt?
-    rescue StandardError => e
-      RubyLLM.logger.error("SwarmSDK: Error in first_message hook: #{e.message}")
-      raise
-    end
   end
 end