swarm_sdk 2.1.3 → 2.2.0

data/lib/swarm_sdk/node_orchestrator.rb

@@ -18,15 +18,39 @@ module SwarmSDK
   #   )
   #   result = orchestrator.execute("Build auth system")
   class NodeOrchestrator
-    attr_reader :swarm_name, :nodes, :start_node
+    attr_reader :swarm_name, :nodes, :start_node, :agent_definitions, :agent_instance_cache, :scratchpad
+    attr_writer :swarm_id, :config_for_hooks
+    attr_accessor :swarm_registry_config
 
-    def initialize(swarm_name:, agent_definitions:, nodes:, start_node:, scratchpad_enabled: true)
+    def initialize(swarm_name:, agent_definitions:, nodes:, start_node:, swarm_id: nil, scratchpad: :enabled, allow_filesystem_tools: nil)
       @swarm_name = swarm_name
+      @swarm_id = swarm_id
       @agent_definitions = agent_definitions
       @nodes = nodes
       @start_node = start_node
-      @scratchpad_enabled = scratchpad_enabled
-      @agent_instance_cache = {} # Cache for preserving agent context across nodes
+      @scratchpad = normalize_scratchpad_mode(scratchpad)
+      @allow_filesystem_tools = allow_filesystem_tools
+      @swarm_registry_config = [] # External swarms config (if using composable swarms)
+      @agent_instance_cache = {
+        primary: {}, # { agent_name => Agent::Chat }
+        delegations: {}, # { "delegate@delegator" => Agent::Chat }
+      }
+
+      # Initialize scratchpad storage based on mode
+      case @scratchpad
+      when :enabled
+        # Enabled mode: single scratchpad shared across all nodes
+        @shared_scratchpad_storage = Tools::Stores::ScratchpadStorage.new
+        @node_scratchpads = nil
+      when :per_node
+        # Per-node mode: separate scratchpad per node (lazy initialized)
+        @shared_scratchpad_storage = nil
+        @node_scratchpads = {}
+      when :disabled
+        # Disabled: no storage at all
+        @shared_scratchpad_storage = nil
+        @node_scratchpads = nil
+      end
 
       validate!
       @execution_order = build_execution_order
@@ -35,6 +59,72 @@ module SwarmSDK
     # Alias for compatibility with Swarm interface
     alias_method :name, :swarm_name
 
+    # Get scratchpad storage for a specific node
+    #
+    # Returns the appropriate scratchpad based on mode:
+    # - :enabled - returns the shared scratchpad (same for all nodes)
+    # - :per_node - returns node-specific scratchpad (lazy initialized)
+    # - :disabled - returns nil
+    #
+    # @param node_name [Symbol] Node name
+    # @return [Tools::Stores::ScratchpadStorage, nil] Scratchpad instance or nil if disabled
+    def scratchpad_for(node_name)
+      case @scratchpad
+      when :enabled
+        @shared_scratchpad_storage
+      when :per_node
+        # Lazy initialization per node
+        @node_scratchpads[node_name] ||= Tools::Stores::ScratchpadStorage.new
+      when :disabled
+        nil
+      end
+    end
+
+    # Get all scratchpad storages (for snapshot/restore)
+    #
+    # @return [Hash] { :shared => scratchpad } or { node_name => scratchpad }
+    def all_scratchpads
+      case @scratchpad
+      when :enabled
+        { shared: @shared_scratchpad_storage }
+      when :per_node
+        @node_scratchpads.dup
+      when :disabled
+        {}
+      end
+    end
+
+    # Check if scratchpad is enabled
+    #
+    # @return [Boolean]
+    def scratchpad_enabled?
+      @scratchpad != :disabled
+    end
+
+    # Check if scratchpad is shared between nodes (enabled mode)
+    #
+    # @return [Boolean]
+    def shared_scratchpad?
+      @scratchpad == :enabled
+    end
+
+    # Check if scratchpad is per-node
+    #
+    # @return [Boolean]
+    def per_node_scratchpad?
+      @scratchpad == :per_node
+    end
+
+    # Backward compatibility accessor
+    #
+    # @return [Tools::Stores::ScratchpadStorage, nil]
+    def shared_scratchpad_storage
+      if @scratchpad == :per_node
+        RubyLLM.logger.warn("NodeOrchestrator: Accessing shared_scratchpad_storage in per-node mode. Use scratchpad_for(node_name) instead.")
+      end
+      @shared_scratchpad_storage
+    end
+
     # Return the lead agent of the start node for CLI compatibility
     #
     # @return [Symbol] Lead agent of the start node
@@ -56,6 +146,9 @@ module SwarmSDK
       results = {}
       @original_prompt = prompt # Store original prompt for NodeContext
 
+      # Set fiber-local execution context for entire workflow
+      Fiber[:execution_id] = generate_execution_id
+
       # Setup logging if block given
       if block_given?
         # Register callback to collect logs and forward to user's block
@@ -77,6 +170,12 @@ module SwarmSDK
         node = @nodes[node_name]
         node_start_time = Time.now
 
+        # Set node-specific swarm_id in fiber storage
+        # Mini-swarms will use ||= to inherit execution_id
+        node_swarm_id = @swarm_id ? "#{@swarm_id}/node:#{node_name}" : nil
+        Fiber[:swarm_id] = node_swarm_id
+        Fiber[:parent_swarm_id] = @swarm_id
+
         # Emit node_start event
         emit_node_start(node_name, node)
 
@@ -226,13 +325,92 @@ module SwarmSDK
 
       last_result
     ensure
+      # NodeOrchestrator always clears (always sets up logging)
+      Fiber[:execution_id] = nil
+      Fiber[:swarm_id] = nil
+      Fiber[:parent_swarm_id] = nil
+
       # Reset logging state for next execution
       LogCollector.reset!
       LogStream.reset!
     end
 
+    # Create snapshot of current workflow state
+    #
+    # Returns a Snapshot object containing agent conversations, context state,
+    # and scratchpad data from all nodes that have been executed. The snapshot
+    # captures the state of agents in the agent_instance_cache (both primary and
+    # delegation instances), as well as scratchpad storage.
+    #
+    # Configuration (agent definitions, nodes, transformers) stays in your code
+    # and is NOT included in snapshots.
+    #
+    # Scratchpad behavior depends on scratchpad mode:
+    # - :enabled (default): single scratchpad shared across all nodes
+    # - :per_node: separate scratchpad per node
+    # - :disabled: no scratchpad data
+    #
+    # @return [Snapshot] Snapshot object with convenient serialization methods
+    #
+    # @example Save snapshot to JSON file
+    #   orchestrator = NodeOrchestrator.new(...)
+    #   orchestrator.execute("Build feature")
+    #   snapshot = orchestrator.snapshot
+    #   snapshot.write_to_file("workflow_session.json")
+    def snapshot
+      StateSnapshot.new(self).snapshot
+    end
+
+    # Restore workflow state from snapshot
+    #
+    # Accepts a Snapshot object, hash, or JSON string. Validates compatibility
+    # between snapshot and current orchestrator configuration. Restores agent
+    # conversations that exist in the agent_instance_cache.
+    #
+    # The orchestrator must be created with the SAME configuration (agent definitions,
+    # nodes) as when the snapshot was created. Only conversation state is restored.
+    #
+    # For agents with reset_context: false, restored conversations will be injected
+    # during node execution. Agents not in cache yet will be skipped (they haven't
+    # been used yet, so there's nothing to restore).
+    #
+    # @param snapshot [Snapshot, Hash, String] Snapshot object, hash, or JSON string
+    # @return [RestoreResult] Result with warnings about skipped agents
+    #
+    # @example Restore from Snapshot object
+    #   orchestrator = NodeOrchestrator.new(...)  # Same config as snapshot
+    #   snapshot = Snapshot.from_file("workflow_session.json")
+    #   result = orchestrator.restore(snapshot)
+    #   if result.success?
+    #     puts "All agents restored"
+    #   else
+    #     puts result.summary
+    #   end
+    #
+    # Restore orchestrator state from snapshot
+    #
+    # By default, uses current system prompts from agent definitions (YAML + SDK defaults + plugin injections).
+    # Set preserve_system_prompts: true to use historical prompts from snapshot.
+    #
+    # @param snapshot [Snapshot, Hash, String] Snapshot object, hash, or JSON string
+    # @param preserve_system_prompts [Boolean] Use historical system prompts instead of current config (default: false)
+    # @return [RestoreResult] Result with warnings about partial restores
+    def restore(snapshot, preserve_system_prompts: false)
+      StateRestorer.new(self, snapshot, preserve_system_prompts: preserve_system_prompts).restore
+    end
+
     private
 
+    # Generate a unique execution ID for workflow
+    #
+    # Creates an execution ID that uniquely identifies a single orchestrator.execute() call.
+    # Format: "exec_workflow_{random_hex}"
+    #
+    # @return [String] Generated execution ID (e.g., "exec_workflow_a3f2b1c8")
+    def generate_execution_id
+      "exec_workflow_#{SecureRandom.hex(8)}"
+    end
+
     # Emit node_start event
     #
     # @param node_name [Symbol] Name of the node
@@ -346,26 +524,46 @@ module SwarmSDK
     # For agents with reset_context: false, injects cached instances
     # to preserve conversation history across nodes.
     #
-    # Inherits scratchpad_enabled setting from NodeOrchestrator.
+    # Scratchpad behavior depends on mode:
+    # - :enabled - all nodes use the same scratchpad instance
+    # - :per_node - each node gets its own scratchpad instance
+    # - :disabled - no scratchpad
     #
     # @param node [Node::Builder] Node configuration
     # @return [Swarm] Configured swarm instance
     def build_swarm_for_node(node)
+      # Build hierarchical swarm_id if parent has one (nil auto-generates)
+      node_swarm_id = @swarm_id ? "#{@swarm_id}/node:#{node.name}" : nil
+
       swarm = Swarm.new(
         name: "#{@swarm_name}:#{node.name}",
-        scratchpad_enabled: @scratchpad_enabled,
+        swarm_id: node_swarm_id,
+        parent_swarm_id: @swarm_id,
+        scratchpad: scratchpad_for(node.name),
+        scratchpad_mode: :enabled, # Mini-swarms always use enabled (scratchpad instance passed in)
+        allow_filesystem_tools: @allow_filesystem_tools,
       )
 
+      # Setup swarm registry if external swarms are registered
+      if @swarm_registry_config&.any?
+        registry = SwarmRegistry.new(parent_swarm_id: node_swarm_id || swarm.swarm_id)
+        @swarm_registry_config.each do |reg|
+          registry.register(reg[:name], source: reg[:source], keep_context: reg[:keep_context])
+        end
+        swarm.swarm_registry = registry
+      end
+
       # Add each agent specified in this node
       node.agent_configs.each do |config|
         agent_name = config[:agent]
         delegates_to = config[:delegates_to]
+        tools_override = config[:tools]
 
         # Get global agent definition
         agent_def = @agent_definitions[agent_name]
 
-        # Clone definition with node-specific delegation
-        node_specific_def = clone_with_delegation(agent_def, delegates_to)
+        # Clone definition with node-specific overrides
+        node_specific_def = clone_agent_for_node(agent_def, delegates_to, tools_override)
 
         swarm.add_agent(node_specific_def)
       end
@@ -379,14 +577,20 @@ module SwarmSDK
       swarm
     end
 
-    # Clone an agent definition with different delegates_to
+    # Clone an agent definition with node-specific overrides
+    #
+    # Allows overriding delegation and tools per node. This enables:
+    # - Different delegation topology per node
+    # - Different tool sets per workflow stage
     #
     # @param agent_def [Agent::Definition] Original definition
     # @param delegates_to [Array<Symbol>] New delegation targets
-    # @return [Agent::Definition] Cloned definition
-    def clone_with_delegation(agent_def, delegates_to)
+    # @param tools [Array<Symbol>, nil] Tool override (nil = use global agent definition)
+    # @return [Agent::Definition] Cloned definition with overrides
+    def clone_agent_for_node(agent_def, delegates_to, tools)
       config = agent_def.to_h
       config[:delegates_to] = delegates_to
+      config[:tools] = tools if tools # Only override if explicitly set
       Agent::Definition.new(agent_def.name, config)
     end
 
@@ -540,18 +744,29 @@ module SwarmSDK
     # @param node [Node::Builder] Node configuration
     # @return [void]
     def cache_agent_instances(swarm, node)
-      return unless swarm.agents # Only cache if agents were initialized
+      return unless swarm.agents
 
       node.agent_configs.each do |config|
         agent_name = config[:agent]
         reset_context = config[:reset_context]
 
-        # Only cache if reset_context is false
+        # Only cache if reset_context: false
         next if reset_context
 
-        # Cache the agent instance
+        # Cache primary agent
         agent_instance = swarm.agents[agent_name]
-        @agent_instance_cache[agent_name] = agent_instance if agent_instance
+        @agent_instance_cache[:primary][agent_name] = agent_instance if agent_instance
+
+        # V7.0: Cache delegation instances atomically (together with primary)
+        agent_def = @agent_definitions[agent_name]
+        agent_def.delegates_to.each do |delegate_name|
+          delegation_key = "#{delegate_name}@#{agent_name}"
+          delegation_instance = swarm.delegation_instances[delegation_key]
+
+          if delegation_instance
+            @agent_instance_cache[:delegations][delegation_key] = delegation_instance
+          end
+        end
       end
     end
 
@@ -565,27 +780,79 @@ module SwarmSDK
     # @return [void]
     def inject_cached_agents(swarm, node)
       # Check if any agents need context preservation
-      has_preserved_agents = node.agent_configs.any? { |c| !c[:reset_context] && @agent_instance_cache[c[:agent]] }
-      return unless has_preserved_agents
+      has_preserved = node.agent_configs.any? do |c|
+        !c[:reset_context] && (
+          @agent_instance_cache[:primary][c[:agent]] ||
+          has_cached_delegations_for?(c[:agent])
+        )
+      end
+      return unless has_preserved
 
-      # Force agent initialization by accessing .agents (triggers lazy init)
-      # Then inject cached instances
+      # V7.0 CRITICAL FIX: Force initialization FIRST
+      # Without this, @agents will be replaced by initialize_all, losing our injected instances
+      swarm.agent(node.agent_configs.first[:agent]) # Triggers lazy init
+
+      # Now safely inject cached instances
       agents_hash = swarm.agents
+      delegation_hash = swarm.delegation_instances
 
+      # Inject cached PRIMARY agents
       node.agent_configs.each do |config|
         agent_name = config[:agent]
-        reset_context = config[:reset_context]
+        next if config[:reset_context]
 
-        # Skip if reset_context is true (want fresh instance)
-        next if reset_context
-
-        # Check if we have a cached instance
-        cached_agent = @agent_instance_cache[agent_name]
+        cached_agent = @agent_instance_cache[:primary][agent_name]
         next unless cached_agent
 
-        # Inject the cached instance (replace the freshly initialized one)
+        # Replace freshly initialized agent with cached instance
         agents_hash[agent_name] = cached_agent
       end
+
+      # Inject cached DELEGATION instances (atomic with primary)
+      node.agent_configs.each do |config|
+        agent_name = config[:agent]
+        next if config[:reset_context]
+
+        agent_def = @agent_definitions[agent_name]
+
+        agent_def.delegates_to.each do |delegate_name|
+          delegation_key = "#{delegate_name}@#{agent_name}"
+          cached_delegation = @agent_instance_cache[:delegations][delegation_key]
+          next unless cached_delegation
+
+          # Replace freshly initialized delegation instance
+          # V7.0: Tool references intact - atomic caching preserves object graph
+          delegation_hash[delegation_key] = cached_delegation
+        end
+      end
+    end
+
+    def has_cached_delegations_for?(agent_name)
+      agent_def = @agent_definitions[agent_name]
+      agent_def.delegates_to.any? do |delegate_name|
+        delegation_key = "#{delegate_name}@#{agent_name}"
+        @agent_instance_cache[:delegations][delegation_key]
+      end
+    end
+
+    # Normalize scratchpad mode parameter
+    #
+    # Accepts symbols: :enabled, :per_node, or :disabled
+    #
+    # @param value [Symbol, String] Scratchpad mode (strings from YAML converted to symbols)
+    # @return [Symbol] Normalized mode (:enabled, :per_node, or :disabled)
+    # @raise [ArgumentError] If value is invalid
+    def normalize_scratchpad_mode(value)
+      # Convert strings from YAML to symbols
+      value = value.to_sym if value.is_a?(String)
+
+      case value
+      when :enabled, :per_node, :disabled
+        value
+      else
+        raise ArgumentError,
+          "Invalid scratchpad mode: #{value.inspect}. Use :enabled, :per_node, or :disabled"
+      end
     end
   end
 end

data/lib/swarm_sdk/proc_helpers.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  # Helper methods for working with Procs and Lambdas
+  #
+  # Provides functionality to convert regular Proc objects into Lambdas to enable
+  # safe use of return statements in DSL blocks (like input/output transformers).
+  module ProcHelpers
+    class << self
+      # Convert a Proc to a Lambda
+      #
+      # The fundamental difference between a Proc and a Lambda is in how they handle
+      # return statements. In a Proc, return exits the enclosing method (or program),
+      # while in a Lambda, return only exits the lambda itself.
+      #
+      # This method converts a Proc to a Lambda by:
+      # 1. Converting the proc to an unbound method via define_method
+      # 2. Wrapping it in a lambda that binds and calls the method
+      # 3. In the method, return exits the method (not the original scope)
+      #
+      # This allows users to write natural control flow with return statements:
+      #
+      # @example
+      #   my_proc = proc { |x| return x * 2 if x > 0; 0 }
+      #   my_lambda = ProcHelpers.to_lambda(my_proc)
+      #   my_lambda.call(5)  # => 10 (return works safely!)
+      #
+      # @param proc [Proc] The proc to convert
+      # @return [Proc] A lambda with the same behavior but safe return semantics
+      def to_lambda(proc)
+        return proc if proc.lambda?
+
+        # Save local reference to proc so we can use it in module_exec/lambda scopes
+        source_proc = proc
+
+        # Convert proc to unbound method
+        # define_method with a block converts the block to a method, where return
+        # exits the method (not the original scope)
+        unbound_method = Module.new.module_exec do
+          instance_method(define_method(:_proc_call, &source_proc))
+        end
+
+        # Return lambda which binds our unbound method to correct receiver and calls it
+        lambda do |*args, **kwargs, &block|
+          # Bind method to the original proc's receiver (the context where it was defined)
+          # This preserves access to instance variables, local variables via closure, etc.
+          receiver = source_proc.binding.eval("self")
+          unbound_method.bind(receiver).call(*args, **kwargs, &block)
+        end
+      end
+    end
+  end
+end

data/lib/swarm_sdk/providers/openai_with_responses.rb

@@ -145,7 +145,7 @@ module SwarmSDK
       rescue NoMethodError => e
         # Catch fetch/dig errors on nil and provide better context
         if e.message.include?("undefined method") && (e.message.include?("fetch") || e.message.include?("dig"))
-          log_parse_error(e.class.name, e.message, response.body)
+          log_parse_error(e.class.name, e.message, response.body, e.backtrace)
           nil
         else
           raise
@@ -377,30 +377,36 @@ module SwarmSDK
       # This differs from chat/completions which nests under 'function':
       # { type: "function", function: { name: "tool_name", ... } }
       #
+      # RubyLLM 1.9.0+: Uses tool.params_schema for unified schema generation.
+      # This supports both old param helper and new params DSL, and includes
+      # proper JSON Schema formatting (strict, additionalProperties, etc.)
+      #
       # @param tool [RubyLLM::Tool] Tool to convert
       # @return [Hash] Tool definition in Responses API format
       def responses_tool_for(tool)
+        # Use tool.params_schema which returns a complete JSON Schema hash
+        # This works with both param helper and params DSL
+        parameters_schema = tool.params_schema || empty_parameters_schema
+
         {
           type: "function",
           name: tool.name,
           description: tool.description,
-          parameters: {
-            type: "object",
-            properties: tool.parameters.transform_values { |param| param_schema(param) },
-            required: tool.parameters.select { |_, p| p.required }.keys,
-          },
+          parameters: parameters_schema,
         }
       end
 
-      # Build parameter schema for a tool parameter
+      # Empty parameter schema for tools with no parameters
       #
-      # @param param [RubyLLM::Tool::Parameter] Parameter to convert
-      # @return [Hash] Parameter schema
-      def param_schema(param)
+      # @return [Hash] Empty JSON Schema matching OpenAI's format
+      def empty_parameters_schema
         {
-          type: param.type,
-          description: param.description,
-        }.compact
+          "type" => "object",
+          "properties" => {},
+          "required" => [],
+          "additionalProperties" => false,
+          "strict" => true,
+        }
       end
 
       # Parse Responses API response
@@ -562,7 +568,7 @@ module SwarmSDK
       # @param error_class [String] Error class name
       # @param error_message [String] Error message
       # @param response_body [Object] Response body that failed to parse
-      def log_parse_error(error_class, error_message, response_body)
+      def log_parse_error(error_class, error_message, response_body, error_backtrace = nil)
         if @agent_name
           # Emit structured JSON log through LogStream
           LogStream.emit(
@@ -570,11 +576,12 @@ module SwarmSDK
             agent: @agent_name,
             error_class: error_class,
             error_message: error_message,
+            error_backtrace: error_backtrace,
             response_body: response_body.inspect,
           )
         else
           # Fallback to RubyLLM logger if agent name not set
-          RubyLLM.logger.error("SwarmSDK: #{error_class}: #{error_message}\nResponse: #{response_body.inspect}")
+          RubyLLM.logger.error("SwarmSDK: #{error_class}: #{error_message}\nResponse: #{response_body.inspect}\nError backtrace: #{error_backtrace.join("\n")}")
         end
       end
     end

data/lib/swarm_sdk/restore_result.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  # Result object returned from snapshot restore operations
+  #
+  # Provides information about the restore process, including any warnings
+  # about agents or delegations that couldn't be restored due to configuration
+  # mismatches.
+  #
+  # @example Successful restore
+  #   result = swarm.restore(snapshot_data)
+  #   if result.success?
+  #     puts "All agents restored successfully"
+  #   end
+  #
+  # @example Partial restore with warnings
+  #   result = swarm.restore(snapshot_data)
+  #   if result.partial_restore?
+  #     puts result.summary
+  #     result.warnings.each do |warning|
+  #       puts "  - #{warning[:message]}"
+  #     end
+  #   end
+  class RestoreResult
+    attr_reader :warnings, :skipped_agents, :skipped_delegations
+
+    # Initialize restore result
+    #
+    # @param warnings [Array<Hash>] Warning messages with details
+    # @param skipped_agents [Array<Symbol>] Names of agents that couldn't be restored
+    # @param skipped_delegations [Array<String>] Names of delegation instances that couldn't be restored
+    def initialize(warnings:, skipped_agents:, skipped_delegations:)
+      @warnings = warnings
+      @skipped_agents = skipped_agents
+      @skipped_delegations = skipped_delegations
+    end
+
+    # Check if restore was completely successful
+    #
+    # @return [Boolean] true if all agents restored without warnings
+    def success?
+      warnings.empty?
+    end
+
+    # Check if restore was partial (some agents skipped)
+    #
+    # @return [Boolean] true if some agents were skipped
+    def partial_restore?
+      !warnings.empty?
+    end
+
+    # Get human-readable summary of restore result
+    #
+    # @return [String] Summary message
+    def summary
+      if success?
+        "Snapshot restored successfully. All agents restored."
+      else
+        "Snapshot restored with warnings. " \
+          "#{skipped_agents.size} agents skipped, " \
+          "#{skipped_delegations.size} delegation instances skipped."
+      end
+    end
+  end
+end