swarm_memory 2.0.0

data/lib/claude_swarm/openai/executor.rb

@@ -0,0 +1,254 @@
+# frozen_string_literal: true
+
+require "openai"
+require "faraday/net_http_persistent"
+require "faraday/retry"
+
+module ClaudeSwarm
+  module OpenAI
+    class Executor < BaseExecutor
+      # Static configuration for Faraday retry middleware
+      FARADAY_RETRY_CONFIG = {
+        max: 3, # Maximum number of retries
+        interval: 0.5, # Initial delay between retries (in seconds)
+        interval_randomness: 0.5, # Randomness factor for retry intervals
+        backoff_factor: 2, # Exponential backoff factor
+        exceptions: [
+          Faraday::TimeoutError,
+          Faraday::ConnectionFailed,
+          Faraday::ServerError, # Retry on 5xx errors
+        ].freeze,
+        retry_statuses: [429, 500, 502, 503, 504].freeze, # HTTP status codes to retry
+      }.freeze
+
+      # Static configuration for OpenAI client
+      OPENAI_CLIENT_CONFIG = {
+        log_errors: true,
+        request_timeout: 1800, # 30 minutes
+      }.freeze
+
+      def initialize(working_directory: Dir.pwd, model: nil, mcp_config: nil, vibe: false,
+        instance_name: nil, instance_id: nil, calling_instance: nil, calling_instance_id: nil,
+        claude_session_id: nil, additional_directories: [], debug: false,
+        temperature: nil, api_version: "chat_completion", openai_token_env: "OPENAI_API_KEY",
+        base_url: nil, reasoning_effort: nil)
+        # Call parent initializer for common attributes
+        super(
+          working_directory: working_directory,
+          model: model,
+          mcp_config: mcp_config,
+          vibe: vibe,
+          instance_name: instance_name,
+          instance_id: instance_id,
+          calling_instance: calling_instance,
+          calling_instance_id: calling_instance_id,
+          claude_session_id: claude_session_id,
+          additional_directories: additional_directories,
+          debug: debug
+        )
+
+        # OpenAI-specific attributes
+        @temperature = temperature
+        @api_version = api_version
+        @base_url = base_url
+        @reasoning_effort = reasoning_effort
+
+        # Conversation state for maintaining context
+        @conversation_messages = []
+        @previous_response_id = nil
+
+        # Setup OpenAI client
+        setup_openai_client(openai_token_env)
+
+        # Setup MCP client for tools
+        setup_mcp_client
+
+        # Create API handler based on api_version
+        @api_handler = create_api_handler
+      end
+
+      def execute(prompt, options = {})
+        # Log the request
+        log_request(prompt)
+
+        # Start timing
+        start_time = Time.now
+
+        # Execute using the appropriate handler
+        result = @api_handler.execute(prompt, options)
+
+        # Calculate duration
+        duration_ms = ((Time.now - start_time) * 1000).round
+
+        # Build and return response
+        build_response(result, duration_ms)
+      rescue StandardError => e
+        logger.error { "Unexpected error for #{@instance_name}: #{e.class} - #{e.message}" }
+        logger.error { "Backtrace: #{e.backtrace.join("\n")}" }
+        raise
+      end
+
+      def reset_session
+        super
+        @api_handler&.reset_session
+      end
+
+      # Session JSON logger for the API handlers
+      def session_json_logger
+        self
+      end
+
+      def log(event)
+        append_to_session_json(event)
+      end
+
+      private
+
+      def setup_openai_client(token_env)
+        openai_client_config = build_openai_client_config(token_env)
+
+        @openai_client = ::OpenAI::Client.new(openai_client_config) do |faraday|
+          # Use persistent HTTP connections for better performance
+          faraday.adapter(:net_http_persistent)
+
+          # Add retry middleware with custom configuration
+          faraday.request(:retry, **build_faraday_retry_config)
+        end
+      rescue KeyError
+        raise ExecutionError, "OpenAI API key not found in environment variable: #{token_env}"
+      end
+
+      def setup_mcp_client
+        return unless @mcp_config && File.exist?(@mcp_config)
+
+        # Read MCP config to find MCP servers
+        mcp_data = JsonHandler.parse_file!(@mcp_config)
+
+        # Build MCP configurations from servers
+        mcp_configs = build_mcp_configs(mcp_data["mcpServers"])
+        return if mcp_configs.empty?
+
+        # Create MCP client with unbundled environment to avoid bundler conflicts
+        # This ensures MCP servers run in a clean environment without inheriting
+        # Claude Swarm's BUNDLE_* environment variables
+        Bundler.with_unbundled_env do
+          @mcp_client = MCPClient.create_client(
+            mcp_server_configs: mcp_configs,
+            logger: @logger,
+          )
+
+          # List available tools from all MCP servers
+          load_mcp_tools(mcp_configs)
+        end
+      rescue StandardError => e
+        logger.error { "Failed to setup MCP client: #{e.message}" }
+        @mcp_client = nil
+        @available_tools = []
+      end
+
+      def calculate_cost(_result)
+        # Simplified cost calculation
+        # In reality, we'd need to track token usage
+        "$0.00"
+      end
+
+      def create_api_handler
+        handler_params = {
+          openai_client: @openai_client,
+          mcp_client: @mcp_client,
+          available_tools: @available_tools,
+          executor: self,
+          instance_name: @instance_name,
+          model: @model,
+          temperature: @temperature,
+          reasoning_effort: @reasoning_effort,
+        }
+
+        if @api_version == "responses"
+          OpenAI::Responses.new(**handler_params)
+        else
+          OpenAI::ChatCompletion.new(**handler_params)
+        end
+      end
+
+      def log_streaming_content(content)
+        # Log streaming content similar to ClaudeCodeExecutor
+        logger.debug { "#{instance_info} streaming: #{content}" }
+      end
+
+      def build_faraday_retry_config
+        FARADAY_RETRY_CONFIG.merge(
+          retry_block: method(:handle_retry_logging),
+        )
+      end
+
+      def handle_retry_logging(env:, options:, retry_count:, exception:, will_retry:)
+        retry_delay = options.interval * (options.backoff_factor**(retry_count - 1))
+        error_info = exception&.message || "HTTP #{env.status}"
+
+        message = if will_retry
+          "Request failed (attempt #{retry_count}/#{options.max}): #{error_info}. Retrying in #{retry_delay} seconds..."
+        else
+          "Request failed after #{retry_count} attempts: #{error_info}. Giving up."
+        end
+
+        @logger.warn(message)
+      end
+
+      def build_openai_client_config(token_env)
+        OPENAI_CLIENT_CONFIG.merge(access_token: ENV.fetch(token_env)).tap do |config|
+          config[:uri_base] = @base_url if @base_url
+        end
+      end
+
+      def build_stdio_config(name, server_config)
+        # Combine command and args into a single array
+        command_array = [server_config["command"]]
+        command_array.concat(server_config["args"] || [])
+
+        MCPClient.stdio_config(
+          command: command_array,
+          name: name,
+        ).tap do |config|
+          config[:read_timeout] = 1800
+        end
+      end
+
+      def build_mcp_configs(mcp_servers)
+        return [] if mcp_servers.nil? || mcp_servers.empty?
+
+        mcp_servers.filter_map do |name, server_config|
+          case server_config["type"]
+          when "stdio"
+            build_stdio_config(name, server_config)
+          when "sse"
+            logger.warn { "SSE MCP servers not yet supported for OpenAI instances: #{name}" }
+            # TODO: Add SSE support when available in ruby-mcp-client
+            nil
+          end
+        end
+      end
+
+      def load_mcp_tools(mcp_configs)
+        @available_tools = @mcp_client.list_tools
+        logger.info { "Loaded #{@available_tools.size} tools from #{mcp_configs.size} MCP server(s)" }
+      rescue StandardError => e
+        logger.error { "Failed to load MCP tools: #{e.message}" }
+        @available_tools = []
+      end
+
+      def build_response(result, duration_ms)
+        {
+          "type" => "result",
+          "result" => result,
+          "duration_ms" => duration_ms,
+          "total_cost" => calculate_cost(result),
+          "session_id" => @session_id,
+        }.tap do |response|
+          log_response(response)
+          @last_response = response
+        end
+      end
+    end
+  end
+end

data/lib/claude_swarm/openai/responses.rb

@@ -0,0 +1,338 @@
+# frozen_string_literal: true
+
+module ClaudeSwarm
+  module OpenAI
+    class Responses
+      MAX_TURNS_WITH_TOOLS = 100_000 # virtually infinite
+
+      def initialize(openai_client:, mcp_client:, available_tools:, executor:, instance_name:, model:, temperature: nil, reasoning_effort: nil)
+        @openai_client = openai_client
+        @mcp_client = mcp_client
+        @available_tools = available_tools
+        @executor = executor
+        @instance_name = instance_name
+        @model = model
+        @temperature = temperature
+        @reasoning_effort = reasoning_effort
+        @system_prompt = nil
+      end
+
+      def execute(prompt, options = {})
+        # Store system prompt for first call
+        @system_prompt = options[:system_prompt] if options[:system_prompt]
+
+        # Start with initial prompt
+        initial_input = prompt
+
+        # Process with recursive tool handling - start with empty conversation
+        process_responses_api(initial_input, [], nil)
+      end
+
+      def reset_session
+        @system_prompt = nil
+      end
+
+      private
+
+      def process_responses_api(input, conversation_array, previous_response_id, depth = 0)
+        # Prevent infinite recursion
+        if depth > MAX_TURNS_WITH_TOOLS
+          @executor.logger.error { "Maximum recursion depth reached in tool execution" }
+          return "Error: Maximum tool call depth exceeded"
+        end
+
+        # Build parameters
+        parameters = {
+          model: @model,
+        }
+
+        # Only add temperature for non-o-series models
+        # O-series models don't support temperature parameter
+        unless @model.match?(ClaudeSwarm::Configuration::O_SERIES_MODEL_PATTERN)
+          parameters[:temperature] = @temperature
+        end
+
+        # Only add reasoning effort for o-series models
+        # reasoning is only supported by o-series models: o1, o1 Preview, o1-mini, o1-pro, o3, o3-mini, o3-pro, o3-deep-research, o4-mini, o4-mini-deep-research, etc.
+        if @reasoning_effort && @model.match?(ClaudeSwarm::Configuration::O_SERIES_MODEL_PATTERN)
+          parameters[:reasoning] = { effort: @reasoning_effort }
+        end
+
+        # On first call, use string input (can include system prompt)
+        # On subsequent calls with function results, use array input
+        if conversation_array.empty?
+          # Initial call - string input
+          parameters[:input] = if depth.zero? && @system_prompt
+            "#{@system_prompt}\n\n#{input}"
+          else
+            input
+          end
+        else
+          # Follow-up call with conversation array (function calls + outputs)
+          parameters[:input] = conversation_array
+
+          # Log conversation array to debug duplicates
+          @executor.logger.info { "Conversation array size: #{conversation_array.size}" }
+          conversation_ids = conversation_array.map do |item|
+            item["call_id"] || item["id"] || "no-id-#{item["type"]}"
+          end.compact
+          @executor.logger.info { "Conversation item IDs: #{conversation_ids.inspect}" }
+        end
+
+        # Add previous response ID for conversation continuity
+        parameters[:previous_response_id] = previous_response_id if previous_response_id
+
+        # Add tools if available
+        if @available_tools&.any?
+          # Convert tools to responses API format
+          parameters[:tools] = @available_tools.map do |tool|
+            {
+              "type" => "function",
+              "name" => tool.name,
+              "description" => tool.description,
+              "parameters" => tool.schema || {},
+            }
+          end
+          @executor.logger.info { "Available tools for responses API: #{parameters[:tools].map { |t| t["name"] }.join(", ")}" }
+        end
+
+        # Log the request parameters
+        @executor.logger.info { "Responses API Request (depth=#{depth}): #{JsonHandler.pretty_generate!(parameters)}" }
+
+        # Append to session JSON
+        append_to_session_json({
+          type: "openai_request",
+          api: "responses",
+          depth: depth,
+          parameters: parameters,
+        })
+
+        # Make the API call without streaming
+        begin
+          response = @openai_client.responses.create(parameters: parameters)
+        rescue StandardError => e
+          @executor.logger.error { "Responses API error: #{e.class} - #{e.message}" }
+          @executor.logger.error { "Request parameters: #{JsonHandler.pretty_generate!(parameters)}" }
+
+          # Try to extract and log the response body for better debugging
+          if e.respond_to?(:response)
+            begin
+              error_body = e.response[:body]
+              @executor.logger.error { "Error response body: #{error_body}" }
+            rescue StandardError => parse_error
+              @executor.logger.error { "Could not parse error response: #{parse_error.message}" }
+            end
+          end
+
+          # Log error to session JSON
+          append_to_session_json({
+            type: "openai_error",
+            api: "responses",
+            error: {
+              class: e.class.to_s,
+              message: e.message,
+              response_body: e.respond_to?(:response) ? e.response[:body] : nil,
+              backtrace: e.backtrace.first(5),
+            },
+          })
+
+          return "Error calling OpenAI responses API: #{e.message}"
+        end
+
+        # Log the full response
+        @executor.logger.info { "Responses API Full Response (depth=#{depth}): #{JsonHandler.pretty_generate!(response)}" }
+
+        # Append to session JSON
+        append_to_session_json({
+          type: "openai_response",
+          api: "responses",
+          depth: depth,
+          response: response,
+        })
+
+        # Extract response details
+        response_id = response["id"]
+
+        # Handle response based on output structure
+        output = response["output"]
+
+        if output.nil?
+          @executor.logger.error { "No output in response" }
+          return "Error: No output in OpenAI response"
+        end
+
+        # Check if output is an array (as per documentation)
+        if output.is_a?(Array) && !output.empty?
+          # Check if there are function calls
+          function_calls = output.select { |item| item["type"] == "function_call" }
+
+          if function_calls.any?
+            # Check if we already have a conversation going
+            if conversation_array.empty?
+              # First depth - build new conversation
+              new_conversation = build_conversation_with_outputs(function_calls)
+            else
+              # Subsequent depth - append to existing conversation
+              # Don't re-add function calls, just add the new ones and their outputs
+              new_conversation = conversation_array.dup
+              append_new_outputs(function_calls, new_conversation)
+            end
+
+            # Recursively process with updated conversation
+            process_responses_api(nil, new_conversation, response_id, depth + 1)
+          else
+            # Look for text response
+            extract_text_response(output)
+          end
+        else
+          @executor.logger.error { "Unexpected output format: #{output.inspect}" }
+          "Error: Unexpected response format"
+        end
+      end
+
+      def extract_text_response(output)
+        text_output = output.find { |item| item["content"] }
+        return "" unless text_output && text_output["content"].is_a?(Array)
+
+        text_content = text_output["content"].find { |item| item["text"] }
+        text_content ? text_content["text"] : ""
+      end
+
+      def build_conversation_with_outputs(function_calls)
+        # Log tool calls
+        @executor.logger.info { "Responses API - Handling #{function_calls.size} function calls" }
+
+        # Log IDs to check for duplicates
+        call_ids = function_calls.map { |fc| fc["call_id"] || fc["id"] }
+        @executor.logger.info { "Function call IDs: #{call_ids.inspect}" }
+        @executor.logger.warn { "WARNING: Duplicate function call IDs detected!" } if call_ids.size != call_ids.uniq.size
+
+        # Append to session JSON
+        append_to_session_json({
+          type: "tool_calls",
+          api: "responses",
+          tool_calls: function_calls,
+        })
+
+        # Build conversation array with function outputs only
+        # The API already knows about the function calls from the previous response
+        conversation = []
+
+        # Then execute tools and add outputs
+        function_calls.each do |function_call|
+          tool_name = function_call["name"]
+          tool_args_str = function_call["arguments"]
+          # Use the call_id field for matching with function outputs
+          call_id = function_call["call_id"]
+
+          # Log both IDs to debug
+          @executor.logger.info { "Function call has id=#{function_call["id"]}, call_id=#{function_call["call_id"]}" }
+
+          begin
+            # Parse arguments
+            tool_args = JsonHandler.parse!(tool_args_str)
+
+            # Log tool execution
+            @executor.logger.info { "Responses API - Executing tool: #{tool_name} with args: #{JsonHandler.pretty_generate!(tool_args)}" }
+
+            # Execute tool via MCP
+            result = @mcp_client.call_tool(tool_name, tool_args)
+
+            # Log result
+            @executor.logger.info { "Responses API - Tool result for #{tool_name}: #{result}" }
+
+            # Append to session JSON
+            append_to_session_json({
+              type: "tool_execution",
+              api: "responses",
+              tool_name: tool_name,
+              arguments: tool_args,
+              result: result.to_s,
+            })
+
+            # Add function output to conversation
+            conversation << {
+              type: "function_call_output",
+              call_id: call_id,
+              output: result.to_json, # Must be JSON string
+            }
+          rescue StandardError => e
+            @executor.logger.error { "Responses API - Tool execution failed for #{tool_name}: #{e.message}" }
+            @executor.logger.error { e.backtrace.join("\n") }
+
+            # Append error to session JSON
+            append_to_session_json({
+              type: "tool_error",
+              api: "responses",
+              tool_name: tool_name,
+              arguments: tool_args_str,
+              error: {
+                class: e.class.to_s,
+                message: e.message,
+                backtrace: e.backtrace.first(5),
+              },
+            })
+
+            # Add error output to conversation
+            conversation << {
+              type: "function_call_output",
+              call_id: call_id,
+              output: { error: e.message }.to_json,
+            }
+          end
+        end
+
+        @executor.logger.info { "Responses API - Built conversation with #{conversation.size} function outputs" }
+        @executor.logger.debug { "Final conversation structure: #{JsonHandler.pretty_generate!(conversation)}" }
+        conversation
+      end
+
+      def append_new_outputs(function_calls, conversation)
+        # Only add the new function outputs
+        # Don't add function calls - the API already knows about them
+
+        function_calls.each do |fc|
+          # Execute and add output only
+          tool_name = fc["name"]
+          tool_args_str = fc["arguments"]
+          call_id = fc["call_id"]
+
+          begin
+            # Parse arguments
+            tool_args = JsonHandler.parse!(tool_args_str)
+
+            # Log tool execution
+            @executor.logger.info { "Responses API - Executing tool: #{tool_name} with args: #{JsonHandler.pretty_generate!(tool_args)}" }
+
+            # Execute tool via MCP
+            result = @mcp_client.call_tool(tool_name, tool_args)
+
+            # Log result
+            @executor.logger.info { "Responses API - Tool result for #{tool_name}: #{result}" }
+
+            # Add function output to conversation
+            conversation << {
+              type: "function_call_output",
+              call_id: call_id,
+              output: result.to_json, # Must be JSON string
+            }
+          rescue StandardError => e
+            @executor.logger.error { "Responses API - Tool execution failed for #{tool_name}: #{e.message}" }
+
+            # Add error output to conversation
+            conversation << {
+              type: "function_call_output",
+              call_id: call_id,
+              output: { error: e.message }.to_json,
+            }
+          end
+        end
+      end
+
+      def append_to_session_json(event)
+        # Delegate to the executor's log method
+        @executor.log(event) if @executor.respond_to?(:log)
+      end
+    end
+  end
+end