swarm_sdk 2.0.3 → 2.0.5

data/lib/swarm_sdk/tools/stores/storage_read_tracker.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module Tools
+    module Stores
+      # StorageReadTracker manages read-entry tracking for all agents
+      #
+      # This module maintains a global registry of which memory entries each agent
+      # has read during their conversation. This enables enforcement of the
+      # "read-before-edit" rule that ensures agents have context before modifying entries.
+      #
+      # Each agent maintains an independent set of read entries, keyed by agent identifier.
+      module StorageReadTracker
+        @read_entries = {}
+        @mutex = Mutex.new
+
+        class << self
+          # Register that an agent has read a storage entry
+          #
+          # @param agent_id [Symbol] The agent identifier
+          # @param entry_path [String] The storage entry path
+          def register_read(agent_id, entry_path)
+            @mutex.synchronize do
+              @read_entries[agent_id] ||= Set.new
+              @read_entries[agent_id] << entry_path
+            end
+          end
+
+          # Check if an agent has read a storage entry
+          #
+          # @param agent_id [Symbol] The agent identifier
+          # @param entry_path [String] The storage entry path
+          # @return [Boolean] true if the agent has read this entry
+          def entry_read?(agent_id, entry_path)
+            @mutex.synchronize do
+              return false unless @read_entries[agent_id]
+
+              @read_entries[agent_id].include?(entry_path)
+            end
+          end
+
+          # Clear read history for an agent (useful for testing)
+          #
+          # @param agent_id [Symbol] The agent identifier
+          def clear(agent_id)
+            @mutex.synchronize do
+              @read_entries.delete(agent_id)
+            end
+          end
+
+          # Clear all read history (useful for testing)
+          def clear_all
+            @mutex.synchronize do
+              @read_entries.clear
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/swarm_sdk/tools/web_fetch.rb

@@ -0,0 +1,261 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module Tools
+    # WebFetch tool for fetching and processing web content
+    #
+    # Fetches content from URLs, converts HTML to markdown, and processes it
+    # using an AI model to extract information based on a provided prompt.
+    class WebFetch < RubyLLM::Tool
+      def initialize
+        super()
+        @cache = {}
+        @cache_ttl = 900 # 15 minutes in seconds
+        @llm_enabled = SwarmSDK.settings.webfetch_llm_enabled?
+      end
+
+      def name
+        "WebFetch"
+      end
+
+      description <<~DESC
+        - Fetches content from a specified URL and converts it to markdown
+        - Optionally processes the content with an LLM if configured
+        - Fetches the URL content, converts HTML to markdown
+        - Returns markdown content or LLM analysis (based on configuration)
+        - Use this tool when you need to retrieve and analyze web content
+
+        Usage notes:
+          - IMPORTANT: If an MCP-provided web fetch tool is available, prefer using that tool instead of this one, as it may have fewer restrictions. All MCP-provided tools start with "mcp__".
+          - The URL must be a fully-formed valid URL
+          - HTTP URLs will be automatically upgraded to HTTPS
+          - This tool is read-only and does not modify any files
+          - Content will be truncated if very large
+          - Includes a self-cleaning 15-minute cache for faster responses
+          - When a URL redirects to a different host, the tool will inform you and provide the redirect URL in a special format. You should then make a new WebFetch request with the redirect URL to fetch the content.
+
+        LLM Processing:
+          - When SwarmSDK is configured with webfetch_provider and webfetch_model, the 'prompt' parameter is required
+          - The tool will process the markdown content with the configured LLM using your prompt
+          - Without this configuration, the tool returns raw markdown and the 'prompt' parameter is optional (ignored if provided)
+          - Configure with: SwarmSDK.configure { |c| c.webfetch_provider = "anthropic"; c.webfetch_model = "claude-3-5-haiku-20241022" }
+      DESC
+
+      param :url,
+        type: "string",
+        desc: "The URL to fetch content from",
+        required: true
+
+      param :prompt,
+        type: "string",
+        desc: "The prompt to run on the fetched content. Required when SwarmSDK is configured with webfetch_provider and webfetch_model. Optional otherwise (ignored if LLM processing not configured).",
+        required: false
+
+      MAX_CONTENT_LENGTH = 100_000 # characters
+      USER_AGENT = "SwarmSDK WebFetch Tool (https://github.com/parruda/claude-swarm)"
+      TIMEOUT = 30 # seconds
+
+      def execute(url:, prompt: nil)
+        # Validate inputs
+        return validation_error("url is required") if url.nil? || url.empty?
+
+        # Validate prompt when LLM processing is enabled
+        if @llm_enabled && (prompt.nil? || prompt.empty?)
+          return validation_error("prompt is required when LLM processing is configured")
+        end
+
+        # Validate and normalize URL
+        normalized_url = normalize_url(url)
+        return validation_error("Invalid URL format: #{url}") unless normalized_url
+
+        # Check cache first (cache key includes prompt if LLM is enabled)
+        cache_key = @llm_enabled ? "#{normalized_url}:#{prompt}" : normalized_url
+        cached = get_from_cache(cache_key)
+        return cached if cached
+
+        # Fetch the URL
+        fetch_result = fetch_url(normalized_url)
+        return fetch_result if fetch_result.is_a?(String) && fetch_result.start_with?("Error")
+
+        # Check for redirects to different hosts
+        if fetch_result[:redirect_url] && different_host?(normalized_url, fetch_result[:redirect_url])
+          return format_redirect_message(fetch_result[:redirect_url])
+        end
+
+        # Convert HTML to markdown
+        markdown_content = html_to_markdown(fetch_result[:body])
+
+        # Truncate if too long
+        if markdown_content.length > MAX_CONTENT_LENGTH
+          markdown_content = markdown_content[0...MAX_CONTENT_LENGTH]
+          markdown_content += "\n\n[Content truncated due to length]"
+        end
+
+        # Process with AI model if LLM is enabled, otherwise return markdown
+        result = if @llm_enabled
+          process_with_llm(markdown_content, prompt, normalized_url)
+        else
+          markdown_content
+        end
+
+        # Cache the result
+        store_in_cache(cache_key, result)
+
+        result
+      rescue StandardError => e
+        error("Unexpected error fetching URL: #{e.class.name} - #{e.message}")
+      end
+
+      private
+
+      def validation_error(message)
+        "<tool_use_error>InputValidationError: #{message}</tool_use_error>"
+      end
+
+      def error(message)
+        "Error: #{message}"
+      end
+
+      def normalize_url(url)
+        # Upgrade HTTP to HTTPS
+        url = url.sub(%r{^http://}, "https://")
+
+        # Validate URL format
+        uri = URI.parse(url)
+        return unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+        return unless uri.host
+
+        uri.to_s
+      rescue URI::InvalidURIError
+        nil
+      end
+
+      def different_host?(url1, url2)
+        uri1 = URI.parse(url1)
+        uri2 = URI.parse(url2)
+        uri1.host != uri2.host
+      rescue URI::InvalidURIError
+        false
+      end
+
+      def fetch_url(url)
+        require "faraday"
+        require "faraday/follow_redirects"
+
+        response = Faraday.new(url: url) do |conn|
+          conn.request(:url_encoded)
+          conn.response(:follow_redirects, limit: 5)
+          conn.adapter(Faraday.default_adapter)
+          conn.options.timeout = TIMEOUT
+          conn.options.open_timeout = TIMEOUT
+        end.get do |req|
+          req.headers["User-Agent"] = USER_AGENT
+          req.headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+        end
+
+        unless response.success?
+          return error("HTTP #{response.status}: Failed to fetch URL")
+        end
+
+        # Check final URL for redirects
+        final_url = response.env.url.to_s
+        redirect_url = final_url if final_url != url
+
+        {
+          body: response.body,
+          redirect_url: redirect_url,
+        }
+      rescue Faraday::TimeoutError
+        error("Request timed out after #{TIMEOUT} seconds")
+      rescue Faraday::ConnectionFailed => e
+        error("Connection failed: #{e.message}")
+      rescue StandardError => e
+        error("Failed to fetch URL: #{e.class.name} - #{e.message}")
+      end
+
+      def html_to_markdown(html)
+        # Use HtmlConverter to handle conversion with optional reverse_markdown gem
+        converter = DocumentConverters::HtmlConverter.new
+        converter.convert_string(html)
+      end
+
+      def process_with_llm(content, prompt, url)
+        # Use configured model for processing
+        # Format the prompt to include the content
+        full_prompt = <<~PROMPT
+          You are analyzing content from the URL: #{url}
+
+          User request: #{prompt}
+
+          Content:
+          #{content}
+
+          Please respond to the user's request based on the content above.
+        PROMPT
+
+        # Get settings
+        config = SwarmSDK.settings
+
+        # Build chat with configured provider and model
+        chat_params = {
+          model: config.webfetch_model,
+          provider: config.webfetch_provider.to_sym,
+        }
+        chat_params[:base_url] = config.webfetch_base_url if config.webfetch_base_url
+
+        chat = RubyLLM.chat(**chat_params).with_params(max_tokens: config.webfetch_max_tokens)
+
+        response = chat.ask(full_prompt)
+
+        # Extract the text response
+        response_text = response.content
+        return error("Failed to process content with LLM: No response text") unless response_text
+
+        response_text
+      rescue StandardError => e
+        error("Failed to process content with LLM: #{e.class.name} - #{e.message}")
+      end
+
+      def format_redirect_message(redirect_url)
+        <<~MESSAGE
+          This URL redirected to a different host.
+
+          Redirect URL: #{redirect_url}
+
+          <system-reminder>
+          The requested URL redirected to a different host. To fetch the content from the redirect URL,
+          make a new WebFetch request with the redirect URL provided above.
+          </system-reminder>
+        MESSAGE
+      end
+
+      def get_from_cache(key)
+        entry = @cache[key]
+        return unless entry
+
+        # Check if cache entry is still valid
+        if Time.now.to_i - entry[:timestamp] > @cache_ttl
+          @cache.delete(key)
+          return
+        end
+
+        entry[:value]
+      end
+
+      def store_in_cache(key, value)
+        # Clean old cache entries
+        clean_cache
+
+        @cache[key] = {
+          value: value,
+          timestamp: Time.now.to_i,
+        }
+      end
+
+      def clean_cache
+        now = Time.now.to_i
+        @cache.delete_if { |_key, entry| now - entry[:timestamp] > @cache_ttl }
+      end
+    end
+  end
+end

data/lib/swarm_sdk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SwarmSDK
-  VERSION = "2.0.3"
+  VERSION = "2.0.5"
 end

data/lib/swarm_sdk.rb

@@ -41,11 +41,50 @@ module SwarmSDK
   class StateError < Error; end
 
   class << self
+    # Settings for SwarmSDK (global configuration)
+    attr_accessor :settings
+
     # Main entry point for DSL
     def build(&block)
       Swarm::Builder.build(&block)
     end
+
+    # Configure SwarmSDK global settings
+    def configure
+      self.settings ||= Settings.new
+      yield(settings)
+    end
+
+    # Reset settings to defaults
+    def reset_settings!
+      self.settings = Settings.new
+    end
+
+    # Alias for backward compatibility
+    alias_method :configuration, :settings
+    alias_method :reset_configuration!, :reset_settings!
   end
+
+  # Settings class for SwarmSDK global settings (not to be confused with Configuration for YAML loading)
+  class Settings
+    # WebFetch tool LLM processing configuration
+    attr_accessor :webfetch_provider, :webfetch_model, :webfetch_base_url, :webfetch_max_tokens
+
+    def initialize
+      @webfetch_provider = nil
+      @webfetch_model = nil
+      @webfetch_base_url = nil
+      @webfetch_max_tokens = 4096
+    end
+
+    # Check if WebFetch LLM processing is enabled
+    def webfetch_llm_enabled?
+      !@webfetch_provider.nil? && !@webfetch_model.nil?
+    end
+  end
+
+  # Initialize default settings
+  self.settings = Settings.new
 end
 
 # Automatically configure RubyLLM from environment variables

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: swarm_sdk
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 2.0.5
 platform: ruby
 authors:
 - Paulo Arruda
@@ -118,6 +118,7 @@ files:
 - lib/swarm_sdk/permissions/validator.rb
 - lib/swarm_sdk/permissions_builder.rb
 - lib/swarm_sdk/prompts/base_system_prompt.md.erb
+- lib/swarm_sdk/prompts/memory.md.erb
 - lib/swarm_sdk/providers/openai_with_responses.rb
 - lib/swarm_sdk/result.rb
 - lib/swarm_sdk/swarm.rb
@@ -130,6 +131,7 @@ files:
 - lib/swarm_sdk/tools/delegate.rb
 - lib/swarm_sdk/tools/document_converters/base_converter.rb
 - lib/swarm_sdk/tools/document_converters/docx_converter.rb
+- lib/swarm_sdk/tools/document_converters/html_converter.rb
 - lib/swarm_sdk/tools/document_converters/pdf_converter.rb
 - lib/swarm_sdk/tools/document_converters/xlsx_converter.rb
 - lib/swarm_sdk/tools/edit.rb
@@ -138,18 +140,29 @@ files:
 - lib/swarm_sdk/tools/image_extractors/docx_image_extractor.rb
 - lib/swarm_sdk/tools/image_extractors/pdf_image_extractor.rb
 - lib/swarm_sdk/tools/image_formats/tiff_builder.rb
+- lib/swarm_sdk/tools/memory/memory_delete.rb
+- lib/swarm_sdk/tools/memory/memory_edit.rb
+- lib/swarm_sdk/tools/memory/memory_glob.rb
+- lib/swarm_sdk/tools/memory/memory_grep.rb
+- lib/swarm_sdk/tools/memory/memory_multi_edit.rb
+- lib/swarm_sdk/tools/memory/memory_read.rb
+- lib/swarm_sdk/tools/memory/memory_write.rb
 - lib/swarm_sdk/tools/multi_edit.rb
 - lib/swarm_sdk/tools/path_resolver.rb
 - lib/swarm_sdk/tools/read.rb
 - lib/swarm_sdk/tools/registry.rb
-- lib/swarm_sdk/tools/scratchpad_list.rb
-- lib/swarm_sdk/tools/scratchpad_read.rb
-- lib/swarm_sdk/tools/scratchpad_write.rb
+- lib/swarm_sdk/tools/scratchpad/scratchpad_list.rb
+- lib/swarm_sdk/tools/scratchpad/scratchpad_read.rb
+- lib/swarm_sdk/tools/scratchpad/scratchpad_write.rb
+- lib/swarm_sdk/tools/stores/memory_storage.rb
 - lib/swarm_sdk/tools/stores/read_tracker.rb
-- lib/swarm_sdk/tools/stores/scratchpad.rb
+- lib/swarm_sdk/tools/stores/scratchpad_storage.rb
+- lib/swarm_sdk/tools/stores/storage.rb
+- lib/swarm_sdk/tools/stores/storage_read_tracker.rb
 - lib/swarm_sdk/tools/stores/todo_manager.rb
 - lib/swarm_sdk/tools/think.rb
 - lib/swarm_sdk/tools/todo_write.rb
+- lib/swarm_sdk/tools/web_fetch.rb
 - lib/swarm_sdk/tools/write.rb
 - lib/swarm_sdk/utils.rb
 - lib/swarm_sdk/version.rb

data/lib/swarm_sdk/tools/scratchpad_list.rb

@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-module SwarmSDK
-  module Tools
-    # Tool for listing scratchpad entries with metadata
-    #
-    # Lists available scratchpad entries with titles and sizes.
-    # Supports filtering by path prefix.
-    class ScratchpadList < RubyLLM::Tool
-      define_method(:name) { "ScratchpadList" }
-
-      description <<~DESC
-        List available scratchpad entries with titles and metadata.
-        Use this to discover what content is available in scratchpad memory.
-        Optionally filter by path prefix.
-      DESC
-
-      param :prefix,
-        desc: "Filter by path prefix (e.g., 'parallel/', 'analysis/'). Leave empty to list all entries.",
-        required: false
-
-      class << self
-        # Create a ScratchpadList tool for a specific scratchpad instance
-        #
-        # @param scratchpad [Stores::Scratchpad] Shared scratchpad instance
-        # @return [ScratchpadList] Tool instance
-        def create_for_scratchpad(scratchpad)
-          new(scratchpad)
-        end
-      end
-
-      # Initialize with scratchpad instance
-      #
-      # @param scratchpad [Stores::Scratchpad] Shared scratchpad instance
-      def initialize(scratchpad)
-        super() # Call RubyLLM::Tool's initialize
-        @scratchpad = scratchpad
-      end
-
-      # Execute the tool
-      #
-      # @param prefix [String, nil] Optional path prefix filter
-      # @return [String] Formatted list of entries
-      def execute(prefix: nil)
-        entries = scratchpad.list(prefix: prefix)
-
-        if entries.empty?
-          return "Scratchpad is empty" if prefix.nil? || prefix.empty?
-
-          return "No entries found with prefix '#{prefix}'"
-        end
-
-        result = []
-        result << "Scratchpad contents (#{entries.size} #{entries.size == 1 ? "entry" : "entries"}):"
-
-        entries.each do |entry|
-          result << "  scratchpad://#{entry[:path]} - \"#{entry[:title]}\" (#{format_bytes(entry[:size])})"
-        end
-
-        result.join("\n")
-      rescue ArgumentError => e
-        validation_error(e.message)
-      end
-
-      private
-
-      attr_reader :scratchpad
-
-      def validation_error(message)
-        "<tool_use_error>InputValidationError: #{message}</tool_use_error>"
-      end
-
-      # Format bytes to human-readable size
-      #
-      # @param bytes [Integer] Number of bytes
-      # @return [String] Formatted size
-      def format_bytes(bytes)
-        if bytes >= 1_000_000
-          "#{(bytes.to_f / 1_000_000).round(1)}MB"
-        elsif bytes >= 1_000
-          "#{(bytes.to_f / 1_000).round(1)}KB"
-        else
-          "#{bytes}B"
-        end
-      end
-    end
-  end
-end

data/lib/swarm_sdk/tools/scratchpad_read.rb

@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-module SwarmSDK
-  module Tools
-    # Tool for reading content from scratchpad memory
-    #
-    # Retrieves content stored by any agent using scratchpad_write.
-    # All agents in the swarm share the same scratchpad.
-    class ScratchpadRead < RubyLLM::Tool
-      define_method(:name) { "ScratchpadRead" }
-
-      description <<~DESC
-        Read content from scratchpad.
-        Use this to retrieve detailed outputs, analysis, or results that were
-        stored using scratchpad_write. Any agent can read any scratchpad content.
-      DESC
-
-      param :file_path,
-        desc: "Path to read from scratchpad (e.g., 'analysis/report', 'parallel/batch1/task_0')",
-        required: true
-
-      class << self
-        # Create a ScratchpadRead tool for a specific scratchpad instance
-        #
-        # @param scratchpad [Stores::Scratchpad] Shared scratchpad instance
-        # @return [ScratchpadRead] Tool instance
-        def create_for_scratchpad(scratchpad)
-          new(scratchpad)
-        end
-      end
-
-      # Initialize with scratchpad instance
-      #
-      # @param scratchpad [Stores::Scratchpad] Shared scratchpad instance
-      def initialize(scratchpad)
-        super() # Call RubyLLM::Tool's initialize
-        @scratchpad = scratchpad
-      end
-
-      # Execute the tool
-      #
-      # @param file_path [String] Path to read from
-      # @return [String] Content at the path or error message
-      def execute(file_path:)
-        scratchpad.read(file_path: file_path)
-      rescue ArgumentError => e
-        validation_error(e.message)
-      end
-
-      private
-
-      attr_reader :scratchpad
-
-      def validation_error(message)
-        "<tool_use_error>InputValidationError: #{message}</tool_use_error>"
-      end
-    end
-  end
-end

data/lib/swarm_sdk/tools/scratchpad_write.rb

@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-module SwarmSDK
-  module Tools
-    # Tool for writing content to scratchpad memory
-    #
-    # Stores content in session-scoped, in-memory storage with metadata.
-    # All agents in the swarm share the same scratchpad.
-    class ScratchpadWrite < RubyLLM::Tool
-      define_method(:name) { "ScratchpadWrite" }
-
-      description <<~DESC
-        Store content in scratchpad for later retrieval.
-        Use this to save detailed outputs, analysis, or results that would
-        otherwise bloat tool responses. Any agent can read this content using scratchpad_read.
-
-        IMPORTANT: You must determine the appropriate file_path based on the task you're performing.
-        Choose a logical, descriptive path that reflects the content type and purpose.
-        Examples: 'analysis/code_review', 'research/findings', 'parallel/batch_1/results', 'logs/debug_trace'
-      DESC
-
-      param :file_path,
-        desc: "File-path-like address you determine based on the task (e.g., 'analysis/report', 'parallel/batch1/task_0')",
-        required: true
-
-      param :content,
-        desc: "Content to store in scratchpad (max 1MB per entry)",
-        required: true
-
-      param :title,
-        desc: "Brief title describing the content (shown in listings)",
-        required: true
-
-      class << self
-        # Create a ScratchpadWrite tool for a specific scratchpad instance
-        #
-        # @param scratchpad [Stores::Scratchpad] Shared scratchpad instance
-        # @return [ScratchpadWrite] Tool instance
-        def create_for_scratchpad(scratchpad)
-          new(scratchpad)
-        end
-      end
-
-      # Initialize with scratchpad instance
-      #
-      # @param scratchpad [Stores::Scratchpad] Shared scratchpad instance
-      def initialize(scratchpad)
-        super() # Call RubyLLM::Tool's initialize
-        @scratchpad = scratchpad
-      end
-
-      # Execute the tool
-      #
-      # @param file_path [String] Path to store content
-      # @param content [String] Content to store
-      # @param title [String] Brief title
-      # @return [String] Success message with path and size
-      def execute(file_path:, content:, title:)
-        entry = scratchpad.write(file_path: file_path, content: content, title: title)
-        "Stored at scratchpad://#{file_path} (#{format_bytes(entry.size)})"
-      rescue ArgumentError => e
-        validation_error(e.message)
-      end
-
-      private
-
-      attr_reader :scratchpad
-
-      def validation_error(message)
-        "<tool_use_error>InputValidationError: #{message}</tool_use_error>"
-      end
-
-      # Format bytes to human-readable size
-      #
-      # @param bytes [Integer] Number of bytes
-      # @return [String] Formatted size
-      def format_bytes(bytes)
-        if bytes >= 1_000_000
-          "#{(bytes.to_f / 1_000_000).round(1)}MB"
-        elsif bytes >= 1_000
-          "#{(bytes.to_f / 1_000).round(1)}KB"
-        else
-          "#{bytes}B"
-        end
-      end
-    end
-  end
-end