swarm_sdk 2.0.4 → 2.0.5

data/lib/swarm_sdk/tools/stores/{scratchpad_read_tracker.rb → storage_read_tracker.rb}

@@ -3,22 +3,22 @@
 module SwarmSDK
   module Tools
     module Stores
-      # ScratchpadReadTracker manages read-entry tracking for all agents
+      # StorageReadTracker manages read-entry tracking for all agents
       #
-      # This module maintains a global registry of which scratchpad entries each agent
+      # This module maintains a global registry of which memory entries each agent
       # has read during their conversation. This enables enforcement of the
       # "read-before-edit" rule that ensures agents have context before modifying entries.
       #
       # Each agent maintains an independent set of read entries, keyed by agent identifier.
-      module ScratchpadReadTracker
+      module StorageReadTracker
         @read_entries = {}
         @mutex = Mutex.new
 
         class << self
-          # Register that an agent has read a scratchpad entry
+          # Register that an agent has read a storage entry
           #
           # @param agent_id [Symbol] The agent identifier
-          # @param entry_path [String] The scratchpad entry path
+          # @param entry_path [String] The storage entry path
           def register_read(agent_id, entry_path)
             @mutex.synchronize do
               @read_entries[agent_id] ||= Set.new
@@ -26,10 +26,10 @@ module SwarmSDK
             end
           end
 
-          # Check if an agent has read a scratchpad entry
+          # Check if an agent has read a storage entry
           #
           # @param agent_id [Symbol] The agent identifier
-          # @param entry_path [String] The scratchpad entry path
+          # @param entry_path [String] The storage entry path
           # @return [Boolean] true if the agent has read this entry
           def entry_read?(agent_id, entry_path)
             @mutex.synchronize do

data/lib/swarm_sdk/tools/web_fetch.rb

@@ -0,0 +1,261 @@
+# frozen_string_literal: true
+
+module SwarmSDK
+  module Tools
+    # WebFetch tool for fetching and processing web content
+    #
+    # Fetches content from URLs, converts HTML to markdown, and processes it
+    # using an AI model to extract information based on a provided prompt.
+    class WebFetch < RubyLLM::Tool
+      def initialize
+        super()
+        @cache = {}
+        @cache_ttl = 900 # 15 minutes in seconds
+        @llm_enabled = SwarmSDK.settings.webfetch_llm_enabled?
+      end
+
+      def name
+        "WebFetch"
+      end
+
+      description <<~DESC
+        - Fetches content from a specified URL and converts it to markdown
+        - Optionally processes the content with an LLM if configured
+        - Fetches the URL content, converts HTML to markdown
+        - Returns markdown content or LLM analysis (based on configuration)
+        - Use this tool when you need to retrieve and analyze web content
+
+        Usage notes:
+          - IMPORTANT: If an MCP-provided web fetch tool is available, prefer using that tool instead of this one, as it may have fewer restrictions. All MCP-provided tools start with "mcp__".
+          - The URL must be a fully-formed valid URL
+          - HTTP URLs will be automatically upgraded to HTTPS
+          - This tool is read-only and does not modify any files
+          - Content will be truncated if very large
+          - Includes a self-cleaning 15-minute cache for faster responses
+          - When a URL redirects to a different host, the tool will inform you and provide the redirect URL in a special format. You should then make a new WebFetch request with the redirect URL to fetch the content.
+
+        LLM Processing:
+          - When SwarmSDK is configured with webfetch_provider and webfetch_model, the 'prompt' parameter is required
+          - The tool will process the markdown content with the configured LLM using your prompt
+          - Without this configuration, the tool returns raw markdown and the 'prompt' parameter is optional (ignored if provided)
+          - Configure with: SwarmSDK.configure { |c| c.webfetch_provider = "anthropic"; c.webfetch_model = "claude-3-5-haiku-20241022" }
+      DESC
+
+      param :url,
+        type: "string",
+        desc: "The URL to fetch content from",
+        required: true
+
+      param :prompt,
+        type: "string",
+        desc: "The prompt to run on the fetched content. Required when SwarmSDK is configured with webfetch_provider and webfetch_model. Optional otherwise (ignored if LLM processing not configured).",
+        required: false
+
+      MAX_CONTENT_LENGTH = 100_000 # characters
+      USER_AGENT = "SwarmSDK WebFetch Tool (https://github.com/parruda/claude-swarm)"
+      TIMEOUT = 30 # seconds
+
+      def execute(url:, prompt: nil)
+        # Validate inputs
+        return validation_error("url is required") if url.nil? || url.empty?
+
+        # Validate prompt when LLM processing is enabled
+        if @llm_enabled && (prompt.nil? || prompt.empty?)
+          return validation_error("prompt is required when LLM processing is configured")
+        end
+
+        # Validate and normalize URL
+        normalized_url = normalize_url(url)
+        return validation_error("Invalid URL format: #{url}") unless normalized_url
+
+        # Check cache first (cache key includes prompt if LLM is enabled)
+        cache_key = @llm_enabled ? "#{normalized_url}:#{prompt}" : normalized_url
+        cached = get_from_cache(cache_key)
+        return cached if cached
+
+        # Fetch the URL
+        fetch_result = fetch_url(normalized_url)
+        return fetch_result if fetch_result.is_a?(String) && fetch_result.start_with?("Error")
+
+        # Check for redirects to different hosts
+        if fetch_result[:redirect_url] && different_host?(normalized_url, fetch_result[:redirect_url])
+          return format_redirect_message(fetch_result[:redirect_url])
+        end
+
+        # Convert HTML to markdown
+        markdown_content = html_to_markdown(fetch_result[:body])
+
+        # Truncate if too long
+        if markdown_content.length > MAX_CONTENT_LENGTH
+          markdown_content = markdown_content[0...MAX_CONTENT_LENGTH]
+          markdown_content += "\n\n[Content truncated due to length]"
+        end
+
+        # Process with AI model if LLM is enabled, otherwise return markdown
+        result = if @llm_enabled
+          process_with_llm(markdown_content, prompt, normalized_url)
+        else
+          markdown_content
+        end
+
+        # Cache the result
+        store_in_cache(cache_key, result)
+
+        result
+      rescue StandardError => e
+        error("Unexpected error fetching URL: #{e.class.name} - #{e.message}")
+      end
+
+      private
+
+      def validation_error(message)
+        "<tool_use_error>InputValidationError: #{message}</tool_use_error>"
+      end
+
+      def error(message)
+        "Error: #{message}"
+      end
+
+      def normalize_url(url)
+        # Upgrade HTTP to HTTPS
+        url = url.sub(%r{^http://}, "https://")
+
+        # Validate URL format
+        uri = URI.parse(url)
+        return unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+        return unless uri.host
+
+        uri.to_s
+      rescue URI::InvalidURIError
+        nil
+      end
+
+      def different_host?(url1, url2)
+        uri1 = URI.parse(url1)
+        uri2 = URI.parse(url2)
+        uri1.host != uri2.host
+      rescue URI::InvalidURIError
+        false
+      end
+
+      def fetch_url(url)
+        require "faraday"
+        require "faraday/follow_redirects"
+
+        response = Faraday.new(url: url) do |conn|
+          conn.request(:url_encoded)
+          conn.response(:follow_redirects, limit: 5)
+          conn.adapter(Faraday.default_adapter)
+          conn.options.timeout = TIMEOUT
+          conn.options.open_timeout = TIMEOUT
+        end.get do |req|
+          req.headers["User-Agent"] = USER_AGENT
+          req.headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+        end
+
+        unless response.success?
+          return error("HTTP #{response.status}: Failed to fetch URL")
+        end
+
+        # Check final URL for redirects
+        final_url = response.env.url.to_s
+        redirect_url = final_url if final_url != url
+
+        {
+          body: response.body,
+          redirect_url: redirect_url,
+        }
+      rescue Faraday::TimeoutError
+        error("Request timed out after #{TIMEOUT} seconds")
+      rescue Faraday::ConnectionFailed => e
+        error("Connection failed: #{e.message}")
+      rescue StandardError => e
+        error("Failed to fetch URL: #{e.class.name} - #{e.message}")
+      end
+
+      def html_to_markdown(html)
+        # Use HtmlConverter to handle conversion with optional reverse_markdown gem
+        converter = DocumentConverters::HtmlConverter.new
+        converter.convert_string(html)
+      end
+
+      def process_with_llm(content, prompt, url)
+        # Use configured model for processing
+        # Format the prompt to include the content
+        full_prompt = <<~PROMPT
+          You are analyzing content from the URL: #{url}
+
+          User request: #{prompt}
+
+          Content:
+          #{content}
+
+          Please respond to the user's request based on the content above.
+        PROMPT
+
+        # Get settings
+        config = SwarmSDK.settings
+
+        # Build chat with configured provider and model
+        chat_params = {
+          model: config.webfetch_model,
+          provider: config.webfetch_provider.to_sym,
+        }
+        chat_params[:base_url] = config.webfetch_base_url if config.webfetch_base_url
+
+        chat = RubyLLM.chat(**chat_params).with_params(max_tokens: config.webfetch_max_tokens)
+
+        response = chat.ask(full_prompt)
+
+        # Extract the text response
+        response_text = response.content
+        return error("Failed to process content with LLM: No response text") unless response_text
+
+        response_text
+      rescue StandardError => e
+        error("Failed to process content with LLM: #{e.class.name} - #{e.message}")
+      end
+
+      def format_redirect_message(redirect_url)
+        <<~MESSAGE
+          This URL redirected to a different host.
+
+          Redirect URL: #{redirect_url}
+
+          <system-reminder>
+          The requested URL redirected to a different host. To fetch the content from the redirect URL,
+          make a new WebFetch request with the redirect URL provided above.
+          </system-reminder>
+        MESSAGE
+      end
+
+      def get_from_cache(key)
+        entry = @cache[key]
+        return unless entry
+
+        # Check if cache entry is still valid
+        if Time.now.to_i - entry[:timestamp] > @cache_ttl
+          @cache.delete(key)
+          return
+        end
+
+        entry[:value]
+      end
+
+      def store_in_cache(key, value)
+        # Clean old cache entries
+        clean_cache
+
+        @cache[key] = {
+          value: value,
+          timestamp: Time.now.to_i,
+        }
+      end
+
+      def clean_cache
+        now = Time.now.to_i
+        @cache.delete_if { |_key, entry| now - entry[:timestamp] > @cache_ttl }
+      end
+    end
+  end
+end

data/lib/swarm_sdk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SwarmSDK
-  VERSION = "2.0.4"
+  VERSION = "2.0.5"
 end

data/lib/swarm_sdk.rb

@@ -41,11 +41,50 @@ module SwarmSDK
   class StateError < Error; end
 
   class << self
+    # Settings for SwarmSDK (global configuration)
+    attr_accessor :settings
+
     # Main entry point for DSL
     def build(&block)
       Swarm::Builder.build(&block)
     end
+
+    # Configure SwarmSDK global settings
+    def configure
+      self.settings ||= Settings.new
+      yield(settings)
+    end
+
+    # Reset settings to defaults
+    def reset_settings!
+      self.settings = Settings.new
+    end
+
+    # Alias for backward compatibility
+    alias_method :configuration, :settings
+    alias_method :reset_configuration!, :reset_settings!
   end
+
+  # Settings class for SwarmSDK global settings (not to be confused with Configuration for YAML loading)
+  class Settings
+    # WebFetch tool LLM processing configuration
+    attr_accessor :webfetch_provider, :webfetch_model, :webfetch_base_url, :webfetch_max_tokens
+
+    def initialize
+      @webfetch_provider = nil
+      @webfetch_model = nil
+      @webfetch_base_url = nil
+      @webfetch_max_tokens = 4096
+    end
+
+    # Check if WebFetch LLM processing is enabled
+    def webfetch_llm_enabled?
+      !@webfetch_provider.nil? && !@webfetch_model.nil?
+    end
+  end
+
+  # Initialize default settings
+  self.settings = Settings.new
 end
 
 # Automatically configure RubyLLM from environment variables

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: swarm_sdk
 version: !ruby/object:Gem::Version
-  version: 2.0.4
+  version: 2.0.5
 platform: ruby
 authors:
 - Paulo Arruda
@@ -118,6 +118,7 @@ files:
 - lib/swarm_sdk/permissions/validator.rb
 - lib/swarm_sdk/permissions_builder.rb
 - lib/swarm_sdk/prompts/base_system_prompt.md.erb
+- lib/swarm_sdk/prompts/memory.md.erb
 - lib/swarm_sdk/providers/openai_with_responses.rb
 - lib/swarm_sdk/result.rb
 - lib/swarm_sdk/swarm.rb
@@ -130,6 +131,7 @@ files:
 - lib/swarm_sdk/tools/delegate.rb
 - lib/swarm_sdk/tools/document_converters/base_converter.rb
 - lib/swarm_sdk/tools/document_converters/docx_converter.rb
+- lib/swarm_sdk/tools/document_converters/html_converter.rb
 - lib/swarm_sdk/tools/document_converters/pdf_converter.rb
 - lib/swarm_sdk/tools/document_converters/xlsx_converter.rb
 - lib/swarm_sdk/tools/edit.rb
@@ -138,22 +140,29 @@ files:
 - lib/swarm_sdk/tools/image_extractors/docx_image_extractor.rb
 - lib/swarm_sdk/tools/image_extractors/pdf_image_extractor.rb
 - lib/swarm_sdk/tools/image_formats/tiff_builder.rb
+- lib/swarm_sdk/tools/memory/memory_delete.rb
+- lib/swarm_sdk/tools/memory/memory_edit.rb
+- lib/swarm_sdk/tools/memory/memory_glob.rb
+- lib/swarm_sdk/tools/memory/memory_grep.rb
+- lib/swarm_sdk/tools/memory/memory_multi_edit.rb
+- lib/swarm_sdk/tools/memory/memory_read.rb
+- lib/swarm_sdk/tools/memory/memory_write.rb
 - lib/swarm_sdk/tools/multi_edit.rb
 - lib/swarm_sdk/tools/path_resolver.rb
 - lib/swarm_sdk/tools/read.rb
 - lib/swarm_sdk/tools/registry.rb
-- lib/swarm_sdk/tools/scratchpad_edit.rb
-- lib/swarm_sdk/tools/scratchpad_glob.rb
-- lib/swarm_sdk/tools/scratchpad_grep.rb
-- lib/swarm_sdk/tools/scratchpad_multi_edit.rb
-- lib/swarm_sdk/tools/scratchpad_read.rb
-- lib/swarm_sdk/tools/scratchpad_write.rb
+- lib/swarm_sdk/tools/scratchpad/scratchpad_list.rb
+- lib/swarm_sdk/tools/scratchpad/scratchpad_read.rb
+- lib/swarm_sdk/tools/scratchpad/scratchpad_write.rb
+- lib/swarm_sdk/tools/stores/memory_storage.rb
 - lib/swarm_sdk/tools/stores/read_tracker.rb
-- lib/swarm_sdk/tools/stores/scratchpad.rb
-- lib/swarm_sdk/tools/stores/scratchpad_read_tracker.rb
+- lib/swarm_sdk/tools/stores/scratchpad_storage.rb
+- lib/swarm_sdk/tools/stores/storage.rb
+- lib/swarm_sdk/tools/stores/storage_read_tracker.rb
 - lib/swarm_sdk/tools/stores/todo_manager.rb
 - lib/swarm_sdk/tools/think.rb
 - lib/swarm_sdk/tools/todo_write.rb
+- lib/swarm_sdk/tools/web_fetch.rb
 - lib/swarm_sdk/tools/write.rb
 - lib/swarm_sdk/utils.rb
 - lib/swarm_sdk/version.rb

data/lib/swarm_sdk/tools/scratchpad_edit.rb

@@ -1,143 +0,0 @@
-# frozen_string_literal: true
-
-module SwarmSDK
-  module Tools
-    # Tool for editing scratchpad entries with exact string replacement
-    #
-    # Performs exact string replacements in scratchpad content.
-    # All agents in the swarm share the same scratchpad.
-    class ScratchpadEdit < RubyLLM::Tool
-      define_method(:name) { "ScratchpadEdit" }
-
-      description <<~DESC
-        Performs exact string replacements in scratchpad entries.
-        Works like the Edit tool but operates on scratchpad content instead of files.
-        You must use ScratchpadRead on the entry before editing it.
-        When editing text from ScratchpadRead output, ensure you preserve the exact indentation as it appears AFTER the line number prefix.
-        The line number prefix format is: spaces + line number + tab. Everything after that tab is the actual content to match.
-        Never include any part of the line number prefix in the old_string or new_string.
-        The edit will FAIL if old_string is not unique in the entry. Either provide a larger string with more surrounding context to make it unique or use replace_all to change every instance of old_string.
-        Use replace_all for replacing and renaming strings across the entry.
-      DESC
-
-      param :file_path,
-        desc: "Path to the scratchpad entry (e.g., 'analysis/report', 'parallel/batch1/task_0')",
-        required: true
-
-      param :old_string,
-        desc: "The exact text to replace (must match exactly including whitespace)",
-        required: true
-
-      param :new_string,
-        desc: "The text to replace it with (must be different from old_string)",
-        required: true
-
-      param :replace_all,
-        desc: "Replace all occurrences of old_string (default false)",
-        required: false
-
-      class << self
-        # Create a ScratchpadEdit tool for a specific scratchpad instance
-        #
-        # @param scratchpad [Stores::Scratchpad] Shared scratchpad instance
-        # @param agent_name [Symbol, String] Agent identifier for tracking reads
-        # @return [ScratchpadEdit] Tool instance
-        def create_for_scratchpad(scratchpad, agent_name)
-          new(scratchpad, agent_name)
-        end
-      end
-
-      # Initialize with scratchpad instance and agent name
-      #
-      # @param scratchpad [Stores::Scratchpad] Shared scratchpad instance
-      # @param agent_name [Symbol, String] Agent identifier
-      def initialize(scratchpad, agent_name)
-        super() # Call RubyLLM::Tool's initialize
-        @scratchpad = scratchpad
-        @agent_name = agent_name.to_sym
-      end
-
-      # Execute the tool
-      #
-      # @param file_path [String] Path to scratchpad entry
-      # @param old_string [String] Text to replace
-      # @param new_string [String] Replacement text
-      # @param replace_all [Boolean] Replace all occurrences
-      # @return [String] Success message or error
-      def execute(file_path:, old_string:, new_string:, replace_all: false)
-        # Validate inputs
-        return validation_error("file_path is required") if file_path.nil? || file_path.to_s.strip.empty?
-        return validation_error("old_string is required") if old_string.nil? || old_string.empty?
-        return validation_error("new_string is required") if new_string.nil?
-
-        # old_string and new_string must be different
-        if old_string == new_string
-          return validation_error("old_string and new_string must be different. They are currently identical.")
-        end
-
-        # Read current content (this will raise ArgumentError if entry doesn't exist)
-        content = scratchpad.read(file_path: file_path)
-
-        # Enforce read-before-edit
-        unless Stores::ScratchpadReadTracker.entry_read?(@agent_name, file_path)
-          return validation_error(
-            "Cannot edit scratchpad entry without reading it first. " \
-              "You must use ScratchpadRead on 'scratchpad://#{file_path}' before editing it. " \
-              "This ensures you have the current content to match against.",
-          )
-        end
-
-        # Check if old_string exists in content
-        unless content.include?(old_string)
-          return validation_error(<<~ERROR.chomp)
-            old_string not found in scratchpad entry. Make sure it matches exactly, including all whitespace and indentation.
-            Do not include line number prefixes from ScratchpadRead tool output.
-          ERROR
-        end
-
-        # Count occurrences
-        occurrences = content.scan(old_string).count
-
-        # If not replace_all and multiple occurrences, error
-        if !replace_all && occurrences > 1
-          return validation_error(<<~ERROR.chomp)
-            Found #{occurrences} occurrences of old_string.
-            Either provide more surrounding context to make the match unique, or use replace_all: true to replace all occurrences.
-          ERROR
-        end
-
-        # Perform replacement
-        new_content = if replace_all
-          content.gsub(old_string, new_string)
-        else
-          content.sub(old_string, new_string)
-        end
-
-        # Get existing entry metadata
-        entries = scratchpad.list
-        existing_entry = entries.find { |e| e[:path] == file_path }
-
-        # Write updated content back (preserving the title)
-        scratchpad.write(
-          file_path: file_path,
-          content: new_content,
-          title: existing_entry[:title],
-        )
-
-        # Build success message
-        replaced_count = replace_all ? occurrences : 1
-        "Successfully replaced #{replaced_count} occurrence(s) in scratchpad://#{file_path}"
-      rescue ArgumentError => e
-        validation_error(e.message)
-      end
-
-      private
-
-      attr_reader :scratchpad
-
-      def validation_error(message)
-        "<tool_use_error>InputValidationError: #{message}</tool_use_error>"
-      end
-    end
-  end
-end

data/lib/swarm_sdk/tools/scratchpad_glob.rb

@@ -1,92 +0,0 @@
-# frozen_string_literal: true
-
-module SwarmSDK
-  module Tools
-    # Tool for searching scratchpad entries by glob pattern
-    #
-    # Finds scratchpad entries matching a glob pattern (like filesystem glob).
-    # All agents in the swarm share the same scratchpad.
-    class ScratchpadGlob < RubyLLM::Tool
-      define_method(:name) { "ScratchpadGlob" }
-
-      description <<~DESC
-        Search scratchpad entries by glob pattern.
-        Works like filesystem glob - use * for wildcards, ** for recursive matching.
-        Use this to discover entries matching specific patterns.
-
-        Examples:
-        - "parallel/*" - all entries directly under parallel/
-        - "parallel/**" - all entries under parallel/ (recursive)
-        - "*/report" - all entries named "report" in any top-level directory
-        - "analysis/*/result_*" - entries like "analysis/foo/result_1"
-      DESC
-
-      param :pattern,
-        desc: "Glob pattern to match (e.g., '**/*.txt', 'parallel/*/task_*')",
-        required: true
-
-      class << self
-        # Create a ScratchpadGlob tool for a specific scratchpad instance
-        #
-        # @param scratchpad [Stores::Scratchpad] Shared scratchpad instance
-        # @return [ScratchpadGlob] Tool instance
-        def create_for_scratchpad(scratchpad)
-          new(scratchpad)
-        end
-      end
-
-      # Initialize with scratchpad instance
-      #
-      # @param scratchpad [Stores::Scratchpad] Shared scratchpad instance
-      def initialize(scratchpad)
-        super() # Call RubyLLM::Tool's initialize
-        @scratchpad = scratchpad
-      end
-
-      # Execute the tool
-      #
-      # @param pattern [String] Glob pattern to match
-      # @return [String] Formatted list of matching entries
-      def execute(pattern:)
-        entries = scratchpad.glob(pattern: pattern)
-
-        if entries.empty?
-          return "No entries found matching pattern '#{pattern}'"
-        end
-
-        result = []
-        result << "Scratchpad entries matching '#{pattern}' (#{entries.size} #{entries.size == 1 ? "entry" : "entries"}):"
-
-        entries.each do |entry|
-          result << "  scratchpad://#{entry[:path]} - \"#{entry[:title]}\" (#{format_bytes(entry[:size])})"
-        end
-
-        result.join("\n")
-      rescue ArgumentError => e
-        validation_error(e.message)
-      end
-
-      private
-
-      attr_reader :scratchpad
-
-      def validation_error(message)
-        "<tool_use_error>InputValidationError: #{message}</tool_use_error>"
-      end
-
-      # Format bytes to human-readable size
-      #
-      # @param bytes [Integer] Number of bytes
-      # @return [String] Formatted size
-      def format_bytes(bytes)
-        if bytes >= 1_000_000
-          "#{(bytes.to_f / 1_000_000).round(1)}MB"
-        elsif bytes >= 1_000
-          "#{(bytes.to_f / 1_000).round(1)}KB"
-        else
-          "#{bytes}B"
-        end
-      end
-    end
-  end
-end