shared_tools 0.4.4 → 0.6.0

data/lib/shared_tools/tools/web_fetch_tool.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require_relative "../../shared_tools"
+require_relative "http_helpers"
+
+module SharedTools
+  module Tools
+    # Fetches a URL over http/https and returns its readable text (HTML is
+    # stripped to text). Every request and redirect hop passes through
+    # UrlGuard, so it can't be pointed at internal/loopback/metadata
+    # addresses. Read-only — no authorization prompt required.
+    #
+    # @example
+    #   tool = SharedTools::Tools::WebFetchTool.new
+    #   tool.execute(url: "https://example.com")
+    class WebFetchTool < ::RubyLLM::Tool
+      include HttpHelpers
+
+      def self.name = 'web_fetch'
+
+      description "Fetch a web page or text/JSON resource over http/https and return its readable " \
+                  "content (HTML is converted to text). Follows redirects safely. Cannot reach " \
+                  "private, loopback, or cloud-metadata addresses."
+
+      parameters do
+        string :url, description: "The http/https URL to fetch."
+      end
+
+      # @param logger [Logger] optional logger
+      def initialize(logger: nil)
+        @logger = logger || RubyLLM.logger
+      end
+
+      # @param url [String]
+      #
+      # @return [String, Hash]
+      def execute(url:)
+        @logger.info("#{self.class.name}#execute url=#{url.inspect}")
+
+        response = guarded_get(url)
+        return { error: "HTTP #{response.status} from #{response.final_url}" } if response.status >= 400
+
+        content_type = content_type_of(response)
+        text = extract_text(response.body, content_type)
+
+        header = "GET #{response.final_url} -> #{response.status} (#{content_type})"
+        "#{header}\n\n#{text}".strip
+      rescue SharedTools::UrlGuard::Blocked => e
+        @logger.error("#{self.class.name} url blocked: #{e.message}")
+        { error: e.message }
+      rescue HttpHelpers::FetchError => e
+        @logger.error("#{self.class.name} fetch failed: #{e.message}")
+        { error: "fetch failed: #{e.message}" }
+      end
+
+      private
+
+      def content_type_of(response)
+        Array(response.headers["content-type"]).first.to_s.split(";").first.to_s.strip
+      end
+
+      def extract_text(body, content_type)
+        text = body.to_s.scrub
+        return collapse(text) unless content_type =~ /html|xml/i
+
+        text = text.dup
+        text.gsub!(%r{<script\b[^>]*>.*?</script>}mi, " ")
+        text.gsub!(%r{<style\b[^>]*>.*?</style>}mi, " ")
+        text.gsub!(/<!--.*?-->/m, " ")
+        text.gsub!(%r{</(?:p|div|br|li|h[1-6]|tr|section|article)\s*/?>}i, "\n")
+        text.gsub!(/<[^>]+>/, " ")
+        collapse(decode_entities(text))
+      end
+
+      def decode_entities(str)
+        str.gsub("&amp;", "&").gsub("&lt;", "<").gsub("&gt;", ">")
+           .gsub("&quot;", '"').gsub("&#39;", "'").gsub("&nbsp;", " ")
+      end
+
+      def collapse(str)
+        str.gsub(/[ \t]+/, " ").gsub(/\n[ \t]*/, "\n").gsub(/\n{3,}/, "\n\n").strip
+      end
+    end
+  end
+end

data/lib/shared_tools/tools/workflow_manager_tool.rb

@@ -27,7 +27,7 @@ module SharedTools
         All workflow state is persisted to disk and can survive process restarts.
       DESCRIPTION
 
-      params do
+      parameters do
         string :action, description: <<~DESC.strip
           Workflow management action to perform:
           - 'start': Initialize a new workflow with initial data and return workflow ID

data/lib/shared_tools/tools/yaml_query_tool.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'json'
+require 'date'
+require_relative '../../shared_tools'
+
+module SharedTools
+  module Tools
+    # Parses YAML (from a file or an inline string) and either pretty-prints
+    # it (as JSON, for readability) or extracts a value with a dot/bracket
+    # path. Uses safe_load — no arbitrary Ruby objects are instantiated from
+    # the document.
+    #
+    # Path syntax matches JsonQueryTool: users[0].name, users[].email, config.port
+    #
+    # @example
+    #   tool = SharedTools::Tools::YamlQueryTool.new
+    #   tool.execute(path: "./docker-compose.yml", query: "services[0].image")
+    class YamlQueryTool < ::RubyLLM::Tool
+      MAX_BYTES = 5 * 1024 * 1024
+      PERMITTED = [Date, Time, Symbol].freeze
+
+      def self.name = 'yaml_query'
+
+      description "Query YAML with a path expression, or pretty-print it. Provide either a file " \
+                  "path or an inline yaml string. Loaded safely (no object deserialization). Path " \
+                  "syntax: keys separated by dots, [n] for array index, [] to map over an array."
+
+      parameters do
+        string :path,  description: "YAML file to read. Provide this or yaml.", required: false
+        string :yaml,  description: "Inline YAML string. Provide this or path.", required: false
+        string :query, description: "Path expression to extract, e.g. 'services[0].image'. Omit to pretty-print. Optional.", required: false
+      end
+
+      # @param logger [Logger] optional logger
+      def initialize(logger: nil)
+        @logger = logger || RubyLLM.logger
+      end
+
+      # @param path [String, nil]
+      # @param yaml [String, nil]
+      # @param query [String, nil]
+      #
+      # @return [String, Hash]
+      def execute(path: nil, yaml: nil, query: nil)
+        @logger.info("#{self.class.name}#execute path=#{path.inspect} query=#{query.inspect}")
+
+        source = load_source(path, yaml)
+        return source if source.is_a?(Hash) # error
+
+        data = YAML.safe_load(source, permitted_classes: PERMITTED, aliases: true)
+        result = query.to_s.strip.empty? ? data : SharedTools::DataPath.query(data, query)
+        JSON.pretty_generate(result)
+      rescue Psych::DisallowedClass => e
+        @logger.error("#{self.class.name} unsafe YAML: #{e.message}")
+        { error: "YAML contains a disallowed type: #{e.message}" }
+      rescue Psych::SyntaxError => e
+        @logger.error("#{self.class.name} invalid YAML: #{e.message}")
+        { error: "invalid YAML: #{e.message}" }
+      rescue SharedTools::DataPath::Error => e
+        @logger.error("#{self.class.name} bad query: #{e.message}")
+        { error: e.message }
+      rescue => e
+        @logger.error("#{self.class.name} failed: #{e.message}")
+        { error: e.message }
+      end
+
+      private
+
+      def load_source(path, yaml)
+        if path && !path.to_s.empty?
+          return { error: "file not found: #{path}" } unless File.exist?(path)
+          return { error: "not a file: #{path}" } unless File.file?(path)
+          return { error: "file too large (> #{MAX_BYTES} bytes)" } if File.size(path) > MAX_BYTES
+
+          File.read(path).scrub
+        elsif yaml && !yaml.to_s.empty?
+          yaml.to_s
+        else
+          { error: "provide either path or yaml" }
+        end
+      end
+    end
+  end
+end

data/lib/shared_tools/tree_tool.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+# Shim: require 'shared_tools/tree_tool'
+require 'shared_tools'
+require 'shared_tools/tools/tree_tool'

data/lib/shared_tools/url_guard.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require "uri"
+require "ipaddr"
+require "resolv"
+
+module SharedTools
+  # SSRF defense for tools that fetch arbitrary URLs (WebFetchTool,
+  # HttpRequestTool, DownloadFileTool). It:
+  #   - allows only http/https,
+  #   - rejects embedded credentials,
+  #   - enforces optional domain allow/deny lists,
+  #   - resolves the host and blocks the request if ANY resolved address is
+  #     private, loopback, link-local (incl. the cloud metadata IP), CGNAT,
+  #     unique-local IPv6, or otherwise reserved.
+  #
+  # resolve! also returns a vetted IP so the caller can pin the socket to it
+  # (Net::HTTP#ipaddr=), closing the DNS-rebinding window: the address that
+  # was vetted is exactly the one connected to, while TLS/SNI/cert checks
+  # still use the hostname. Re-run resolve! on every redirect hop (the http
+  # helper does).
+  class UrlGuard
+    class Blocked < StandardError; end
+
+    # A vetted result: the parsed URI plus a concrete IP that every
+    # connection should be pinned to, so a second DNS lookup at connect time
+    # can't swap in a private address (DNS rebinding).
+    Resolution = Struct.new(:uri, :address, keyword_init: true)
+
+    ALLOWED_SCHEMES = %w[http https].freeze
+
+    BLOCKED_RANGES = %w[
+      0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 127.0.0.0/8 169.254.0.0/16
+      172.16.0.0/12 192.0.0.0/24 192.0.2.0/24 192.168.0.0/16 198.18.0.0/15
+      198.51.100.0/24 203.0.113.0/24 224.0.0.0/4 240.0.0.0/4 255.255.255.255/32
+      ::1/128 ::/128 fc00::/7 fe80::/10
+    ].map { |cidr| IPAddr.new(cidr) }.freeze
+
+    def initialize(allowlist: [], denylist: [])
+      @allowlist = Array(allowlist).map { |d| d.to_s.downcase }
+      @denylist = Array(denylist).map { |d| d.to_s.downcase }
+    end
+
+    # Returns a parsed URI if the URL is safe to fetch; raises Blocked
+    # otherwise.
+    def check!(url)
+      resolve!(url).uri
+    end
+
+    # Like check!, but also returns a vetted IP address to pin the connection
+    # to (see Resolution). Raises Blocked otherwise.
+    def resolve!(url)
+      uri = parse(url)
+      raise Blocked, "only http/https URLs are allowed" unless ALLOWED_SCHEMES.include?(uri.scheme)
+      raise Blocked, "URL must include a host" if uri.host.nil? || uri.host.empty?
+      raise Blocked, "URLs with embedded credentials are not allowed" if uri.userinfo
+
+      host = uri.host.downcase
+      enforce_domain_lists(host)
+      address = vetted_address(host)
+      Resolution.new(uri: uri, address: address)
+    end
+
+    private
+
+    def parse(url)
+      URI.parse(url.to_s)
+    rescue URI::InvalidURIError => e
+      raise Blocked, "invalid URL: #{e.message}"
+    end
+
+    def enforce_domain_lists(host)
+      raise Blocked, "host is denylisted: #{host}" if @denylist.any? { |d| host == d || host.end_with?(".#{d}") }
+      return if @allowlist.empty?
+
+      allowed = @allowlist.any? { |d| host == d || host.end_with?(".#{d}") }
+      raise Blocked, "host is not on the allowlist: #{host}" unless allowed
+    end
+
+    # Resolve the host, block the request if ANY resolved address is in a
+    # blocked range, and return the first address for the caller to pin to.
+    def vetted_address(host)
+      addresses = resolve(host)
+      raise Blocked, "could not resolve host: #{host}" if addresses.empty?
+
+      addresses.each do |addr|
+        raise Blocked, "host resolves to a blocked address (#{addr})" if blocked_ip?(addr)
+      end
+      addresses.first
+    end
+
+    def resolve(host)
+      Resolv.getaddresses(host)
+    rescue StandardError
+      []
+    end
+
+    def blocked_ip?(addr)
+      ip = IPAddr.new(addr.to_s)
+      ip = ip.native if ip.respond_to?(:ipv4_mapped?) && ip.ipv4_mapped?
+      BLOCKED_RANGES.any? { |range| range.include?(ip) }
+    rescue IPAddr::Error
+      true # if we can't parse it, don't fetch it
+    end
+  end
+end

data/lib/shared_tools/utilities.rb

@@ -15,8 +15,8 @@ module SharedTools
     # MCP load tracking
     # ---------------------------------------------------------------------------
 
-    # Record the outcome of loading a single MCP client.
-    # Called by mcp.rb from within each loader thread.
+    # Record whether a single MCP server definition is usable (its required
+    # environment variables are set). Called by mcp.rb for each server class.
     #
     #   SharedTools.record_mcp_result("tavily")              # success
     #   SharedTools.record_mcp_result("notion", error: e)    # failure

data/lib/shared_tools/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SharedTools
-  VERSION = "0.4.4"
+  VERSION = "0.6.0"
 end

data/lib/shared_tools/web_fetch_tool.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+# Shim: require 'shared_tools/web_fetch_tool'
+require 'shared_tools'
+require 'shared_tools/tools/web_fetch_tool'

data/lib/shared_tools/yaml_query_tool.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+# Shim: require 'shared_tools/yaml_query_tool'
+require 'shared_tools'
+require 'shared_tools/tools/yaml_query_tool'

data/lib/shared_tools.rb

@@ -17,8 +17,8 @@ SharedToolsLoader.ignore("#{__dir__}/shared_tools/tools/disk.rb")
 SharedToolsLoader.ignore("#{__dir__}/shared_tools/tools/doc.rb")
 SharedToolsLoader.ignore("#{__dir__}/shared_tools/tools/docker.rb")
 SharedToolsLoader.ignore("#{__dir__}/shared_tools/tools/eval.rb")
+SharedToolsLoader.ignore("#{__dir__}/shared_tools/tools/git.rb")
 SharedToolsLoader.ignore("#{__dir__}/shared_tools/tools/notification.rb")
-SharedToolsLoader.ignore("#{__dir__}/shared_tools/tools/version.rb")  # Defines VERSION constant, not Version class
 SharedToolsLoader.ignore("#{__dir__}/shared_tools/tools/incomplete")  # Empty/incomplete tools directory
 SharedToolsLoader.ignore("#{__dir__}/shared_tools/tools/enabler.rb") # Experimental; defines Tools::Enabler not SharedTools::Tools::Enabler