kward 0.67.1 → 0.69.0

data/lib/kward/tools/search/web_fetch.rb

@@ -0,0 +1,202 @@
+require "nokogiri"
+require "uri"
+require_relative "web"
+
+# Namespace for the Kward CLI agent runtime.
+module Kward
+  # Fetches specific web resources for agent research workflows.
+  class WebFetch
+    DEFAULT_MAX_BYTES = 16 * 1024
+    MAX_MAX_BYTES = 128 * 1024
+    MAX_REDIRECTS = 5
+    HTTP_TIMEOUT_SECONDS = 10
+
+    # Creates a fetcher for web content and raw resources.
+    def initialize(http_client: WebSearch::NetHttpClient.new)
+      @http_client = http_client
+    end
+
+    # Fetches a URL and extracts readable text for human-facing pages.
+    def fetch_content(args)
+      url = args_value(args, "url").to_s.strip
+      return "Error: url is required" if url.empty?
+
+      max_bytes = bounded_max_bytes(args_value(args, "max_bytes") || args_value(args, "maxBytes"))
+      extract = normalize_extract(args_value(args, "extract") || "auto")
+      return "Error: extract must be one of: auto, text, markdown" unless extract
+
+      response = fetch_url(url, max_bytes: max_bytes)
+      return response if response.is_a?(String)
+
+      body = response[:body].to_s
+      content_type = header_value(response[:headers], "content-type")
+      text = extract_readable_text(body, content_type: content_type, mode: extract)
+      text = truncate_bytes(text, max_bytes)
+
+      [
+        "# Fetched content",
+        "- URL: #{response[:url]}",
+        "- Content type: #{content_type.empty? ? "unknown" : content_type}",
+        "- Bytes returned: #{text.bytesize}",
+        "",
+        text.empty? ? "(No readable text extracted.)" : text
+      ].join("\n")
+    rescue StandardError => e
+      "Error: fetch_content failed: #{e.message}"
+    end
+
+    # Fetches a URL and returns bounded raw response content.
+    def fetch_raw(args)
+      url = args_value(args, "url").to_s.strip
+      return "Error: url is required" if url.empty?
+
+      max_bytes = bounded_max_bytes(args_value(args, "max_bytes") || args_value(args, "maxBytes"))
+      accept = args_value(args, "accept").to_s.strip
+      response = fetch_url(url, max_bytes: max_bytes, accept: accept.empty? ? "*/*" : accept)
+      return response if response.is_a?(String)
+
+      body = truncate_bytes(response[:body].to_s, max_bytes)
+      content_type = header_value(response[:headers], "content-type")
+      [
+        "# Fetched raw content",
+        "- URL: #{response[:url]}",
+        "- Content type: #{content_type.empty? ? "unknown" : content_type}",
+        "- Bytes returned: #{body.bytesize}",
+        "",
+        body
+      ].join("\n")
+    rescue StandardError => e
+      "Error: fetch_raw failed: #{e.message}"
+    end
+
+    private
+
+    def fetch_url(url, max_bytes:, accept: "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.8")
+      current_url = normalize_url(url)
+      redirects = 0
+
+      loop do
+        response = @http_client.get(current_url, headers: browser_headers(accept))
+        code = response.code.to_i
+        headers = response_headers(response)
+
+        if redirect?(code)
+          return "Error: too many redirects" if redirects >= MAX_REDIRECTS
+
+          location = header_value(headers, "location")
+          return "Error: redirect missing Location header" if location.empty?
+
+          current_url = normalize_url(URI.join(current_url, location).to_s)
+          redirects += 1
+          next
+        end
+
+        return "Error: fetch failed with HTTP #{response.code}" unless code.between?(200, 299)
+
+        body = response.body.to_s
+        body = truncate_bytes(body, max_bytes)
+        return { url: current_url, headers: headers, body: body }
+      end
+    end
+
+    def normalize_url(value)
+      uri = URI.parse(value.to_s.strip)
+      raise "url must use http or https" unless %w[http https].include?(uri.scheme)
+      raise "url host is required" if uri.host.to_s.empty?
+
+      uri.to_s
+    rescue URI::InvalidURIError
+      raise "invalid url"
+    end
+
+    def response_headers(response)
+      return {} unless response.respond_to?(:headers) && response.headers.is_a?(Hash)
+
+      response.headers.transform_keys { |key| key.to_s.downcase }
+    end
+
+    def header_value(headers, key)
+      headers[key.to_s.downcase].to_s
+    end
+
+    def redirect?(code)
+      [301, 302, 303, 307, 308].include?(code)
+    end
+
+    def bounded_max_bytes(value)
+      number = value.to_i
+      number = DEFAULT_MAX_BYTES if number <= 0
+      [number, MAX_MAX_BYTES].min
+    end
+
+    def normalize_extract(value)
+      normalized = value.to_s.strip.downcase
+      %w[auto text markdown].include?(normalized) ? normalized : nil
+    end
+
+    def extract_readable_text(body, content_type:, mode:)
+      return clean_text(body) if mode == "text" || !html_content?(content_type, body)
+
+      document = Nokogiri::HTML(body)
+      document.css("script, style, noscript, svg, nav, footer, form").remove
+      title = document.at_css("title")&.text.to_s.strip
+      root = document.at_css("article") || document.at_css("main") || document.at_css("body") || document
+      parts = []
+      parts << "# #{clean_text(title)}" unless title.empty?
+      root.css("h1, h2, h3, h4, h5, h6, p, li, pre, code, blockquote").each do |node|
+        text = clean_text(node.text)
+        next if text.empty?
+
+        parts << format_html_node(node, text, mode: mode)
+      end
+      parts.uniq.join("\n\n")
+    end
+
+    def html_content?(content_type, body)
+      content_type.to_s.include?("html") || body.to_s.lstrip.start_with?("<!DOCTYPE html", "<html", "<HTML")
+    end
+
+    def format_html_node(node, text, mode:)
+      return text if mode == "text"
+
+      case node.name
+      when /^h([1-6])$/
+        "#{"#" * Regexp.last_match(1).to_i} #{text}"
+      when "li"
+        "- #{text}"
+      when "pre", "code"
+        "```\n#{text}\n```"
+      when "blockquote"
+        "> #{text}"
+      else
+        text
+      end
+    end
+
+    def clean_text(text)
+      text.to_s.gsub(/\s+/, " ").strip
+    end
+
+    def truncate_bytes(text, max_bytes)
+      return text if text.bytesize <= max_bytes
+
+      "#{text.byteslice(0, max_bytes).to_s.scrub}\n... truncated to #{max_bytes} bytes"
+    end
+
+    def browser_headers(accept)
+      {
+        "Accept" => accept,
+        "Accept-Language" => "en-US,en;q=0.9",
+        "User-Agent" => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
+      }
+    end
+
+    def args_value(args, key)
+      return nil unless args.is_a?(Hash)
+      return args[key] if args.key?(key)
+      return args[key.to_sym] if args.key?(key.to_sym)
+
+      nil
+    end
+  end
+end

data/lib/kward/tools/tool_call.rb

@@ -1,6 +1,14 @@
 require "json"
+require_relative "../message_access"
 
+# Namespace for the Kward CLI agent runtime.
 module Kward
+  # Reads and normalizes model tool-call hashes.
+  #
+  # Tool calls arrive from several providers and may be restored from session
+  # files. This module keeps provider/string/symbol compatibility in one place
+  # and exposes small helpers used by the agent loop, tool registry, transcript
+  # formatters, and RPC event normalizers.
   module ToolCall
     TOOL_NAME_MAP = {
       "read_file" => "read",
@@ -8,26 +16,37 @@ module Kward
       "write_file" => "write",
       "run_shell_command" => "bash",
       "list_directory" => "list_directory",
+      "code_search" => "code_search",
       "web_search" => "web_search",
+      "fetch_content" => "fetch_content",
+      "fetch_raw" => "fetch_raw",
       "read_skill" => "read_skill",
       "ask_user_question" => "ask_user_question"
     }.freeze
 
     module_function
 
+    # @return [String, nil] provider tool-call id
     def id(tool_call)
       value(tool_call, :id)
     end
 
+    # @return [String, nil] requested tool/function name
     def name(tool_call)
       value(function(tool_call), :name)
     end
 
+    # Returns the short name used in compact UI labels.
+    #
+    # @return [String] display label such as `read`, `edit`, or `bash`
     def display_name(tool_call)
       raw_name = name(tool_call)
       normalized_name(raw_name) || raw_name || "unknown_tool"
     end
 
+    # Parses the requested tool arguments.
+    #
+    # @return [Hash] decoded argument object, or an empty hash for invalid JSON
     def arguments(tool_call)
       parse_arguments(raw_arguments(tool_call))
     end
@@ -44,6 +63,10 @@ module Kward
       TOOL_NAME_MAP[name.to_s]
     end
 
+    # Converts provider argument payloads into hashes.
+    #
+    # Providers normally send JSON strings, while tests and compatibility callers
+    # may pass hashes directly.
     def parse_arguments(arguments)
       return {} if arguments.nil? || (arguments.respond_to?(:empty?) && arguments.empty?)
       return arguments if arguments.is_a?(Hash)
@@ -53,6 +76,9 @@ module Kward
       {}
     end
 
+    # Recursively converts snake_case hash keys to camelCase symbols.
+    #
+    # @return [Hash] camelized copy of `args`
     def camelize_args(args)
       return {} unless args.is_a?(Hash)
 
@@ -62,11 +88,7 @@ module Kward
     end
 
     def value(object, key)
-      return nil unless object.respond_to?(:key?)
-      return object[key] if object.key?(key)
-      return object[key.to_s] if object.key?(key.to_s)
-
-      nil
+      MessageAccess.value(object, key)
     end
 
     def camelize_value(item)

data/lib/kward/tools/web_search.rb

@@ -1,8 +1,13 @@
 require_relative "base"
+require_relative "search/web"
 
+# Namespace for the Kward CLI agent runtime.
 module Kward
+  # Model-callable tool wrappers and their argument schemas.
   module Tools
+    # Live web-search implementation with provider fallbacks.
     class WebSearch < Base
+      # Builds the tool schema and stores the execution dependency.
       def initialize(web_search:)
         @web_search = web_search
         super(
@@ -22,7 +27,7 @@ module Kward
             },
             provider: {
               type: "string",
-              enum: %w[auto exa perplexity gemini legacy duckduckgo],
+              enum: Kward::WebSearch::PROVIDERS,
               description: "Provider override; default auto."
             },
             recency_filter: {
@@ -40,6 +45,7 @@ module Kward
         )
       end
 
+      # Executes the tool and returns model-facing output text.
       def call(args, _conversation, cancellation: nil)
         @web_search.search(args)
       end

data/lib/kward/tools/write_file.rb

@@ -1,8 +1,12 @@
 require_relative "base"
 
+# Namespace for the Kward CLI agent runtime.
 module Kward
+  # Model-callable tool wrappers and their argument schemas.
   module Tools
+    # Tool wrapper for guarded full-file writes.
     class WriteFile < Base
+      # Builds the tool schema and stores the execution dependency.
       def initialize(workspace:)
         @workspace = workspace
         super(
@@ -16,6 +20,7 @@ module Kward
         )
       end
 
+      # Executes the tool and returns model-facing output text.
       def call(args, conversation, cancellation: nil)
         path = argument(args, :path, "")
         content = argument(args, :content, "")

data/lib/kward/transcript_export.rb

@@ -1,7 +1,9 @@
 require "cgi"
 require_relative "markdown_transcript"
 
+# Namespace for the Kward CLI agent runtime.
 module Kward
+  # Serializes conversations for transcript export formats.
   class TranscriptExport
     SUPPORTED_FORMATS = ["markdown", "html"].freeze
 

data/lib/kward/version.rb

@@ -1,4 +1,5 @@
+# Namespace for the Kward CLI agent runtime.
 module Kward
   # Current gem version.
-  VERSION = "0.67.1"
+  VERSION = "0.69.0"
 end

data/lib/kward/workspace.rb

@@ -3,7 +3,19 @@ require "pathname"
 require "timeout"
 require_relative "session_diff"
 
+# Namespace for the Kward CLI agent runtime.
 module Kward
+  # Filesystem and shell-command boundary for workspace tools.
+  #
+  # `Workspace` is deliberately low-level: it validates paths, enforces output
+  # limits, applies exact edits, writes files, and runs shell commands from one
+  # root directory. It should not know about model prompts, sessions, telemetry,
+  # or UI confirmation. Tool wrappers and frontends provide those policies.
+  #
+  # Guardrails are enabled by default and require all file paths to resolve under
+  # `root`. RPC may report when guardrails are disabled, but callers should avoid
+  # bypassing this class for local filesystem mutation so read-before-write and
+  # path safety remain consistent.
   class Workspace
     MAX_FILE_BYTES = 256 * 1024
     MAX_READ_OUTPUT_BYTES = 50 * 1024
@@ -12,6 +24,7 @@ module Kward
     MAX_EDIT_DIFF_BYTES = 8 * 1024
     DEFAULT_COMMAND_TIMEOUT_SECONDS = 30
 
+    # Creates an object for workspace filesystem and shell operations.
     def initialize(root: Dir.pwd, max_file_bytes: MAX_FILE_BYTES, max_read_output_bytes: MAX_READ_OUTPUT_BYTES, max_read_output_lines: MAX_READ_OUTPUT_LINES, max_command_output_bytes: MAX_COMMAND_OUTPUT_BYTES, guardrails: true)
       @root = Pathname.new(root).realpath
       @guardrails = guardrails
@@ -21,8 +34,10 @@ module Kward
       @max_command_output_bytes = max_command_output_bytes
     end
 
+    # @return [Pathname] canonical workspace root used as the base for file and shell tools
     attr_reader :root
 
+    # Lists immediate directory children after resolving `path` through workspace guardrails.
     def list_directory(path)
       resolved = workspace_path(path)
       return "Error: not a directory: #{path}" unless File.directory?(resolved)
@@ -34,6 +49,11 @@ module Kward
       "Error: #{e.message}"
     end
 
+    # Reads a bounded text slice from a workspace file.
+    #
+    # The returned string is user/model-facing and includes continuation notices
+    # when output is truncated. Errors are returned as `"Error: ..."` strings so
+    # tool calls can be persisted in the conversation without raising.
     def read_file(path, offset: nil, limit: nil)
       resolved = workspace_path(path)
       return "Error: not a file: #{path}" unless File.file?(resolved)
@@ -41,11 +61,20 @@ module Kward
       size = File.size(resolved)
       return "Error: file too large: #{path} is #{size} bytes; limit is #{@max_file_bytes} bytes" if size > @max_file_bytes
 
-      read_file_slice(File.read(resolved), offset: offset, limit: limit)
+      content = File.read(resolved)
+      return "Error: not a text file: #{path}" if binary_content?(content)
+
+      read_file_slice(content, offset: offset, limit: limit)
     rescue SecurityError, Errno::ENOENT => e
       "Error: #{e.message}"
     end
 
+    # Writes complete file content after enforcing read-before-write for
+    # existing files.
+    #
+    # `read_paths` must contain resolved paths previously observed by
+    # `ReadFile`; this keeps tool-driven edits explicit and prevents overwriting
+    # unseen user files.
     def write_file(path, content, read_paths:)
       resolved = workspace_write_path(path)
 
@@ -53,10 +82,6 @@ module Kward
         return "Error: existing file must be read before writing: #{path}"
       end
 
-      if block_given? && !yield(relative_path(resolved), content.bytesize)
-        return "Declined: write_file was not approved for #{path}"
-      end
-
       old_content = File.exist?(resolved) ? File.read(resolved) : nil
       File.write(resolved, content)
       output = "Wrote #{content.bytesize} bytes to #{path}"
@@ -66,6 +91,11 @@ module Kward
       "Error: #{e.message}"
     end
 
+    # Applies exact non-overlapping replacements to a previously read file.
+    #
+    # Each `old_text` must match exactly once. This favors predictable model edits
+    # over fuzzy patching and returns readable error strings when more context is
+    # needed.
     def edit_file(path, edits, read_paths:)
       resolved = workspace_path(path)
       return "Error: not a file: #{path}" unless File.file?(resolved)
@@ -84,6 +114,11 @@ module Kward
       "Error: #{e.message}"
     end
 
+    # Runs a shell command from the workspace root with timeout, cancellation,
+    # and bounded combined output.
+    #
+    # This method intentionally does not ask for confirmation; CLI/RPC policy
+    # must decide whether a command is allowed before reaching this boundary.
     def run_shell_command(command, timeout_seconds: DEFAULT_COMMAND_TIMEOUT_SECONDS, cancellation: nil)
       command = command.to_s.strip
       return "Error: command is required" if command.empty?
@@ -114,6 +149,7 @@ module Kward
       "Error: #{e.message}"
     end
 
+    # Resolves a path with the same guardrails used by file tools.
     def resolved_path(path)
       workspace_path(path)
     end
@@ -189,6 +225,10 @@ module Kward
       output
     end
 
+    def binary_content?(content)
+      content.include?("\x00")
+    end
+
     def read_start_index(offset)
       return 0 if offset.nil?