kward 0.69.1 → 0.71.0

data/lib/kward/tool_output_compactor.rb

@@ -0,0 +1,127 @@
+# Namespace for the Kward CLI agent runtime.
+module Kward
+  # Deterministically trims large tool outputs before they are appended to the
+  # model-facing transcript.
+  #
+  # The original output is still handed to session/tool-execution persistence by
+  # ToolRegistry; this object only decides what the next model call sees. Keep it
+  # conservative: small outputs and short errors are more valuable verbatim than
+  # compacted.
+  class ToolOutputCompactor
+    MIN_BYTES = 12 * 1024
+    ERROR_OUTPUT_MAX_BYTES = 8 * 1024
+    HEAD_LINES = 40
+    TAIL_LINES = 40
+    ERROR_CONTEXT_LINES = 2
+
+    ERROR_PATTERN = /\b(error|fatal|failed|failure|exception|traceback|panic|segmentation fault|assertion)\b/i.freeze
+    TEST_PATTERN = /(^\s*\d+\)\s|\b(\d+\s+(tests?|examples?|runs?|assertions?|failures?|errors?|skips?)|finished in|failures?:|seed\s+\d+)\b)/i.freeze
+    SEARCH_PATTERN = /(^\#{1,6}\s+\S+|^[-*]\s+\S+|\S+:\d+:|https?:\/\/\S+)/.freeze
+
+    def compact(tool_name, content, artifact_id: nil)
+      text = normalize(content)
+      return text unless text.bytesize > MIN_BYTES
+      return text if error_output?(text) && text.bytesize <= ERROR_OUTPUT_MAX_BYTES
+
+      compacted = tool_name.to_s == "run_shell_command" ? compact_shell_output(text) : compact_lines(text)
+      return text if compacted == text
+      return text if compacted.bytesize >= text.bytesize
+
+      artifact_id = yield if artifact_id.nil? && block_given?
+      header = compacted_header(tool_name, text, compacted, artifact_id: artifact_id)
+      candidate = "#{header}\n\n#{compacted}"
+      candidate.bytesize < text.bytesize ? candidate : text
+    end
+
+    private
+
+    def normalize(content)
+      return content unless content.is_a?(String)
+
+      Conversation.normalize_tool_content(content)
+    end
+
+    def error_output?(text)
+      text.match?(ERROR_PATTERN)
+    end
+
+    def compact_shell_output(text)
+      sections = shell_sections(text)
+      return compact_lines(text) if sections.empty?
+
+      sections.map do |heading, body|
+        next heading if body.empty?
+
+        "#{heading}\n#{compact_lines(body)}"
+      end.join("\n")
+    end
+
+    def shell_sections(text)
+      parts = text.split(/\n(?=STDOUT:\n|STDERR:\n)/)
+      return [] if parts.length < 2
+
+      parts.map do |part|
+        heading, body = part.split("\n", 2)
+        [heading, body.to_s]
+      end
+    end
+
+    def compact_lines(text)
+      lines = text.split("\n", -1)
+      selected = selected_line_indexes(lines)
+      return text if selected.length >= lines.length
+
+      render_selected_lines(lines, selected)
+    end
+
+    def selected_line_indexes(lines)
+      indexes = []
+      indexes.concat((0...[HEAD_LINES, lines.length].min).to_a)
+      indexes.concat(priority_context_indexes(lines))
+
+      tail_start = [lines.length - TAIL_LINES, 0].max
+      indexes.concat((tail_start...lines.length).to_a)
+      indexes.uniq.sort
+    end
+
+    def priority_context_indexes(lines)
+      indexes = []
+      lines.each_with_index do |line, index|
+        next unless line.match?(ERROR_PATTERN) || line.match?(TEST_PATTERN) || line.match?(SEARCH_PATTERN)
+
+        first = [index - ERROR_CONTEXT_LINES, 0].max
+        last = [index + ERROR_CONTEXT_LINES, lines.length - 1].min
+        indexes.concat((first..last).to_a)
+      end
+      indexes
+    end
+
+    def render_selected_lines(lines, selected)
+      output = []
+      previous = nil
+      selected.each do |index|
+        if previous && index > previous + 1
+          output << "[... omitted lines #{previous + 2}-#{index} ...]"
+        end
+        output << lines[index]
+        previous = index
+      end
+      output.join("\n")
+    end
+
+    def compacted_header(tool_name, original, compacted, artifact_id: nil)
+      [
+        "[Tool output compacted by Kward: #{original.bytesize} bytes -> #{compacted.bytesize} bytes]",
+        "Tool: #{tool_name}",
+        "Preserved first #{HEAD_LINES} lines, last #{TAIL_LINES} lines, and error/failure/search context.",
+        retrieval_instruction(artifact_id)
+      ].join("\n")
+    end
+
+    def retrieval_instruction(artifact_id)
+      return "Full output is retained outside model context." if artifact_id.to_s.empty?
+
+      "Full output id: #{artifact_id}. Use retrieve_tool_output to inspect it."
+    end
+  end
+end

data/lib/kward/tools/base.rb

@@ -24,8 +24,8 @@ module Kward
             description: @description,
             parameters: {
               type: "object",
-              properties: @properties,
-              required: @required,
+              properties: sorted_properties,
+              required: @required.sort,
               additionalProperties: false
             }
           }
@@ -34,6 +34,12 @@ module Kward
 
       private
 
+      def sorted_properties
+        @properties.keys.sort_by(&:to_s).each_with_object({}) do |key, result|
+          result[key] = @properties[key]
+        end
+      end
+
       # Reads a tool argument while accepting symbol or string keys from restored calls.
       def argument(args, key, default = nil)
         return args[key] if args.key?(key)

data/lib/kward/tools/registry.rb

@@ -1,4 +1,5 @@
 require_relative "../config_files"
+require_relative "../conversation"
 require_relative "ask_user_question"
 require_relative "code_search"
 require_relative "edit_file"
@@ -8,12 +9,16 @@ require_relative "list_directory"
 require_relative "read_file"
 require_relative "read_skill"
 require_relative "run_shell_command"
+require_relative "summarize_file_structure"
+require_relative "retrieve_tool_output"
 require_relative "web_search"
 require_relative "write_file"
 require_relative "search/code"
 require_relative "search/web"
 require_relative "search/web_fetch"
 require_relative "tool_call"
+require_relative "../telemetry/logger"
+require_relative "../tool_output_compactor"
 require_relative "../workspace"
 
 # Namespace for the Kward CLI agent runtime.
@@ -53,7 +58,7 @@ module Kward
     # @param web_search_enabled [Boolean, nil] override for web search exposure
     # @param skills [Array<ConfigFiles::Skill>, nil] override discovered skills
     # @param ask_user_question_enabled [Boolean, nil] override question exposure
-    def initialize(workspace: Workspace.new, prompt: nil, web_search: WebSearch.new, web_fetch: WebFetch.new, code_search: CodeSearch.new, web_search_enabled: nil, skills: nil, ask_user_question_enabled: nil)
+    def initialize(workspace: Workspace.new, prompt: nil, web_search: WebSearch.new, web_fetch: WebFetch.new, code_search: CodeSearch.new, web_search_enabled: nil, skills: nil, ask_user_question_enabled: nil, tool_output_compactor: ToolOutputCompactor.new, telemetry_logger: TelemetryLogger.new)
       @workspace = workspace
       @prompt = prompt
       @web_search = web_search
@@ -62,6 +67,8 @@ module Kward
       @skills = skills
       @web_search_enabled = web_search_enabled
       @ask_user_question_enabled = ask_user_question_enabled
+      @tool_output_compactor = tool_output_compactor
+      @telemetry_logger = telemetry_logger
       @tools = build_tools.freeze
       @schemas = build_schema_tools.map(&:schema).freeze
     end
@@ -87,26 +94,48 @@ module Kward
                 else
                   "Unknown tool: #{name}"
                 end
-
+      content = Conversation.normalize_tool_content(content)
+      duplicate_id = conversation.tool_output_artifact_id_for(tool_name: name, content: content)
+      if conversation.tool_output_artifacts.key?(duplicate_id)
+        content = "[Same as previous tool output #{duplicate_id}; not repeated. Use retrieve_tool_output to inspect it.]"
+      end
+
+      artifact_id = nil
+      model_content = @tool_output_compactor.compact(name, content) do
+        artifact_id ||= conversation.store_tool_output_artifact(tool_name: name, content: content)
+      end
+      log_tool_output_compaction(name, artifact_id: artifact_id, before: content, after: model_content) if model_content != content
       conversation.append_tool(
         tool_call_id: tool_call["id"] || tool_call[:id],
         name: name,
-        content: content
+        content: model_content
       )
       conversation.append_tool_execution(tool_call: tool_call, content: content)
 
-      content
+      model_content
     end
 
     private
 
+    def log_tool_output_compaction(name, artifact_id:, before:, after:)
+      @telemetry_logger.log(
+        "compaction",
+        "tool_output",
+        "tool_name" => name,
+        "artifact_id" => artifact_id,
+        "bytes_before" => before.bytesize,
+        "bytes_after" => after.bytesize,
+        "bytes_saved" => before.bytesize - after.bytesize
+      )
+    end
+
     def build_tools
       all_tools.to_h { |tool| [tool.name, tool] }
     end
 
     def build_schema_tools
       tools = @tools.values_at(
-        "list_directory", "read_file", "write_file", "edit_file", "run_shell_command", "code_search"
+        "list_directory", "read_file", "write_file", "edit_file", "run_shell_command", "code_search", "summarize_file_structure", "retrieve_tool_output"
       )
       tools.concat(@tools.values_at("web_search", "fetch_content", "fetch_raw")) if web_search_available?
       tools << @tools["read_skill"] if skills_available?
@@ -131,7 +160,9 @@ module Kward
         Tools::WriteFile.new(workspace: @workspace),
         Tools::EditFile.new(workspace: @workspace),
         Tools::RunShellCommand.new(workspace: @workspace),
-        Tools::CodeSearch.new(code_search: @code_search)
+        Tools::CodeSearch.new(code_search: @code_search),
+        Tools::SummarizeFileStructure.new(workspace: @workspace),
+        Tools::RetrieveToolOutput.new
       ]
     end
 

data/lib/kward/tools/retrieve_tool_output.rb

@@ -0,0 +1,71 @@
+require_relative "base"
+
+# Namespace for the Kward CLI agent runtime.
+module Kward
+  # Model-callable tool wrappers and their argument schemas.
+  module Tools
+    # Retrieves original tool outputs that were compacted out of model context.
+    class RetrieveToolOutput < Base
+      DEFAULT_LIMIT = 120
+
+      # Builds the retrieval tool schema.
+      def initialize
+        super(
+          "retrieve_tool_output",
+          "Retrieve original output from a compacted previous tool result.",
+          properties: {
+            id: { type: "string", description: "Tool output id, for example toolout_abc123." },
+            query: { type: "string", description: "Optional case-insensitive text search within the original output." },
+            offset: { type: "integer", description: "1-indexed line offset for returned output." },
+            limit: { type: "integer", description: "Maximum lines to return; default 120." }
+          },
+          required: ["id"]
+        )
+      end
+
+      # Executes retrieval from the active conversation artifact store.
+      def call(args, conversation, cancellation: nil)
+        cancellation&.raise_if_cancelled!
+        id = argument(args, :id, "").to_s
+        return "Error: id is required" if id.empty?
+
+        artifact = conversation.tool_output_artifacts[id]
+        return "Error: unknown tool output id: #{id}" unless artifact
+
+        content = artifact[:content].to_s
+        query = argument(args, :query, "").to_s
+        lines = query.empty? ? content.split("\n", -1) : matching_lines(content, query)
+        return "No matching lines for #{query.inspect} in #{id}." if lines.empty?
+
+        slice_lines(id, lines, offset: argument(args, :offset), limit: argument(args, :limit), query: query)
+      end
+
+      private
+
+      def matching_lines(content, query)
+        needle = query.downcase
+        content.split("\n", -1).each_with_index.filter_map do |line, index|
+          next unless line.downcase.include?(needle)
+
+          "#{index + 1}: #{line}"
+        end
+      end
+
+      def slice_lines(id, lines, offset:, limit:, query:)
+        start_index = [offset.to_i - 1, 0].max
+        return "Error: offset #{offset} is beyond output (#{lines.length} lines total)" if start_index >= lines.length
+
+        line_limit = limit.to_i.positive? ? limit.to_i : DEFAULT_LIMIT
+        selected = lines[start_index, line_limit] || []
+        header = "[Retrieved tool output #{id}"
+        header << " matching #{query.inspect}" unless query.empty?
+        header << ": lines #{start_index + 1}-#{start_index + selected.length} of #{lines.length}]"
+        output = "#{header}\n#{selected.join("\n")}".rstrip
+        if start_index + selected.length < lines.length
+          output << "\n\n[#{lines.length - start_index - selected.length} more lines. Use offset=#{start_index + selected.length + 1} to continue.]"
+        end
+        output
+      end
+    end
+  end
+end

data/lib/kward/tools/search/web.rb

@@ -619,8 +619,8 @@ module Kward
       return web_config[snake] if web_config.key?(snake)
       return web_config[camel] if web_config.key?(camel)
       return config[prefixed] if config.key?(prefixed)
-      return config[snake] if config.key?(snake)
-      return config[camel] if config.key?(camel)
+      return config[snake] if key.to_s != "provider" && config.key?(snake)
+      return config[camel] if key.to_s != "provider" && config.key?(camel)
 
       nil
     end

data/lib/kward/tools/summarize_file_structure.rb

@@ -0,0 +1,29 @@
+require_relative "base"
+
+# Namespace for the Kward CLI agent runtime.
+module Kward
+  # Model-callable tool wrappers and their argument schemas.
+  module Tools
+    # Returns a compact symbol outline for a workspace source file.
+    class SummarizeFileStructure < Base
+      # Builds the tool schema and stores the execution dependency.
+      def initialize(workspace:)
+        @workspace = workspace
+        super(
+          "summarize_file_structure",
+          "Return a compact outline of classes, modules, methods, and functions in a workspace source file.",
+          properties: {
+            path: { type: "string", description: "Workspace-relative source file path." }
+          },
+          required: ["path"]
+        )
+      end
+
+      # Executes the structure summary tool.
+      def call(args, _conversation, cancellation: nil)
+        cancellation&.raise_if_cancelled!
+        @workspace.summarize_file_structure(argument(args, :path, ""))
+      end
+    end
+  end
+end

data/lib/kward/tools/tool_call.rb

@@ -17,6 +17,8 @@ module Kward
       "run_shell_command" => "bash",
       "list_directory" => "list_directory",
       "code_search" => "code_search",
+      "summarize_file_structure" => "summarize_file_structure",
+      "retrieve_tool_output" => "retrieve_tool_output",
       "web_search" => "web_search",
       "fetch_content" => "fetch_content",
       "fetch_raw" => "fetch_raw",

data/lib/kward/version.rb

@@ -1,5 +1,5 @@
 # Namespace for the Kward CLI agent runtime.
 module Kward
   # Current gem version.
-  VERSION = "0.69.1"
+  VERSION = "0.71.0"
 end

data/lib/kward/workspace.rb

@@ -20,7 +20,7 @@ module Kward
     MAX_FILE_BYTES = 256 * 1024
     MAX_READ_OUTPUT_BYTES = 50 * 1024
     MAX_READ_OUTPUT_LINES = 2_000
-    MAX_COMMAND_OUTPUT_BYTES = 20 * 1024
+    MAX_COMMAND_OUTPUT_BYTES = 128 * 1024
     MAX_EDIT_DIFF_BYTES = 8 * 1024
     DEFAULT_COMMAND_TIMEOUT_SECONDS = 30
 
@@ -64,7 +64,27 @@ module Kward
       content = File.read(resolved)
       return "Error: not a text file: #{path}" if binary_content?(content)
 
-      read_file_slice(content, offset: offset, limit: limit)
+      large_file_outline_response(path, content, offset: offset, limit: limit) || read_file_slice(content, offset: offset, limit: limit)
+    rescue SecurityError, Errno::ENOENT => e
+      "Error: #{e.message}"
+    end
+
+    # Returns a compact outline of recognizable source-code declarations.
+    def summarize_file_structure(path)
+      resolved = workspace_path(path)
+      return "Error: not a file: #{path}" unless File.file?(resolved)
+
+      size = File.size(resolved)
+      return "Error: file too large: #{path} is #{size} bytes; limit is #{@max_file_bytes} bytes" if size > @max_file_bytes
+
+      content = File.read(resolved)
+      return "Error: not a text file: #{path}" if binary_content?(content)
+
+      lines = content.split("\n", -1)
+      outline = source_outline(lines)
+      return "No recognizable source structure found in #{path}." if outline.empty?
+
+      (["# File structure: #{path}", "- Lines: #{lines.length}", "- Bytes: #{content.bytesize}", "", "## Outline"] + outline).join("\n")
     rescue SecurityError, Errno::ENOENT => e
       "Error: #{e.message}"
     end
@@ -196,6 +216,42 @@ module Kward
       Pathname.new(path).relative_path_from(@root).to_s
     end
 
+    def large_file_outline_response(path, content, offset:, limit:)
+      return nil unless offset.nil? && limit.nil?
+      lines = content.split("\n", -1)
+      return nil unless lines.length > @max_read_output_lines || content.bytesize > @max_read_output_bytes
+
+      outline = source_outline(lines)
+      return nil if outline.empty?
+
+      preview_limit = [120, @max_read_output_lines].min
+      preview = lines.first(preview_limit).join("\n")
+      [
+        "File has #{lines.length} lines (#{content.bytesize} bytes). Showing an outline and the first #{preview_limit} lines to reduce model context.",
+        "",
+        "Outline:",
+        outline.join("\n"),
+        "",
+        "First #{preview_limit} lines:",
+        preview,
+        "",
+        "[Use read_file with offset=#{preview_limit + 1} and limit to continue, or request a specific section from the outline.]"
+      ].join("\n")
+    end
+
+    def source_outline(lines)
+      outline = []
+      lines.each_with_index do |line, index|
+        stripped = line.strip
+        next unless stripped.match?(/\A(class|module|def)\s+/) || stripped.match?(/\A(function|async function)\s+/) || stripped.match?(/\A(export\s+)?(class|interface|type)\s+/)
+
+        indent = line[/\A\s*/].to_s.length
+        outline << "line #{index + 1}: #{'  ' * [indent / 2, 6].min}#{stripped}"
+        break if outline.length >= 80
+      end
+      outline
+    end
+
     def read_file_slice(content, offset:, limit:)
       lines = content.split("\n", -1)
       lines = [""] if lines.empty?