openclacky 0.6.1 → 0.6.3

data/lib/clacky/tools/file_reader.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "base"
+require_relative "../utils/file_processor"
 
 module Clacky
   module Tools
@@ -17,14 +18,28 @@ module Clacky
           },
           max_lines: {
             type: "integer",
-            description: "Maximum number of lines to read (optional)",
-            default: 1000
+            description: "Maximum number of lines to read from start (default: 500)",
+            default: 500
+          },
+          keyword: {
+            type: "string",
+            description: "Search keyword and return matching lines with context (recommended for large files)"
+          },
+          start_line: {
+            type: "integer",
+            description: "Start line number (1-indexed, e.g., 100 reads from line 100)"
+          },
+          end_line: {
+            type: "integer",
+            description: "End line number (1-indexed, e.g., 200 reads up to line 200)"
           }
         },
         required: ["path"]
       }
+      
 
-      def execute(path:, max_lines: 1000)
+
+      def execute(path:, max_lines: 500, keyword: nil, start_line: nil, end_line: nil)
         # Expand ~ to home directory
         expanded_path = File.expand_path(path)
         
@@ -50,15 +65,67 @@ module Clacky
         end
 
         begin
-          lines = File.readlines(expanded_path).first(max_lines)
-          content = lines.join
-          truncated = File.readlines(expanded_path).size > max_lines
+          # Check if file is binary
+          if binary_file?(expanded_path)
+            return handle_binary_file(expanded_path)
+          end
+
+          # Handle keyword search with context
+          if keyword && !keyword.empty?
+            return find_with_context(expanded_path, keyword)
+          end
+
+          # Read text file with optional line range
+          all_lines = File.readlines(expanded_path)
+          total_lines = all_lines.size
+
+          # Calculate start index (convert 1-indexed to 0-indexed)
+          start_idx = start_line ? [start_line - 1, 0].max : 0
+
+          # Calculate end index based on parameters
+          if end_line
+            # User specified end_line directly
+            end_idx = [end_line - 1, total_lines - 1].min
+          elsif start_line
+            # start_line + max_lines - 1 (relative to start_line, inclusive)
+            calculated_end_line = start_line + max_lines - 1
+            end_idx = [calculated_end_line - 1, total_lines - 1].min
+          else
+            # Read from beginning with max_lines limit
+            end_idx = [max_lines - 1, total_lines - 1].min
+          end
+
+          # Check if start_line exceeds file length first
+          if start_idx >= total_lines
+            return {
+              path: expanded_path,
+              content: nil,
+              lines_read: 0,
+              error: "Invalid line range: start_line #{start_line} exceeds total lines (#{total_lines})"
+            }
+          end
+
+          # Validate range
+          if start_idx > end_idx
+            return {
+              path: expanded_path,
+              content: nil,
+              lines_read: 0,
+              error: "Invalid line range: start_line #{start_line} > end_line #{end_line || (start_line + max_lines)}"
+            }
+          end
+
+          lines = all_lines[start_idx..end_idx] || []
+          truncated = total_lines > max_lines && !keyword
 
           {
             path: expanded_path,
-            content: content,
+            content: lines.join,
             lines_read: lines.size,
+            total_lines: total_lines,
             truncated: truncated,
+            start_line: start_line,
+            end_line: end_line,
             error: nil
           }
         rescue StandardError => e
@@ -86,16 +153,185 @@ module Clacky
           return "Listed #{entries} entries (#{dirs} directories, #{files} files)"
         end
 
-        # Handle file reading
+        # Handle binary file
+        if result[:binary] || result['binary']
+          format_type = result[:format] || result['format'] || 'unknown'
+          size = result[:size_bytes] || result['size_bytes'] || 0
+          
+          # Check if it has base64 data (LLM-compatible format)
+          if result[:base64_data] || result['base64_data']
+            size_warning = size > 5_000_000 ? " (WARNING: large file)" : ""
+            return "Binary file (#{format_type}, #{format_file_size(size)}) - sent to LLM#{size_warning}"
+          else
+            return "Binary file (#{format_type}, #{format_file_size(size)}) - cannot be read as text"
+          end
+        end
+
+        # Handle text file reading
         lines = result[:lines_read] || result['lines_read'] || 0
         truncated = result[:truncated] || result['truncated']
         "Read #{lines} lines#{truncated ? ' (truncated)' : ''}"
       end
+      
+      # Format result for LLM - handles both text and binary (image/PDF) content
+      # This method is called by the agent to format tool results before sending to LLM
+      def format_result_for_llm(result)
+        # For LLM-compatible binary files with base64 data, return as content blocks
+        if result[:binary] && result[:base64_data]
+          # Create a text description
+          description = "File: #{result[:path]}\nType: #{result[:format]}\nSize: #{format_file_size(result[:size_bytes])}"
+          
+          # Add size warning for large files
+          if result[:size_bytes] > 5_000_000
+            description += "\nWARNING: Large file (>5MB) - may consume significant tokens"
+          end
+          
+          # For images, return both description and image content
+          if result[:mime_type]&.start_with?("image/")
+            return {
+              type: "image",
+              path: result[:path],
+              format: result[:format],
+              size_bytes: result[:size_bytes],
+              mime_type: result[:mime_type],
+              base64_data: result[:base64_data],
+              description: description
+            }
+          end
+          
+          # For PDFs and other binary formats, just return metadata with base64
+          return {
+            type: "document",
+            path: result[:path],
+            format: result[:format],
+            size_bytes: result[:size_bytes],
+            mime_type: result[:mime_type],
+            base64_data: result[:base64_data],
+            description: description
+          }
+        end
+        
+        # For other cases, return the result as-is (agent will JSON.generate it)
+        result
+      end
+
+      # Find lines matching keyword with context (5 lines before and after each match)
+      private def find_with_context(path, keyword)
+        context_lines_count = 5
+        all_lines = File.readlines(path)
+        total_lines = all_lines.size
+        matches = []
+
+        # Find all matching line indices (case-insensitive)
+        all_lines.each_with_index do |line, index|
+          if line.include?(keyword)
+            start_idx = [index - context_lines_count, 0].max
+            end_idx = [index + context_lines_count, total_lines - 1].min
+            matches << {
+              line_number: index + 1,
+              content: all_lines[start_idx..end_idx].join,
+              start_line: start_idx + 1,
+              end_line: end_idx + 1,
+              match_line: index + 1
+            }
+          end
+        end
+
+        if matches.empty?
+          {
+            path: path,
+            content: nil,
+            matches_count: 0,
+            error: "Keyword '#{keyword}' not found in file"
+          }
+        else
+          # Combine all matches with separator
+          combined_content = matches.map do |m|
+            "... Lines #{m[:start_line]}-#{m[:end_line]} (match at line #{m[:line_number]}):\n#{m[:content]}"
+          end.join("\n---\n")
+
+          {
+            path: path,
+            content: combined_content,
+            matches_count: matches.size,
+            keyword: keyword,
+            error: nil
+          }
+        end
+      end
+
+      private def binary_file?(path)
+        # Use FileProcessor to detect binary files
+        File.open(path, 'rb') do |file|
+          sample = file.read(8192) || ""
+          Utils::FileProcessor.binary_file?(sample)
+        end
+      rescue StandardError
+        # If we can't read the file, assume it's not binary
+        false
+      end
+      
+      private def handle_binary_file(path)
+        # Check if it's a supported format using FileProcessor
+        if Utils::FileProcessor.supported_binary_file?(path)
+          # Use FileProcessor to convert to base64
+          begin
+            result = Utils::FileProcessor.file_to_base64(path)
+            {
+              path: path,
+              binary: true,
+              format: result[:format],
+              mime_type: result[:mime_type],
+              size_bytes: result[:size_bytes],
+              base64_data: result[:base64_data],
+              error: nil
+            }
+          rescue ArgumentError => e
+            # File too large or other error
+            file_size = File.size(path)
+            ext = File.extname(path).downcase
+            {
+              path: path,
+              binary: true,
+              format: ext.empty? ? "unknown" : ext[1..-1],
+              size_bytes: file_size,
+              content: nil,
+              error: e.message
+            }
+          end
+        else
+          # Binary file that we can't send to LLM
+          file_size = File.size(path)
+          ext = File.extname(path).downcase
+          {
+            path: path,
+            binary: true,
+            format: ext.empty? ? "unknown" : ext[1..-1],
+            size_bytes: file_size,
+            content: nil,
+            error: "Binary file detected. This format cannot be read as text. File size: #{format_file_size(file_size)}"
+          }
+        end
+      end
+      
+      private def detect_mime_type(path, data)
+        Utils::FileProcessor.detect_mime_type(path, data)
+      end
+      
+      private def format_file_size(bytes)
+        if bytes < 1024
+          "#{bytes} bytes"
+        elsif bytes < 1024 * 1024
+          "#{(bytes / 1024.0).round(2)} KB"
+        else
+          "#{(bytes / (1024.0 * 1024)).round(2)} MB"
+        end
+      end
 
       private
 
       # List first-level directory contents (files and directories)
-      def list_directory_contents(path)
+      private def list_directory_contents(path)
         begin
           entries = Dir.entries(path).reject { |entry| entry == "." || entry == ".." }
           

data/lib/clacky/tools/grep.rb

@@ -59,11 +59,6 @@ module Clacky
             description: "Maximum file size in bytes to search (default: 1MB)",
             default: MAX_FILE_SIZE
           },
-          max_files_to_search: {
-            type: "integer",
-            description: "Maximum number of files to search",
-            default: 500
-          }
         },
         required: %w[pattern]
       }
@@ -78,7 +73,7 @@ module Clacky
         max_matches_per_file: 50,
         max_total_matches: 200,
         max_file_size: MAX_FILE_SIZE,
-        max_files_to_search: 500
+        max_files_to_search: 10000
       )
         # Validate pattern
         if pattern.nil? || pattern.strip.empty?
@@ -135,7 +130,7 @@ module Clacky
             end
 
             # Skip if file should be ignored (unless it's a config file)
-            if Clacky::Utils::FileIgnoreHelper.should_ignore_file?(file, expanded_path, gitignore) && 
+            if Clacky::Utils::FileIgnoreHelper.should_ignore_file?(file, expanded_path, gitignore) &&
                !Clacky::Utils::FileIgnoreHelper.is_config_file?(file)
               skipped[:ignored] += 1
               next
@@ -217,12 +212,12 @@ module Clacky
           matches = result[:total_matches] || 0
           files = result[:files_with_matches] || 0
           msg = "[OK] Found #{matches} matches in #{files} files"
-          
+
           # Add truncation info if present
           if result[:truncated] && result[:truncation_reason]
             msg += " (truncated: #{result[:truncation_reason]})"
           end
-          
+
           msg
         end
       end
@@ -275,12 +270,12 @@ module Clacky
 
       def search_file(file, regex, context_lines, max_matches)
         matches = []
-        
+
         # Use File.foreach for memory-efficient line-by-line reading
         File.foreach(file, chomp: true).with_index do |line, index|
           # Stop if we have enough matches for this file
           break if matches.length >= max_matches
-          
+
           next unless line.match?(regex)
 
           # Truncate long lines
@@ -315,10 +310,10 @@ module Clacky
         (start_line..end_line).each do |i|
           line_content = lines[i]
           # Truncate long lines in context too
-          display_content = line_content.length > MAX_LINE_LENGTH ? 
-                          "#{line_content[0...MAX_LINE_LENGTH]}..." : 
+          display_content = line_content.length > MAX_LINE_LENGTH ?
+                          "#{line_content[0...MAX_LINE_LENGTH]}..." :
                           line_content
-          
+
           context << {
             line_number: i + 1,
             content: display_content,

data/lib/clacky/tools/safe_shell.rb

@@ -32,20 +32,26 @@ module Clacky
         required: ["command"]
       }
 
-      def execute(command:, timeout: nil, max_output_lines: 1000)
+      def execute(command:, timeout: nil, max_output_lines: 1000, skip_safety_check: false)
         # Get project root directory
         project_root = Dir.pwd
 
         begin
           # 1. Extract timeout from command if it starts with "timeout N"
           command, extracted_timeout = extract_timeout_from_command(command)
-          
+
           # Use extracted timeout if not explicitly provided
           timeout ||= extracted_timeout
 
-          # 2. Use safety replacer to process command
-          safety_replacer = CommandSafetyReplacer.new(project_root)
-          safe_command = safety_replacer.make_command_safe(command)
+          # 2. Use safety replacer to process command (skip if user already confirmed)
+          if skip_safety_check
+            # User has confirmed, execute command as-is (no safety modifications)
+            safe_command = command
+            safety_replacer = nil
+          else
+            safety_replacer = CommandSafetyReplacer.new(project_root)
+            safe_command = safety_replacer.make_command_safe(command)
+          end
 
           # 3. Calculate timeouts: soft_timeout is fixed at 5s, hard_timeout from timeout parameter
           soft_timeout = 5
@@ -55,7 +61,7 @@ module Clacky
           result = super(command: safe_command, soft_timeout: soft_timeout, hard_timeout: hard_timeout, max_output_lines: max_output_lines)
 
           # 5. Enhance result information
-          enhance_result(result, command, safe_command)
+          enhance_result(result, command, safe_command, safety_replacer)
 
         rescue SecurityError => e
           # Security error, return friendly error message
@@ -145,9 +151,9 @@ module Clacky
         end
       end
 
-      def enhance_result(result, original_command, safe_command)
+      def enhance_result(result, original_command, safe_command, safety_replacer = nil)
         # If command was replaced, add security information
-        if original_command != safe_command
+        if safety_replacer && original_command != safe_command
           result[:security_enhanced] = true
           result[:original_command] = original_command
           result[:safe_command] = safe_command
@@ -190,6 +196,24 @@ module Clacky
         end
       end
 
+      # Override format_result_for_llm to preserve security fields
+      def format_result_for_llm(result)
+        # If security blocked, return as-is (small and important)
+        return result if result[:security_blocked]
+        
+        # Call parent's format_result_for_llm to truncate output
+        compact = super(result)
+        
+        # Add security enhancement fields if present (they're small and important for LLM to understand)
+        if result[:security_enhanced]
+          compact[:security_enhanced] = true
+          compact[:original_command] = result[:original_command]
+          compact[:safe_command] = result[:safe_command]
+        end
+        
+        compact
+      end
+
       private def format_non_zero_exit(exit_code, stdout, stderr)
         stdout_lines = stdout.lines.size
         has_output = stdout_lines > 0
@@ -331,7 +355,12 @@ module Clacky
 
       def replace_chmod_command(command)
         # Parse chmod command to ensure it's safe
-        parts = Shellwords.split(command)
+        begin
+          parts = Shellwords.split(command)
+        rescue Shellwords::BadQuotedString => e
+          # If Shellwords.split fails, use simple split as fallback
+          parts = command.split(/\s+/)
+        end
 
         # Only allow chmod +x on files in project directory
         files = parts[2..-1] || []
@@ -340,8 +369,6 @@ module Clacky
         # Allow chmod +x as it's generally safe
         log_replacement("chmod", command, "chmod +x is allowed - file permissions will be modified")
         command
-      rescue Shellwords::BadQuotedString
-        raise SecurityError, "Invalid chmod command syntax: #{command}"
       end
 
       def replace_curl_pipe_command(command)
@@ -370,7 +397,14 @@ module Clacky
 
       def validate_and_allow(command)
         # Check basic file operation commands
-        parts = Shellwords.split(command)
+        begin
+          parts = Shellwords.split(command)
+        rescue Shellwords::BadQuotedString => e
+          # If Shellwords.split fails due to quote issues, try simple split as fallback
+          # This handles cases where paths don't actually need shell escaping
+          parts = command.split(/\s+/)
+        end
+        
         cmd = parts.first
         args = parts[1..-1]
 
@@ -384,8 +418,6 @@ module Clacky
         end
 
         command
-      rescue Shellwords::BadQuotedString
-        raise SecurityError, "Invalid command syntax: #{command}"
       end
 
       def validate_general_command(command)
@@ -419,11 +451,15 @@ module Clacky
       end
 
       def parse_rm_files(command)
-        parts = Shellwords.split(command)
+        begin
+          parts = Shellwords.split(command)
+        rescue Shellwords::BadQuotedString => e
+          # If Shellwords.split fails, use simple split as fallback
+          parts = command.split(/\s+/)
+        end
+        
         # Skip rm command itself and option parameters
         parts.drop(1).reject { |part| part.start_with?('-') }
-      rescue Shellwords::BadQuotedString
-        raise SecurityError, "Invalid command syntax: #{command}"
       end
 
       def validate_file_path(path)

data/lib/clacky/tools/shell.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "tmpdir"
 require_relative "base"
 
 module Clacky
@@ -35,7 +36,7 @@ module Clacky
       INTERACTION_PATTERNS = [
         [/\[Y\/n\]|\[y\/N\]|\(yes\/no\)|\(Y\/n\)|\(y\/N\)/i, 'confirmation'],
         [/[Pp]assword\s*:\s*$|Enter password|enter password/, 'password'],
-        [/^\s*>>>\s*$|^\s*>>?\s*$|^irb\(.*\):\d+:\d+[>*]\s*$|^>\s*$/, 'repl'],
+        [/^\s*>>>\s*$|^\s*>>?\s*$|^irb\(.*\):\d+:\d+[>*]\s*$|^\>\s*$/, 'repl'],
         [/^\s*:\s*$|\(END\)|--More--|Press .* to continue|lines \d+-\d+/, 'pager'],
         [/Are you sure|Continue\?|Proceed\?|Confirm|Overwrite/i, 'question'],
         [/Enter\s+\w+:|Input\s+\w+:|Please enter|please provide/i, 'input'],
@@ -143,7 +144,7 @@ module Clacky
 
             stdout_output = stdout_buffer.string
             stderr_output = stderr_buffer.string
-            
+
             {
               command: command,
               stdout: truncate_output(stdout_output, max_output_lines),
@@ -157,7 +158,7 @@ module Clacky
         rescue StandardError => e
           stdout_output = stdout_buffer.string
           stderr_output = "Error executing command: #{e.message}\n#{e.backtrace.first(3).join("\n")}"
-          
+
           {
             command: command,
             stdout: truncate_output(stdout_output, max_output_lines),
@@ -253,10 +254,10 @@ module Clacky
       # Truncate output to max_lines, adding a truncation notice if needed
       def truncate_output(output, max_lines)
         return output if output.nil? || output.empty?
-        
+
         lines = output.lines
         return output if lines.length <= max_lines
-        
+
         truncated_lines = lines.first(max_lines)
         truncation_notice = "\n\n... [Output truncated: showing #{max_lines} of #{lines.length} lines] ...\n"
         truncated_lines.join + truncation_notice
@@ -290,6 +291,109 @@ module Clacky
           "[Exit #{exit_code}] #{error_msg[0..50]}"
         end
       end
+
+      # Format result for LLM consumption - limit output size to save tokens
+      # Maximum characters to include in LLM output
+      MAX_LLM_OUTPUT_CHARS = 1000
+
+      def format_result_for_llm(result)
+        # Return error info as-is if command failed or timed out
+        return result if result[:error] || result[:state] == 'TIMEOUT' || result[:state] == 'WAITING_INPUT'
+
+        stdout = result[:stdout] || ""
+        stderr = result[:stderr] || ""
+        exit_code = result[:exit_code] || 0
+
+        # Build compact result with truncated output
+        compact = {
+          command: result[:command],
+          exit_code: exit_code,
+          success: result[:success]
+        }
+
+        # Add elapsed time if available (keep original precision)
+        compact[:elapsed] = result[:elapsed] if result[:elapsed]
+
+        # Extract command name for temp file naming
+        command_name = extract_command_name(result[:command])
+
+        # Process stdout: truncate and optionally save to temp file
+        stdout_info = truncate_and_save(stdout, MAX_LLM_OUTPUT_CHARS, "stdout", command_name)
+        compact[:stdout] = stdout_info[:content]
+        compact[:stdout_full] = stdout_info[:temp_file] if stdout_info[:temp_file]
+
+        # Process stderr: truncate and optionally save to temp file
+        stderr_info = truncate_and_save(stderr, MAX_LLM_OUTPUT_CHARS, "stderr", command_name)
+        compact[:stderr] = stderr_info[:content]
+        compact[:stderr_full] = stderr_info[:temp_file] if stderr_info[:temp_file]
+
+        # Add output_truncated flag if present
+        compact[:output_truncated] = true if result[:output_truncated]
+
+        compact
+      end
+
+      # Extract command name from full command for temp file naming
+      def extract_command_name(command)
+        first_word = command.strip.split(/\s+/).first
+        File.basename(first_word, ".*")
+      end
+
+      # Truncate output for LLM and optionally save full content to temp file
+      def truncate_and_save(output, max_chars, label, command_name)
+        return { content: "", temp_file: nil } if output.empty?
+
+        return { content: output, temp_file: nil } if output.length <= max_chars
+
+        # Sanitize command name for file name
+        safe_name = command_name.gsub(/[^\w\-.]/, "_")[0...50]
+
+        # Use Ruby tmpdir for safe temp file creation
+        temp_dir = Dir.mktmpdir
+        temp_file = File.join(temp_dir, "#{safe_name}_#{Time.now.strftime("%Y%m%d_%H%M%S")}.output")
+
+        # Write full output to temp file
+        File.write(temp_file, output)
+
+        # For LLM display: show first and last lines to preserve structure
+        lines = output.lines
+        return { content: output, temp_file: nil } if lines.length <= 2
+
+        # Reserve 60 chars for truncation notice
+        available_chars = max_chars - 60
+
+        # Take first few lines
+        first_part = []
+        accumulated = 0
+        lines.each do |line|
+          break if accumulated + line.length > available_chars / 2
+          first_part << line
+          accumulated += line.length
+        end
+
+        # Take last few lines
+        last_part = []
+        accumulated = 0
+        lines.reverse_each do |line|
+          break if accumulated + line.length > available_chars / 2
+          last_part.unshift(line)
+          accumulated += line.length
+        end
+
+        total_lines = lines.length
+
+        # Create notice message based on label
+        if label == "stderr"
+          notice = "... [Error output truncated for LLM: showing #{first_part.length} of #{total_lines} lines, full content: #{temp_file} (use grep to search)] ..."
+        else
+          notice = "... [Output truncated for LLM: showing #{first_part.length} of #{total_lines} lines, full content: #{temp_file} (use grep to search)] ..."
+        end
+
+        # Combine with compact notice
+        content = first_part.join + "\n#{notice}\n" + last_part.join
+
+        { content: content, temp_file: temp_file }
+      end
     end
   end
 end