openclacky 0.5.3 → 0.5.5

data/lib/clacky/model_pricing.rb

@@ -0,0 +1,280 @@
+# frozen_string_literal: true
+
+module Clacky
+  # Module for handling AI model pricing
+  # Supports different pricing tiers and prompt caching
+  module ModelPricing
+    # Pricing per 1M tokens (MTok) in USD
+    # All pricing is based on official API documentation
+    PRICING_TABLE = {
+      # Claude 4.5 models - tiered pricing based on prompt length
+      "claude-opus-4.5" => {
+        input: {
+          default: 5.00,              # $5/MTok for prompts ≤ 200K tokens
+          over_200k: 5.00             # same for all tiers
+        },
+        output: {
+          default: 25.00,             # $25/MTok for prompts ≤ 200K tokens
+          over_200k: 25.00            # same for all tiers
+        },
+        cache: {
+          write: 6.25,                # $6.25/MTok cache write
+          read: 0.50                  # $0.50/MTok cache read
+        }
+      },
+      
+      "claude-sonnet-4.5" => {
+        input: {
+          default: 3.00,              # $3/MTok for prompts ≤ 200K tokens
+          over_200k: 6.00             # $6/MTok for prompts > 200K tokens
+        },
+        output: {
+          default: 15.00,             # $15/MTok for prompts ≤ 200K tokens
+          over_200k: 22.50            # $22.50/MTok for prompts > 200K tokens
+        },
+        cache: {
+          write_default: 3.75,        # $3.75/MTok cache write (≤ 200K)
+          write_over_200k: 7.50,      # $7.50/MTok cache write (> 200K)
+          read_default: 0.30,         # $0.30/MTok cache read (≤ 200K)
+          read_over_200k: 0.60        # $0.60/MTok cache read (> 200K)
+        }
+      },
+      
+      "claude-haiku-4.5" => {
+        input: {
+          default: 1.00,              # $1/MTok
+          over_200k: 1.00             # same for all tiers
+        },
+        output: {
+          default: 5.00,              # $5/MTok
+          over_200k: 5.00             # same for all tiers
+        },
+        cache: {
+          write: 1.25,                # $1.25/MTok cache write
+          read: 0.10                  # $0.10/MTok cache read
+        }
+      },
+
+      # Claude 3.5 models (for backwards compatibility)
+      "claude-3-5-sonnet-20241022" => {
+        input: {
+          default: 3.00,
+          over_200k: 6.00
+        },
+        output: {
+          default: 15.00,
+          over_200k: 22.50
+        },
+        cache: {
+          write_default: 3.75,
+          write_over_200k: 7.50,
+          read_default: 0.30,
+          read_over_200k: 0.60
+        }
+      },
+
+      "claude-3-5-sonnet-20240620" => {
+        input: {
+          default: 3.00,
+          over_200k: 6.00
+        },
+        output: {
+          default: 15.00,
+          over_200k: 22.50
+        },
+        cache: {
+          write_default: 3.75,
+          write_over_200k: 7.50,
+          read_default: 0.30,
+          read_over_200k: 0.60
+        }
+      },
+
+      "claude-3-5-haiku-20241022" => {
+        input: {
+          default: 1.00,
+          over_200k: 1.00
+        },
+        output: {
+          default: 5.00,
+          over_200k: 5.00
+        },
+        cache: {
+          write: 1.25,
+          read: 0.10
+        }
+      },
+
+      # Default fallback pricing (conservative estimates)
+      "default" => {
+        input: {
+          default: 0.50,
+          over_200k: 0.50
+        },
+        output: {
+          default: 1.50,
+          over_200k: 1.50
+        },
+        cache: {
+          write: 0.625,
+          read: 0.05
+        }
+      }
+    }.freeze
+
+    # Threshold for tiered pricing (200K tokens)
+    TIERED_PRICING_THRESHOLD = 200_000
+
+    class << self
+      # Calculate cost for the given model and usage
+      # 
+      # @param model [String] Model identifier
+      # @param usage [Hash] Usage statistics containing:
+      #   - prompt_tokens: number of input tokens
+      #   - completion_tokens: number of output tokens
+      #   - cache_creation_input_tokens: tokens written to cache (optional)
+      #   - cache_read_input_tokens: tokens read from cache (optional)
+      # @return [Hash] Hash containing:
+      #   - cost: Cost in USD (Float)
+      #   - source: Cost source (:price or :default) (Symbol)
+      def calculate_cost(model:, usage:)
+        pricing_result = get_pricing_with_source(model)
+        pricing = pricing_result[:pricing]
+        source = pricing_result[:source]
+        
+        prompt_tokens = usage[:prompt_tokens] || 0
+        completion_tokens = usage[:completion_tokens] || 0
+        cache_write_tokens = usage[:cache_creation_input_tokens] || 0
+        cache_read_tokens = usage[:cache_read_input_tokens] || 0
+        
+        # Determine if we're in the over_200k tier
+        total_input_tokens = prompt_tokens + cache_write_tokens + cache_read_tokens
+        over_threshold = total_input_tokens > TIERED_PRICING_THRESHOLD
+        
+        # Calculate regular input cost (non-cached tokens)
+        regular_input_tokens = prompt_tokens - cache_write_tokens - cache_read_tokens
+        input_rate = over_threshold ? pricing[:input][:over_200k] : pricing[:input][:default]
+        input_cost = (regular_input_tokens / 1_000_000.0) * input_rate
+        
+        # Calculate output cost
+        output_rate = over_threshold ? pricing[:output][:over_200k] : pricing[:output][:default]
+        output_cost = (completion_tokens / 1_000_000.0) * output_rate
+        
+        # Calculate cache costs
+        cache_cost = calculate_cache_cost(
+          pricing: pricing,
+          cache_write_tokens: cache_write_tokens,
+          cache_read_tokens: cache_read_tokens,
+          over_threshold: over_threshold
+        )
+        
+        {
+          cost: input_cost + output_cost + cache_cost,
+          source: source
+        }
+      end
+      
+      # Get pricing for a specific model
+      # Falls back to default pricing if model not found
+      # 
+      # @param model [String] Model identifier
+      # @return [Hash] Pricing structure for the model
+      def get_pricing(model)
+        get_pricing_with_source(model)[:pricing]
+      end
+      
+      # Get pricing with source information
+      # 
+      # @param model [String] Model identifier
+      # @return [Hash] Hash containing:
+      #   - pricing: Pricing structure for the model
+      #   - source: :price (matched model) or :default (fallback)
+      def get_pricing_with_source(model)
+        # Normalize model name (remove version suffixes, handle variations)
+        normalized_model = normalize_model_name(model)
+        
+        if normalized_model == "default"
+          # Using default fallback pricing
+          {
+            pricing: PRICING_TABLE["default"],
+            source: :default
+          }
+        else
+          # Found specific pricing for this model
+          {
+            pricing: PRICING_TABLE[normalized_model],
+            source: :price
+          }
+        end
+      end
+      
+      private
+      
+      # Normalize model name to match pricing table keys
+      def normalize_model_name(model)
+        return "default" if model.nil? || model.empty?
+        
+        model = model.downcase.strip
+        
+        # Direct match
+        return model if PRICING_TABLE.key?(model)
+        
+        # Check for Claude model variations
+        # Support both dot and dash separators (e.g., "4.5" or "4-5")
+        case model
+        when /claude.*opus.*4[.-]?5/i
+          "claude-opus-4.5"
+        when /claude.*sonnet.*4[.-]?5/i
+          "claude-sonnet-4.5"
+        when /claude.*haiku.*4[.-]?5/i
+          "claude-haiku-4.5"
+        when /claude-3-5-sonnet-20241022/i
+          "claude-3-5-sonnet-20241022"
+        when /claude-3-5-sonnet-20240620/i
+          "claude-3-5-sonnet-20240620"
+        when /claude-3-5-haiku-20241022/i
+          "claude-3-5-haiku-20241022"
+        else
+          "default"
+        end
+      end
+      
+      # Calculate cache-related costs
+      def calculate_cache_cost(pricing:, cache_write_tokens:, cache_read_tokens:, over_threshold:)
+        cache_cost = 0.0
+        
+        # Cache write cost
+        if cache_write_tokens > 0
+          write_rate = if pricing[:cache].key?(:write)
+                         # Simple pricing (Opus 4.5, Haiku 4.5)
+                         pricing[:cache][:write]
+                       elsif over_threshold
+                         # Tiered pricing (Sonnet 4.5)
+                         pricing[:cache][:write_over_200k]
+                       else
+                         pricing[:cache][:write_default]
+                       end
+          
+          cache_cost += (cache_write_tokens / 1_000_000.0) * write_rate
+        end
+        
+        # Cache read cost
+        if cache_read_tokens > 0
+          read_rate = if pricing[:cache].key?(:read)
+                        # Simple pricing (Opus 4.5, Haiku 4.5)
+                        pricing[:cache][:read]
+                      elsif over_threshold
+                        # Tiered pricing (Sonnet 4.5)
+                        pricing[:cache][:read_over_200k]
+                      else
+                        pricing[:cache][:read_default]
+                      end
+          
+          cache_cost += (cache_read_tokens / 1_000_000.0) * read_rate
+        end
+        
+        cache_cost
+      end
+    end
+  end
+end

data/lib/clacky/tools/grep.rb

@@ -3,6 +3,42 @@
 module Clacky
   module Tools
     class Grep < Base
+      # Default patterns to ignore when .gitignore is not available
+      DEFAULT_IGNORED_PATTERNS = [
+        'node_modules',
+        'vendor/bundle',
+        '.git',
+        '.svn',
+        'tmp',
+        'log',
+        'coverage',
+        'dist',
+        'build',
+        '.bundle',
+        '.sass-cache',
+        '.DS_Store',
+        '*.log'
+      ].freeze
+
+      # Config file patterns that should always be searchable
+      CONFIG_FILE_PATTERNS = [
+        /\.env/,
+        /\.ya?ml$/,
+        /\.json$/,
+        /\.toml$/,
+        /\.ini$/,
+        /\.conf$/,
+        /\.config$/,
+        /config\//,
+        /\.config\//
+      ].freeze
+
+      # Maximum file size to search (1MB)
+      MAX_FILE_SIZE = 1_048_576
+
+      # Maximum line length to display (to avoid huge outputs)
+      MAX_LINE_LENGTH = 500
+
       self.tool_name = "grep"
       self.tool_description = "Search file contents using regular expressions. Returns matching lines with context."
       self.tool_category = "file_system"
@@ -30,53 +66,144 @@ module Clacky
           },
           context_lines: {
             type: "integer",
-            description: "Number of context lines to show before and after each match",
+            description: "Number of context lines to show before and after each match (max: 10)",
             default: 0
           },
-          max_matches: {
+          max_files: {
             type: "integer",
             description: "Maximum number of matching files to return",
             default: 50
+          },
+          max_matches_per_file: {
+            type: "integer",
+            description: "Maximum number of matches to return per file",
+            default: 50
+          },
+          max_total_matches: {
+            type: "integer",
+            description: "Maximum total number of matches to return across all files",
+            default: 200
+          },
+          max_file_size: {
+            type: "integer",
+            description: "Maximum file size in bytes to search (default: 1MB)",
+            default: MAX_FILE_SIZE
+          },
+          max_files_to_search: {
+            type: "integer",
+            description: "Maximum number of files to search",
+            default: 500
           }
         },
         required: %w[pattern]
       }
 
-      def execute(pattern:, path: ".", file_pattern: "**/*", case_insensitive: false, context_lines: 0, max_matches: 50)
+      def execute(
+        pattern:,
+        path: ".",
+        file_pattern: "**/*",
+        case_insensitive: false,
+        context_lines: 0,
+        max_files: 50,
+        max_matches_per_file: 50,
+        max_total_matches: 200,
+        max_file_size: MAX_FILE_SIZE,
+        max_files_to_search: 500
+      )
         # Validate pattern
         if pattern.nil? || pattern.strip.empty?
           return { error: "Pattern cannot be empty" }
         end
 
-        # Validate path
-        unless File.exist?(path)
+        # Validate and expand path
+        begin
+          expanded_path = File.expand_path(path)
+        rescue StandardError => e
+          return { error: "Invalid path: #{e.message}" }
+        end
+
+        unless File.exist?(expanded_path)
           return { error: "Path does not exist: #{path}" }
         end
 
+        # Limit context_lines
+        context_lines = [[context_lines, 0].max, 10].min
+
         begin
           # Compile regex
           regex_options = case_insensitive ? Regexp::IGNORECASE : 0
           regex = Regexp.new(pattern, regex_options)
 
+          # Initialize gitignore parser
+          gitignore_path = find_gitignore(expanded_path)
+          gitignore = gitignore_path ? Clacky::GitignoreParser.new(gitignore_path) : nil
+
           results = []
           total_matches = 0
+          files_searched = 0
+          skipped = {
+            binary: 0,
+            too_large: 0,
+            ignored: 0
+          }
+          truncation_reason = nil
 
           # Get files to search
-          files = if File.file?(path)
-                    [path]
+          files = if File.file?(expanded_path)
+                    [expanded_path]
                   else
-                    Dir.glob(File.join(path, file_pattern))
+                    Dir.glob(File.join(expanded_path, file_pattern))
                        .select { |f| File.file?(f) }
-                       .reject { |f| binary_file?(f) }
                   end
 
           # Search each file
           files.each do |file|
-            break if results.length >= max_matches
+            # Check if we've searched enough files
+            if files_searched >= max_files_to_search
+              truncation_reason ||= "max_files_to_search limit reached"
+              break
+            end
+
+            # Skip if file should be ignored (unless it's a config file)
+            if should_ignore_file?(file, expanded_path, gitignore) && !is_config_file?(file)
+              skipped[:ignored] += 1
+              next
+            end
+
+            # Skip binary files
+            if binary_file?(file)
+              skipped[:binary] += 1
+              next
+            end
+
+            # Skip files that are too large
+            if File.size(file) > max_file_size
+              skipped[:too_large] += 1
+              next
+            end
+
+            files_searched += 1
+
+            # Check if we've found enough matching files
+            if results.length >= max_files
+              truncation_reason ||= "max_files limit reached"
+              break
+            end
+
+            # Check if we've found enough total matches
+            if total_matches >= max_total_matches
+              truncation_reason ||= "max_total_matches limit reached"
+              break
+            end
 
-            matches = search_file(file, regex, context_lines)
+            # Search the file
+            matches = search_file(file, regex, context_lines, max_matches_per_file)
             next if matches.empty?
 
+            # Add remaining matches respecting max_total_matches
+            remaining_matches = max_total_matches - total_matches
+            matches = matches.take(remaining_matches) if remaining_matches < matches.length
+
             results << {
               file: File.expand_path(file),
               matches: matches
@@ -87,9 +214,11 @@ module Clacky
           {
             results: results,
             total_matches: total_matches,
-            files_searched: files.length,
+            files_searched: files_searched,
             files_with_matches: results.length,
-            truncated: results.length >= max_matches,
+            skipped_files: skipped,
+            truncated: !truncation_reason.nil?,
+            truncation_reason: truncation_reason,
             error: nil
           }
         rescue RegexpError => e
@@ -116,36 +245,119 @@ module Clacky
         else
           matches = result[:total_matches] || 0
           files = result[:files_with_matches] || 0
-          "✓ Found #{matches} matches in #{files} files"
+          msg = "✓ Found #{matches} matches in #{files} files"
+          
+          # Add truncation info if present
+          if result[:truncated] && result[:truncation_reason]
+            msg += " (truncated: #{result[:truncation_reason]})"
+          end
+          
+          msg
         end
       end
 
       private
 
-      def search_file(file, regex, context_lines)
-        matches = []
-        lines = File.readlines(file, chomp: true)
+      # Find .gitignore file in the search path or parent directories
+      # Only searches within the search path and up to the current working directory
+      def find_gitignore(path)
+        search_path = File.directory?(path) ? path : File.dirname(path)
+        
+        # Look for .gitignore in current and parent directories
+        current = File.expand_path(search_path)
+        cwd = File.expand_path(Dir.pwd)
+        root = File.expand_path('/')
+        
+        # Limit search: only go up to current working directory
+        # This prevents finding .gitignore files from unrelated parent directories
+        # when searching in temporary directories (like /tmp in tests)
+        search_limit = if current.start_with?(cwd)
+                        cwd
+                      else
+                        current
+                      end
+        
+        loop do
+          gitignore = File.join(current, '.gitignore')
+          return gitignore if File.exist?(gitignore)
+          
+          # Stop if we've reached the search limit or root
+          break if current == search_limit || current == root
+          current = File.dirname(current)
+        end
+        
+        nil
+      end
+
+      # Check if file should be ignored based on .gitignore or default patterns
+      def should_ignore_file?(file, base_path, gitignore)
+        # Always calculate path relative to base_path for consistency
+        # Expand both paths to handle symlinks and relative paths correctly
+        expanded_file = File.expand_path(file)
+        expanded_base = File.expand_path(base_path)
+        
+        # For files, use the directory as base
+        expanded_base = File.dirname(expanded_base) if File.file?(expanded_base)
+        
+        # Calculate relative path
+        if expanded_file.start_with?(expanded_base)
+          relative_path = expanded_file[(expanded_base.length + 1)..-1] || File.basename(expanded_file)
+        else
+          # File is outside base path - use just the filename
+          relative_path = File.basename(expanded_file)
+        end
+        
+        # Clean up relative path
+        relative_path = relative_path.sub(/^\.\//, '') if relative_path
+        
+        if gitignore
+          # Use .gitignore rules
+          gitignore.ignored?(relative_path)
+        else
+          # Use default ignore patterns - only match against relative path components
+          DEFAULT_IGNORED_PATTERNS.any? do |pattern|
+            if pattern.include?('*')
+              File.fnmatch(pattern, relative_path, File::FNM_PATHNAME | File::FNM_DOTMATCH)
+            else
+              # Match pattern as a path component (not substring of absolute path)
+              relative_path.start_with?("#{pattern}/") || 
+              relative_path.include?("/#{pattern}/") ||
+              relative_path == pattern ||
+              File.basename(relative_path) == pattern
+            end
+          end
+        end
+      end
 
-        lines.each_with_index do |line, index|
+      # Check if file is a config file (should not be ignored even if in .gitignore)
+      def is_config_file?(file)
+        CONFIG_FILE_PATTERNS.any? { |pattern| file.match?(pattern) }
+      end
+
+      def search_file(file, regex, context_lines, max_matches)
+        matches = []
+        
+        # Use File.foreach for memory-efficient line-by-line reading
+        File.foreach(file, chomp: true).with_index do |line, index|
+          # Stop if we have enough matches for this file
+          break if matches.length >= max_matches
+          
           next unless line.match?(regex)
 
-          # Get context
-          start_line = [0, index - context_lines].max
-          end_line = [lines.length - 1, index + context_lines].min
+          # Truncate long lines
+          display_line = line.length > MAX_LINE_LENGTH ? "#{line[0...MAX_LINE_LENGTH]}..." : line
 
-          context = []
-          (start_line..end_line).each do |i|
-            context << {
-              line_number: i + 1,
-              content: lines[i],
-              is_match: i == index
-            }
+          # Get context if requested
+          if context_lines > 0
+            context = get_line_context(file, index, context_lines)
+          else
+            context = nil
           end
 
           matches << {
             line_number: index + 1,
-            line: line,
-            context: context_lines > 0 ? context : nil
+            line: display_line,
+            context: context
           }
         end
 
@@ -154,6 +366,32 @@ module Clacky
         []
       end
 
+      # Get context lines around a match
+      def get_line_context(file, match_index, context_lines)
+        lines = File.readlines(file, chomp: true)
+        start_line = [0, match_index - context_lines].max
+        end_line = [lines.length - 1, match_index + context_lines].min
+
+        context = []
+        (start_line..end_line).each do |i|
+          line_content = lines[i]
+          # Truncate long lines in context too
+          display_content = line_content.length > MAX_LINE_LENGTH ? 
+                          "#{line_content[0...MAX_LINE_LENGTH]}..." : 
+                          line_content
+          
+          context << {
+            line_number: i + 1,
+            content: display_content,
+            is_match: i == match_index
+          }
+        end
+
+        context
+      rescue StandardError
+        nil
+      end
+
       def binary_file?(file)
         # Simple heuristic: check if file contains null bytes in first 8KB
         return false unless File.exist?(file)

data/lib/clacky/tools/safe_shell.rb

@@ -26,12 +26,17 @@ module Clacky
           hard_timeout: {
             type: "integer",
             description: "Hard timeout in seconds (force kill)"
+          },
+          max_output_lines: {
+            type: "integer",
+            description: "Maximum number of output lines to return (default: 1000)",
+            default: 1000
           }
         },
         required: ["command"]
       }
 
-      def execute(command:, soft_timeout: nil, hard_timeout: nil)
+      def execute(command:, soft_timeout: nil, hard_timeout: nil, max_output_lines: 1000)
         # Get project root directory
         project_root = Dir.pwd
 
@@ -41,7 +46,7 @@ module Clacky
           safe_command = safety_replacer.make_command_safe(command)
 
           # 2. Call parent class execution method
-          result = super(command: safe_command, soft_timeout: soft_timeout, hard_timeout: hard_timeout)
+          result = super(command: safe_command, soft_timeout: soft_timeout, hard_timeout: hard_timeout, max_output_lines: max_output_lines)
 
           # 3. Enhance result information
           enhance_result(result, command, safe_command)
@@ -112,8 +117,8 @@ module Clacky
         return "safe_shell(<no command>)" if cmd.empty?
 
         # Truncate long commands intelligently
-        if cmd.length > 50
-          "safe_shell(\"#{cmd[0..47]}...\")"
+        if cmd.length > 150
+          "safe_shell(\"#{cmd[0..147]}...\")"
         else
           "safe_shell(\"#{cmd}\")"
         end