chaos_to_the_rescue 0.1.0

data/lib/chaos_to_the_rescue/verifier.rb

@@ -0,0 +1,277 @@
+# frozen_string_literal: true
+
+module ChaosToTheRescue
+  # Verifies that method outputs match expected behavior using LLM analysis
+  class Verifier
+    # Result of a verification check
+    VerificationResult = Struct.new(:success, :diagnosis, :suggestion, :fixed_code, :actual_result, keyword_init: true) do
+      def passed?
+        success
+      end
+
+      def failed?
+        !success
+      end
+    end
+
+    def initialize
+      @llm_client = LLMClient.new
+      @redactor = Redactor.new
+    end
+
+    # Verify a method call's output
+    # @param object [Object] the object to call the method on
+    # @param method_name [Symbol] the method to call
+    # @param args [Array] arguments to pass
+    # @param kwargs [Hash] keyword arguments to pass
+    # @param expected [Object] optional expected result (if known)
+    # @param auto_apply [Boolean] whether to automatically apply fixes
+    # @return [VerificationResult]
+    def verify(object:, method_name:, args: [], kwargs: {}, expected: nil, auto_apply: false)
+      ChaosToTheRescue.logger.info("🔍 Verifying #{object.class.name}##{method_name}")
+
+      # Call the method and capture the result
+      actual_result = if kwargs.empty?
+        object.send(method_name, *args)
+      else
+        object.send(method_name, *args, **kwargs)
+      end
+
+      # Get any stored guidance for this method
+      guidance = get_method_guidance(object.class, method_name)
+
+      # Get the method source if possible
+      method_source = extract_method_source(object.class, method_name)
+
+      # Ask LLM to verify the result
+      verification = verify_with_llm(
+        class_name: object.class.name,
+        method_name: method_name,
+        args: args,
+        kwargs: kwargs,
+        actual_result: actual_result,
+        expected_result: expected,
+        guidance: guidance,
+        method_source: method_source
+      )
+
+      if verification[:success]
+        ChaosToTheRescue.logger.info("✅ Verification passed for #{object.class.name}##{method_name}")
+        VerificationResult.new(
+          success: true,
+          diagnosis: verification[:diagnosis],
+          actual_result: actual_result
+        )
+      else
+        ChaosToTheRescue.logger.warn("❌ Verification failed for #{object.class.name}##{method_name}")
+        ChaosToTheRescue.logger.warn("Diagnosis: #{verification[:diagnosis]}")
+
+        result = VerificationResult.new(
+          success: false,
+          diagnosis: verification[:diagnosis],
+          suggestion: verification[:suggestion],
+          fixed_code: verification[:fixed_code],
+          actual_result: actual_result
+        )
+
+        # Auto-apply fix if requested
+        if auto_apply && verification[:fixed_code]
+          apply_fix(object.class, method_name, verification[:fixed_code])
+          ChaosToTheRescue.logger.info("🔧 Auto-applied fix for #{object.class.name}##{method_name}")
+        end
+
+        result
+      end
+    rescue => e
+      ChaosToTheRescue.logger.error("Failed to verify method: #{e.message}")
+      VerificationResult.new(
+        success: false,
+        diagnosis: "Verification failed: #{e.message}",
+        actual_result: actual_result
+      )
+    end
+
+    # Verify a method call using a block
+    # @param auto_apply [Boolean] whether to automatically apply fixes
+    # @yield the method call to verify
+    # @return [VerificationResult]
+    def verify_block(auto_apply: false, &block)
+      # Use TracePoint to capture the method call
+      method_info = capture_method_call(&block)
+
+      verify(
+        object: method_info[:object],
+        method_name: method_info[:method_name],
+        args: method_info[:args],
+        kwargs: method_info[:kwargs],
+        auto_apply: auto_apply
+      )
+    end
+
+    private
+
+    # Capture method call information using TracePoint
+    def capture_method_call(&block)
+      captured = nil
+
+      trace = TracePoint.new(:call) do |tp|
+        captured = {
+          object: tp.self,
+          method_name: tp.method_id,
+          args: tp.binding.local_variables.map { |v| tp.binding.local_variable_get(v) }.compact,
+          kwargs: {}
+        }
+        trace.disable
+      end
+
+      trace.enable
+      result = block.call
+      trace.disable
+
+      captured[:result] = result
+      captured
+    end
+
+    # Get guidance for a method
+    def get_method_guidance(klass, method_name)
+      return nil unless klass.respond_to?(:chaos_method_guidance)
+
+      klass.chaos_method_guidance[method_name]
+    end
+
+    # Extract method source code
+    def extract_method_source(klass, method_name)
+      method = klass.instance_method(method_name)
+      source_location = method.source_location
+      return nil unless source_location
+
+      file_path, line_number = source_location
+      return nil unless File.exist?(file_path)
+
+      # Read the file and extract the method
+      lines = File.readlines(file_path)
+      method_lines = extract_method_lines(lines, line_number)
+
+      method_lines.join
+    rescue => e
+      ChaosToTheRescue.logger.debug("Could not extract method source: #{e.message}")
+      nil
+    end
+
+    # Extract method lines from source file
+    def extract_method_lines(lines, start_line)
+      result = []
+      indent_level = nil
+      in_method = false
+
+      lines[start_line - 1..].each do |line|
+        if line.strip.start_with?("def ")
+          in_method = true
+          indent_level = line.index("def")
+          result << line
+        elsif in_method
+          current_indent = line.index(/\S/)
+          if current_indent && current_indent <= indent_level && line.strip == "end"
+            result << line
+            break
+          else
+            result << line
+          end
+        end
+      end
+
+      result
+    end
+
+    # Verify with LLM
+    def verify_with_llm(class_name:, method_name:, args:, kwargs:, actual_result:, expected_result:, guidance:, method_source:)
+      prompt = build_verification_prompt(
+        class_name: class_name,
+        method_name: method_name,
+        args: args,
+        kwargs: kwargs,
+        actual_result: actual_result,
+        expected_result: expected_result,
+        guidance: guidance,
+        method_source: method_source
+      )
+
+      redacted_prompt = @redactor.redact(prompt)
+      response = @llm_client.send(:call_llm, redacted_prompt)
+
+      parse_verification_response(response)
+    end
+
+    # Build verification prompt
+    def build_verification_prompt(class_name:, method_name:, args:, kwargs:, actual_result:, expected_result:, guidance:, method_source:)
+      <<~PROMPT
+        You are a Ruby code verification assistant. Analyze whether a method's output is correct.
+
+        Class: #{class_name}
+        Method: #{method_name}
+        Arguments: #{args.inspect}
+        #{"Keyword Arguments: #{kwargs.inspect}" unless kwargs.empty?}
+
+        Actual Result: #{actual_result.inspect}
+        #{"Expected Result: #{expected_result.inspect}" if expected_result}
+
+        #{guidance ? "Method Guidance:\n#{guidance}\n" : ""}
+
+        #{method_source ? "Method Source:\n```ruby\n#{method_source}\n```\n" : ""}
+
+        Based on the method name, arguments, and any provided guidance, determine if the actual result is correct.
+
+        Please respond in the following format:
+        SUCCESS: <true or false>
+        DIAGNOSIS: <explanation of whether the result is correct or what went wrong>
+        SUGGESTION: <if incorrect, suggest what the correct result should be>
+        FIXED_CODE:
+        <if incorrect and you have the source, provide fixed Ruby code>
+      PROMPT
+    end
+
+    # Parse verification response
+    def parse_verification_response(response)
+      lines = response.split("\n")
+
+      success_line = lines.find { |l| l.start_with?("SUCCESS:") }
+      success = success_line&.include?("true") || false
+
+      diagnosis_line = lines.find { |l| l.start_with?("DIAGNOSIS:") }
+      diagnosis = diagnosis_line&.sub("DIAGNOSIS:", "")&.strip || "No diagnosis provided"
+
+      suggestion_line = lines.find { |l| l.start_with?("SUGGESTION:") }
+      suggestion = suggestion_line&.sub("SUGGESTION:", "")&.strip
+
+      # Extract fixed code if present
+      code_start = lines.index { |l| l.strip == "FIXED_CODE:" }
+      fixed_code = if code_start
+        lines[(code_start + 1)..].join("\n").strip
+      else
+        nil
+      end
+
+      {
+        success: success,
+        diagnosis: diagnosis,
+        suggestion: suggestion,
+        fixed_code: fixed_code
+      }
+    end
+
+    # Apply a fix to a method
+    def apply_fix(klass, method_name, fixed_code)
+      ChaosToTheRescue.logger.info("🔧 Applying fix to #{klass.name}##{method_name}")
+      ChaosToTheRescue.logger.debug("Fixed code:\n#{fixed_code}")
+
+      begin
+        # WARNING: This executes LLM-generated code!
+        eval(fixed_code, TOPLEVEL_BINDING)
+        ChaosToTheRescue.logger.info("✅ Successfully applied fix to #{klass.name}##{method_name}")
+      rescue => e
+        ChaosToTheRescue.logger.error("❌ Failed to apply fix: #{e.message}")
+        raise
+      end
+    end
+  end
+end

data/lib/chaos_to_the_rescue/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ChaosToTheRescue
+  VERSION = "0.1.0"
+end

data/lib/chaos_to_the_rescue.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require_relative "chaos_to_the_rescue/version"
+require_relative "chaos_to_the_rescue/configuration"
+require_relative "chaos_to_the_rescue/redactor"
+require_relative "chaos_to_the_rescue/logger"
+require_relative "chaos_to_the_rescue/llm_client"
+require_relative "chaos_to_the_rescue/verifier"
+require_relative "chaos_to_the_rescue/chaos_rescue"
+
+# Load Rails integration if Rails is available
+if defined?(Rails)
+  require_relative "chaos_to_the_rescue/railtie"
+end
+
+module ChaosToTheRescue
+  class Error < StandardError; end
+
+  class << self
+    # Get the global configuration instance
+    # @return [Configuration]
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    # Configure the gem with a block
+    # @yield [Configuration] the configuration object
+    def configure
+      configuration.configure do |config|
+        yield config
+      end
+    end
+
+    # Reset configuration to defaults
+    def reset_configuration!
+      @configuration = Configuration.new
+    end
+
+    # Get the global logger instance
+    # @return [Logger]
+    def logger
+      @logger ||= ChaosToTheRescue::Logger.instance
+    end
+
+    # Verify a method call using a block
+    # @param auto_apply [Boolean] whether to automatically apply fixes
+    # @yield the method call to verify
+    # @return [Verifier::VerificationResult]
+    # @example
+    #   result = ChaosToTheRescue.verify { Calculator.new.pow(-5, 2) }
+    #   puts result.diagnosis if result.failed?
+    def verify(auto_apply: false, &block)
+      verifier = Verifier.new
+      verifier.verify_block(auto_apply: auto_apply, &block)
+    end
+  end
+end

data/lib/generators/chaos_to_the_rescue/install_generator.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module ChaosToTheRescue
+  module Generators
+    # Rails generator for installing ChaosToTheRescue initializer
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Creates a ChaosToTheRescue initializer in config/initializers"
+
+      def copy_initializer_file
+        copy_file "chaos_to_the_rescue.rb", "config/initializers/chaos_to_the_rescue.rb"
+      end
+
+      def show_readme
+        readme "INSTALL_README" if behavior == :invoke
+      end
+
+      private
+
+      def readme(filename)
+        say "\n" + "=" * 80
+        say "ChaosToTheRescue has been installed!"
+        say "=" * 80
+        say "\nConfiguration file created at: config/initializers/chaos_to_the_rescue.rb"
+        say "\nIMPORTANT: ChaosToTheRescue is DISABLED by default for safety."
+        say "Edit the initializer to enable it and configure your preferences."
+        say "\nTo use LLM features, add to your Gemfile:"
+        say "  gem 'ruby_llm', '~> 0.1'"
+        say "\nDocumentation: https://github.com/codenamev/chaos_to_the_rescue"
+        say "=" * 80 + "\n"
+      end
+    end
+  end
+end

data/lib/generators/chaos_to_the_rescue/templates/chaos_to_the_rescue.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+# ChaosToTheRescue Configuration
+#
+# This gem provides LLM-powered method generation and Rails exception rescue suggestions.
+# It is DISABLED by default for safety. Enable only in development/test environments.
+#
+# Documentation: https://github.com/codenamev/chaos_to_the_rescue
+
+ChaosToTheRescue.configure do |config|
+  # === CORE SETTINGS ===
+
+  # Enable/disable the gem globally (default: false)
+  # IMPORTANT: Only enable in development/test environments!
+  config.enabled = Rails.env.development? || Rails.env.test?
+
+  # Automatically define generated methods on classes (default: false)
+  # When false, generated code is only returned as a string
+  # When true, methods are dynamically defined (use with caution!)
+  # config.auto_define_methods = false
+
+  # === METHOD GENERATION ALLOWLIST ===
+
+  # Regex patterns for allowed method names
+  # Only methods matching these patterns will be generated
+  # Example: Allow methods starting with "calc_"
+  # config.allowed_method_name_patterns = [/^calc_/, /^compute_/]
+  config.allowed_method_name_patterns = []
+
+  # Explicit allowlist of method names (as strings or symbols)
+  # Example: Allow specific method names
+  # config.allowlist = [:my_dynamic_method, "another_method"]
+  config.allowlist = []
+
+  # === LLM SETTINGS ===
+
+  # LLM model to use (default: "gpt-4o-mini")
+  # Requires ruby_llm gem: gem "ruby_llm", "~> 0.1"
+  # config.model = "gpt-4o-mini"
+
+  # Maximum characters to send in prompts (default: 8000)
+  # Prevents accidentally sending huge prompts to LLM
+  # config.max_prompt_chars = 8000
+
+  # === LOGGING SETTINGS ===
+
+  # Log level (:debug, :info, :warn, :error, :fatal)
+  # config.log_level = :info
+
+  # Enable logging suggestions to file (default: true)
+  # config.log_suggestions = true
+
+  # Path for suggestion log file (default: "tmp/chaos_to_the_rescue_suggestions.log")
+  # config.suggestions_log_path = "tmp/chaos_to_the_rescue_suggestions.log"
+
+  # === SECURITY SETTINGS ===
+
+  # Additional regex patterns to redact from prompts
+  # Default patterns already cover: ENV vars, AWS keys, OpenAI tokens,
+  # GitHub tokens, Rails credentials, and generic API_KEY/SECRET/TOKEN patterns
+  #
+  # Add custom patterns if you have other sensitive data:
+  # config.redact_patterns << /CUSTOM_SECRET[=:]\s*['"]?([^'"\s]+)['"]?/i
+end
+
+# === USAGE EXAMPLES ===
+
+# 1. Method Generation with ChaosRescue module:
+#
+#   class Calculator
+#     include ChaosToTheRescue::ChaosRescue
+#     chaos_rescue_enabled! # Enable for this class
+#   end
+#
+#   calc = Calculator.new
+#   calc.calc_fibonacci(10) # Method generated on-the-fly if allowlisted
+#
+
+# 2. Rails rescue_from integration:
+#
+#   class ApplicationController < ActionController::Base
+#     include ChaosToTheRescue::RescueFrom
+#
+#     rescue_from StandardError do |exception|
+#       suggestion = chaos_suggest_fix(
+#         exception,
+#         guidance: "This is a payment processing error"
+#       )
+#
+#       # Use suggestion to fix the issue or log it
+#       Rails.logger.info("Suggestion: #{suggestion[:title]}")
+#       Rails.logger.info("Diagnosis: #{suggestion[:diagnosis]}")
+#
+#       # Render error page
+#       render "errors/500", status: :internal_server_error
+#     end
+#   end

data/sig/chaos_to_the_rescue.rbs

@@ -0,0 +1,4 @@
+module ChaosToTheRescue
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,89 @@
+--- !ruby/object:Gem::Specification
+name: chaos_to_the_rescue
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Valentino Stoll
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+description: ChaosToTheRescue uses LLMs to generate missing methods on-the-fly and
+  suggest fixes for Rails exceptions. Features comprehensive security (secret redaction,
+  opt-in behavior, no auto-execution) and is disabled by default for production safety.
+email:
+- v@codenamev.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".claude/CLAUDE.md"
+- ".claude/memory.sqlite3"
+- ".claude/rules/claude_memory.generated.md"
+- ".claude/settings.local.json"
+- ".devcontainer/.env.example"
+- ".devcontainer/Dockerfile.standalone"
+- ".devcontainer/README.md"
+- ".devcontainer/devcontainer.json"
+- ".devcontainer/docker-compose.yml"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- FEATURES.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- examples/guidance_and_verification.rb
+- lib/chaos_to_the_rescue.rb
+- lib/chaos_to_the_rescue/chaos_rescue.rb
+- lib/chaos_to_the_rescue/configuration.rb
+- lib/chaos_to_the_rescue/llm_client.rb
+- lib/chaos_to_the_rescue/logger.rb
+- lib/chaos_to_the_rescue/railtie.rb
+- lib/chaos_to_the_rescue/redactor.rb
+- lib/chaos_to_the_rescue/rescue_from.rb
+- lib/chaos_to_the_rescue/verifier.rb
+- lib/chaos_to_the_rescue/version.rb
+- lib/generators/chaos_to_the_rescue/install_generator.rb
+- lib/generators/chaos_to_the_rescue/templates/chaos_to_the_rescue.rb
+- sig/chaos_to_the_rescue.rbs
+homepage: https://github.com/codenamev/chaos_to_the_rescue
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/codenamev/chaos_to_the_rescue
+  source_code_uri: https://github.com/codenamev/chaos_to_the_rescue
+  changelog_uri: https://github.com/codenamev/chaos_to_the_rescue/blob/main/CHANGELOG.md
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.4
+specification_version: 4
+summary: Safe-by-default LLM-powered method generation and Rails error rescue suggestions
+test_files: []