ai_refactor 0.3.1 → 0.4.0

data/lib/ai_refactor/file_processor.rb

@@ -1,18 +1,19 @@
 # frozen_string_literal: true
 
+require "fileutils"
 require "openai"
 require "json"
 
 module AIRefactor
   class FileProcessor
-    attr_reader :file_path, :output_path, :logger
+    attr_reader :input_file_path, :output_path, :logger, :options
 
-    def initialize(input_path:, prompt_file_path:, ai_client:, logger:, output_path: nil)
-      @file_path = input_path
-      @prompt_file_path = prompt_file_path
+    def initialize(prompt:, ai_client:, logger:, output_path: nil, options: {})
+      @prompt = prompt
       @ai_client = ai_client
       @logger = logger
       @output_path = output_path
+      @options = options
     end
 
     def output_exists?
@@ -20,20 +21,26 @@ module AIRefactor
       File.exist?(output_path)
     end
 
-    def process!(options)
-      logger.debug("Processing #{file_path} with prompt in #{@prompt_file_path}")
-      prompt = File.read(@prompt_file_path)
-      input = File.read(@file_path)
-      messages = [
-        {role: "system", content: prompt},
-        {role: "user", content: "Convert: ```#{input}```"}
-      ]
-      content, finished_reason, usage = generate_next_message(messages, prompt, options, options[:ai_max_attempts] || 3)
+    def process!
+      logger.debug("Processing #{@prompt.input_file_path} with prompt in #{@prompt.prompt_file_path}")
+      logger.debug("Options: #{options.inspect}")
+      messages = @prompt.chat_messages
+      if options[:review_prompt]
+        logger.info "Review prompt:\n"
+        messages.each do |message|
+          logger.info "\n-- Start of prompt for Role #{message[:role]} --\n"
+          logger.info message[:content]
+          logger.info "\n-- End of prompt for Role #{message[:role]} --\n"
+        end
+        return [nil, "Skipped as review prompt was requested", nil]
+      end
+
+      content, finished_reason, usage = generate_next_message(messages, options, ai_max_attempts)
 
       content = if content && content.length > 0
         processed = block_given? ? yield(content) : content
         if output_path
-          File.write(output_path, processed)
+          write_output(output_path, processed)
           logger.verbose "Wrote output to #{output_path}..."
         end
         processed
@@ -44,14 +51,18 @@ module AIRefactor
 
     private
 
-    def generate_next_message(messages, prompt, options, attempts_left)
+    def ai_max_attempts
+      options[:ai_max_attempts] || 1
+    end
+
+    def generate_next_message(messages, options, attempts_left)
       logger.verbose "Generate AI output. Generation attempts left: #{attempts_left}"
       logger.debug "Options: #{options.inspect}"
       logger.debug "Messages: #{messages.inspect}"
 
       response = @ai_client.chat(
         parameters: {
-          model: options[:ai_model] || "gpt-3.5-turbo",
+          model: options[:ai_model] || "gpt-4",
           messages: messages,
           temperature: options[:ai_temperature] || 0.7,
           max_tokens: options[:ai_max_tokens] || 1500
@@ -69,7 +80,7 @@ module AIRefactor
         generate_next_message(messages + [
           {role: "assistant", content: content},
           {role: "user", content: "Continue"}
-        ], prompt, options, attempts_left - 1)
+        ], options, attempts_left - 1)
       else
         previous_messages = messages.filter { |m| m[:role] == "assistant" }.map { |m| m[:content] }.join
         content = if previous_messages.length > 0
@@ -80,5 +91,11 @@ module AIRefactor
         [content, finished_reason, response["usage"]]
       end
     end
+
+    def write_output(output_path, processed)
+      dir = File.dirname(output_path)
+      FileUtils.mkdir_p(dir) unless File.directory?(dir)
+      File.write(output_path, processed)
+    end
   end
 end

data/lib/ai_refactor/prompt.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module AIRefactor
+  class Prompt
+    INPUT_FILE_PATH_MARKER = "__{{input_file_path}}__"
+    OUTPUT_FILE_PATH_MARKER = "__{{output_file_path}}__"
+    HEADER_MARKER = "__{{prompt_header}}__"
+    FOOTER_MARKER = "__{{prompt_footer}}__"
+    CONTEXT_MARKER = "__{{context}}__"
+    CONTENT_MARKER = "__{{content}}__"
+
+    attr_reader :input_file_path, :prompt_file_path
+
+    def initialize(options:, logger:, context: nil, input_content: nil, input_path: nil, output_file_path: nil, prompt_file_path: nil, prompt_header: nil, prompt_footer: nil)
+      @input_content = input_content
+      @input_file_path = input_path
+      @output_file_path = output_file_path
+      @prompt_file_path = prompt_file_path
+      @logger = logger
+      @header = prompt_header
+      @footer = prompt_footer
+      @diff = options[:diff]
+      @context = context
+    end
+
+    def chat_messages
+      [
+        {role: "system", content: system_prompt},
+        {role: "user", content: user_prompt}
+      ]
+    end
+
+    private
+
+    def system_prompt
+      prompt = expand_prompt(system_prompt_template, HEADER_MARKER, @header || "")
+      prompt = expand_prompt(prompt, CONTEXT_MARKER, @context&.prepare_context || "")
+      prompt = expand_prompt(prompt, INPUT_FILE_PATH_MARKER, @input_file_path || "")
+      prompt = expand_prompt(prompt, OUTPUT_FILE_PATH_MARKER, @output_file_path || "")
+      expand_prompt(prompt, FOOTER_MARKER, system_prompt_footer)
+    end
+
+    def system_prompt_template
+      File.read(@prompt_file_path)
+    end
+
+    def system_prompt_footer
+      if @diff && @footer
+        "#{@footer}\n\n#{diff_prompt}"
+      elsif @diff
+        diff_prompt
+      elsif @footer
+        @footer
+      else
+        ""
+      end
+    end
+
+    def diff_prompt
+      File.read(prompt_path("diff.md"))
+    end
+
+    def prompt_path(file)
+      File.join(File.dirname(File.expand_path(__FILE__)), "prompts", file)
+    end
+
+    def user_prompt
+      expand_prompt(input_prompt, CONTENT_MARKER, input_to_process)
+    end
+
+    def input_to_process
+      return File.read(@input_file_path) if @input_file_path
+      @input_content
+    end
+
+    def input_prompt
+      File.read(prompt_path("input.md"))
+    end
+
+    def expand_prompt(prompt, marker, content)
+      prompt.gsub(marker, content)
+    end
+  end
+end

data/lib/ai_refactor/prompts/diff.md

@@ -0,0 +1,17 @@
+You MUST generate a diff in a format that can be understood and applied using git.
+
+Generate diff hunks that capture the modifications you see. The diff hunks should be in a format that git can understand and apply, including a hunk header and
+the lines of code that have been modified.
+Finally, output the generated diff as your answer. Do not provide further instruction.
+
+Example diff:
+
+```
+@@ -27,7 +27,7 @@ module AIRefactor
+       File.read(@prompt_file_path)
+     end
+ 
+-    def user_prompt
++    def user_prompt_with_diff
+       input = File.read(@file_path)
+```

data/lib/ai_refactor/prompts/input.md

@@ -0,0 +1 @@
+```__{{content}}__```

data/lib/ai_refactor/refactors/base_refactor.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+module AIRefactor
+  module Refactors
+    class BaseRefactor
+      # All subclasses must register themselves with the Registry
+      def self.inherited(subclass)
+        super
+        Refactors.register(subclass)
+      end
+
+      def self.description
+        "(No description provided)"
+      end
+
+      def self.takes_input_files?
+        true
+      end
+
+      attr_reader :input_file, :options, :logger
+      attr_accessor :input_content
+      attr_writer :failed_message
+
+      def initialize(input_file, options, logger)
+        @input_file = input_file
+        @options = options
+        @logger = logger
+      end
+
+      def run
+        raise NotImplementedError
+      end
+
+      def failed_message
+        @failed_message || "Reason not specified"
+      end
+
+      private
+
+      def file_processor
+        context = ::AIRefactor::Context.new(files: options[:context_file_paths], text: options[:context_text], logger: logger)
+        prompt = ::AIRefactor::Prompt.new(input_content: input_content, input_path: input_file, output_file_path: output_file_path, prompt_file_path: prompt_file_path, context: context, logger: logger, options: options)
+        AIRefactor::FileProcessor.new(prompt: prompt, ai_client: ai_client, output_path: output_file_path, logger: logger, options: options)
+      end
+
+      def process!(strip_ticks: true)
+        processor = file_processor
+
+        if processor.output_exists?
+          return false unless overwrite_existing_output?(output_file_path)
+        end
+
+        logger.verbose "Processing #{input_file}..."
+
+        begin
+          output_content, finished_reason, usage = processor.process! do |content|
+            if block_given?
+              yield content
+            elsif strip_ticks
+              content.gsub("```ruby", "").gsub("```", "")
+            else
+              content
+            end
+          end
+
+          logger.verbose "AI finished, with reason '#{finished_reason}'..."
+          logger.verbose "Used tokens: #{usage["total_tokens"]}".colorize(:light_black) if usage
+          if finished_reason == "length"
+            logger.warn "Translation may contain an incomplete output as the max token length was reached. You can try using the '--continue' option next time to increase the length of generated output."
+          end
+
+          if !output_content || output_content.length == 0
+            logger.warn "Skipping #{input_file}, no translated output..."
+            logger.error "Failed to translate #{input_file}, finished reason #{finished_reason}"
+            self.failed_message = "AI conversion failed, no output was generated"
+            raise NoOutputError, "No output"
+          end
+
+          output_content
+        rescue => e
+          logger.error "Request to AI failed: #{e.message}"
+          logger.warn "Skipping #{input_file}..."
+          self.failed_message = "Request to OpenAI failed"
+          raise e
+        end
+      end
+
+      def overwrite_existing_output?(output_path)
+        overwrite = options && options[:overwrite]&.downcase
+        answer = if ["y", "n"].include? overwrite
+          overwrite
+        else
+          logger.info "Do you wish to overwrite #{output_path}? (y/n)"
+          $stdin.gets.chomp.downcase
+        end
+        if answer == "y"
+          logger.verbose "Overwriting #{output_path}..."
+          return true
+        end
+        logger.warn "Skipping #{input_file}..."
+        self.failed_message = "Skipped as output file already exists"
+        false
+      end
+
+      def prompt_file_path
+        file = if options && options[:prompt_file_path]&.length&.positive?
+          options[:prompt_file_path]
+        else
+          location = Module.const_source_location(::AIRefactor::Refactors::BaseRefactor.name)
+          File.join(File.dirname(location.first), "#{refactor_name}.md")
+        end
+        file.tap do |prompt|
+          raise "No prompt file '#{prompt}' found for #{refactor_name}" unless File.exist?(prompt)
+        end
+      end
+
+      def output_file_path
+        @output_file_path ||= determine_output_file_path
+      end
+
+      def determine_output_file_path
+        return output_file_path_from_template if output_template_path
+
+        path = options[:output_file_path]
+        return default_output_path unless path
+
+        if path == true
+          input_file
+        else
+          path
+        end
+      end
+
+      def default_output_path
+        nil
+      end
+
+      def output_template_path
+        options[:output_template_path]
+      end
+
+      def output_file_path_from_template
+        path = output_template_path.gsub("[FILE]", File.basename(input_file))
+          .gsub("[NAME]", File.basename(input_file, ".*"))
+          .gsub("[DIR]", File.dirname(input_file))
+          .gsub("[REFACTOR]", self.class.refactor_name)
+          .gsub("[EXT]", File.extname(input_file))
+        raise "Output template could not be used" unless path.length.positive?
+        path
+      end
+
+      def ai_client
+        @ai_client ||= OpenAI::Client.new
+      end
+
+      def refactor_name
+        self.class.refactor_name
+      end
+
+      class << self
+        def command_line_options
+          []
+        end
+
+        def refactor_name
+          name.gsub("AIRefactor::Refactors::", "")
+            .gsub(/::/, "/")
+            .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+            .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+            .tr("-", "_")
+            .downcase
+        end
+      end
+    end
+  end
+end

data/lib/ai_refactor/refactors/generic.rb

@@ -7,74 +7,23 @@ module AIRefactor
         logger.verbose "Generic refactor to #{input_file}... (using user supplied prompt #{prompt_file_path})"
         logger.verbose "Write output to #{output_file_path}..." if output_file_path
 
-        processor = AIRefactor::FileProcessor.new(
-          input_path: input_file,
-          prompt_file_path: prompt_file_path,
-          ai_client: ai_client,
-          logger: logger,
-          output_path: output_file_path
-        )
-
-        if processor.output_exists?
-          return false unless can_overwrite_output_file?(output_file_path)
-        end
-
-        logger.verbose "Converting #{input_file}..."
-
         begin
-          output_content, finished_reason, usage = processor.process!(options)
+          output_content = process!(strip_ticks: false)
         rescue => e
-          logger.error "Request to OpenAI failed: #{e.message}"
-          logger.warn "Skipping #{input_file}..."
-          self.failed_message = "Request to OpenAI failed"
+          logger.error "Failed to process #{input_file}: #{e.message}"
           return false
         end
 
-        logger.verbose "OpenAI finished, with reason '#{finished_reason}'..."
-        logger.verbose "Used tokens: #{usage["total_tokens"]}".colorize(:light_black) if usage
-
-        if finished_reason == "length"
-          logger.warn "Translation may contain an incomplete output as the max token length was reached. You can try using the '--continue' option next time to increase the length of generated output."
-        end
-
-        if !output_content || output_content.length == 0
-          logger.warn "Skipping #{input_file}, no translated output..."
-          logger.error "Failed to translate #{input_file}, finished reason #{finished_reason}"
-          self.failed_message = "AI conversion failed, no output was generated"
-          return false
-        end
+        return false unless output_content
 
         output_file_path ? true : output_content
       end
 
-      private
-
-      def output_file_path
-        return output_file_path_from_template if output_template_path
-
-        path = options[:output_file_path]
-        return unless path
-
-        if path == true
-          input_file
-        else
-          path
-        end
-      end
-
-      def output_template_path
-        options[:output_template_path]
+      def self.description
+        "Generic refactor using user supplied prompt"
       end
 
-      def output_file_path_from_template
-        path = output_template_path.gsub("[FILE]", File.basename(input_file))
-          .gsub("[NAME]", File.basename(input_file, ".*"))
-          .gsub("[DIR]", File.dirname(input_file))
-          .gsub("[REFACTOR]", self.class.refactor_name)
-          .gsub("[EXT]", File.extname(input_file))
-        raise "Output template could not be used" unless path.length.positive?
-        path
-      end
+      private
 
       def prompt_file_path
         specified_prompt_path = options[:prompt_file_path]
@@ -89,25 +38,6 @@ module AIRefactor
         end
         exit 1
       end
-
-      class << self
-        def command_line_options
-          [
-            {
-              key: :output_file_path,
-              long: "--output [FILE]",
-              type: String,
-              help: "Write output to file instead of stdout. If no path provided will overwrite input file (will prompt to overwrite existing files)"
-            },
-            {
-              key: :output_template_path,
-              long: "--output-template TEMPLATE",
-              type: String,
-              help: "Write outputs to files instead of stdout. The template is used to create the output name, where the it can have substitutions, '[FILE]', '[NAME]', '[DIR]', '[REFACTOR]' & '[EXT]'. Eg `[DIR]/[NAME]_[REFACTOR][EXT]` (will prompt to overwrite existing files)"
-            }
-          ]
-        end
-      end
     end
   end
 end

data/lib/ai_refactor/refactors/minitest/write_test_for_class.md

@@ -0,0 +1,11 @@
+You are an expert Ruby senior software developer. Write a unit test for the class show below, using minitest and minitest/mock if necessary.
+Test 100% of the code. 
+
+The path to the file to test is: __{{input_file_path}}__
+The output file path is: __{{output_file_path}}__
+
+Only show me the test file code. Do NOT provide any other description of your work. Always enclose the output code in triple backticks (```).
+
+__{{context}}__
+
+The class to test is:

data/lib/ai_refactor/refactors/minitest/write_test_for_class.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module AIRefactor
+  module Refactors
+    module Minitest
+      class WriteTestForClass < BaseRefactor
+        def run
+          logger.verbose "'Write minitest test' refactor for #{input_file}..."
+          logger.verbose "Write output to #{output_file_path}..." if output_file_path
+
+          begin
+            output_content = process!
+          rescue => e
+            logger.error "Failed to process #{input_file}: #{e.message}"
+            return false
+          end
+
+          return false unless output_file_path
+
+          logger.verbose "Generated #{output_file_path} from #{input_file} ..." if output_content
+
+          minitest_runner = AIRefactor::TestRunners::MinitestRunner.new(output_file_path, command_template: "bundle exec ruby __FILE__")
+
+          logger.verbose "Run generated test file #{output_file_path} (#{minitest_runner.command})..."
+          test_run = minitest_runner.run
+
+          if test_run.failed?
+            logger.warn "#{input_file} was translated to #{output_file_path} but the resulting test is failing..."
+            logger.error "Failed to run test, exited with status #{test_run.exitstatus}. Stdout: #{test_run.stdout}\n\nStderr: #{test_run.stderr}\n\n"
+            logger.error "New test failed!", bold: true
+            self.failed_message = "Generated test file failed to run correctly"
+            return false
+          end
+
+          logger.verbose "\nNew test file ran and returned the following results:"
+          logger.verbose ">> Runs: #{test_run.example_count}, Failures: #{test_run.failure_count}, Skips: #{test_run.pending_count}\n"
+
+          output_file_path ? true : output_content
+        end
+
+        def self.description
+          "Write a minitest test for a class"
+        end
+
+        def default_output_path
+          File.join("test", input_file.gsub(/\.rb$/, "_test.rb"))
+        end
+      end
+    end
+  end
+end

data/lib/ai_refactor/refactors/project/write_changelog_from_history.md

@@ -0,0 +1,35 @@
+You are a senior Ruby software developer. You are diligent about writing clear changelogs for your users.
+
+You have a git history of changes to your project. You want to write a changelog from this history.
+
+The format of the changelog you will create entries for is from keepachangelog.com version 1.0.0.
+
+Here is an example of a set of changelog entries:
+
+```
+## [0.2.0] - 2015-10-06
+
+### Changed
+
+- Remove exclusionary mentions of "open source" since this project can
+  benefit both "open" and "closed" source projects equally.
+
+## [0.1.0] - 2015-10-06
+
+### Added
+
+- Answer "Should you ever rewrite a change log?".
+
+### Changed
+
+- Improve argument against commit logs.
+- Start following [SemVer](https://semver.org) properly.
+
+### Fixed
+
+- Fix typos in recent CHANGELOG entries.
+```
+
+If the history contains any mention of changes of version (such as "Bump version to 1.0.0"), then you should use that version number in a new changelog entry.
+
+Now write changelog entries from the following git history.

data/lib/ai_refactor/refactors/project/write_changelog_from_history.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module AIRefactor
+  module Refactors
+    module Project
+      class WriteChangelogFromHistory < BaseRefactor
+        def self.description
+          "Write changelog entries from the git history"
+        end
+
+        def self.takes_input_files?
+          false
+        end
+
+        def run
+          logger.verbose "Creating changelog entries for project from #{options[:git_commit_count] || 3} commits..."
+          logger.verbose "Write output to #{output_file_path}..." if output_file_path
+
+          self.input_content = `git log -#{options[:git_commit_count] || 3} --pretty=format:"%ci %d %s"`.split("\n").map { |line| "- #{line}" }.join("\n")
+          logger.debug "\nInput messages: \n#{input_content}\n\n"
+          begin
+            output_content = process!(strip_ticks: false)
+          rescue => e
+            logger.error "Failed to process: #{e.message}"
+            return false
+          end
+
+          return false unless output_content
+
+          output_file_path ? true : output_content
+        end
+
+        def default_output_path
+          nil
+        end
+
+        def self.command_line_options
+          [
+            {
+              key: :git_commit_count,
+              long: "--commits N",
+              type: Integer,
+              help: "The number of commits to analyse when creating the changelog entries (defaults to 3)"
+            }
+          ]
+        end
+      end
+    end
+  end
+end