committer 0.5.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d6586bef8883555a10ed5f9f1e5a5b80100356a1aba0d94209ab2174898406f7
-  data.tar.gz: d2a292a436612443581b1259ad4133b6e6cadf08611a8a40a07f125647744a3a
+  metadata.gz: 15d7ba36e033d0d50db617ab0e02673cead10eea1fa8a86e95180bed40ea3734
+  data.tar.gz: b7e3e8ddb14cf500b1bce8bec83ad1f4ce9f345bf950945365b400550e6a6851
 SHA512:
-  metadata.gz: fab0c91147d5cde773a423d789c35e2eebb224de868ae958294cfa1a06db1ac2e444ab608c51c4ed1c13edd0e0ed289229668b5f61305ba6e1b798c267aaf7a7
-  data.tar.gz: 74a49ce5edf20dc730a2bcaafb5b31d5a78cb55bcf220b8581c14a8dc98e68aff5650c11d0322a0749df9bb1e16826124ef658b68636d408a0fc766afad84d72
+  metadata.gz: 5a180861a3641814d2f517194639f57178c8b4470331a21c1b8a7e404ea437c1c2b4b1ab750c8b1588aee56e0bc2b5837ee6a95e92c72088c84d385268046c2c
+  data.tar.gz: 3a9efb4acca99255a025eea8caa25f9d4583b8a75583f82db363769f64d250f6b195dc83a9fc74428e81c93aa17ae9256c46e66c4f24394fff9d8ae87d71e5e4

data/README.md

@@ -96,6 +96,7 @@ committer setup-git-hook
 ```
 
 This command will:
+
 1. Verify you're in the root of a git repository
 2. Install the prepare-commit-msg hook in your `.git/hooks` directory
 3. Make the hook executable
@@ -103,6 +104,7 @@ This command will:
 ### How the Git Hook Works
 
 Once installed, the git hook will:
+
 1. Automatically run whenever you execute `git commit`
 2. Generate an AI-powered commit message based on your staged changes
 3. Pre-fill your commit message editor with the generated message
@@ -127,6 +129,70 @@ bundle install
 bundle exec rake spec
 ```
 
+## Evaluation Tools
+
+The `eval` directory contains tools for benchmarking and testing commit message generation across different AI models. These tools help evaluate the quality and consistency of generated commit messages.
+
+### Usage
+
+1. **Install dependencies**:
+
+   ```bash
+   npm install
+   npm install -g promptfoo
+   ```
+
+2. **Set environment variables**:
+
+   ```bash
+   export ANTHROPIC_API_KEY=your_anthropic_api_key_here
+   export OPENAI_API_KEY=your_openai_api_key_here
+   ```
+
+3. **Dump commits** from a repository to a database:
+
+   ```bash
+   node eval.mjs dump-commits --eval-data-dir ./path --repo ./repo-path
+   ```
+
+4. **Process commits** for evaluation:
+
+   ```bash
+   node eval.mjs process-commits --eval-data-dir ./path
+   ```
+
+5. **Run evaluations** against different AI models:
+   ```bash
+   node eval.mjs run-evaluation --eval-data-dir ./path [--sha commit_sha] [--limit number]
+   ```
+
+The system compares multiple AI models (Claude 3.7 Sonnet, Claude 3.5 Haiku, GPT-4o, GPT-4o Mini) and tests their ability to generate properly formatted conventional commit messages based on commit diffs.
+
+### Viewing Evaluation Results prompt versions
+
+select your verison from https://drive.google.com/drive/folders/1xHpOkNSss2PM-cnLKGaCHLNfWvxl9hAX
+
+download .promptfoo file and move it to ~/.promptfoo promptfoo.sqlite
+
+```bash
+npm install -g promptfoo
+promptfoo view -y
+```
+
+### Running Evaluation for prompt versions
+
+select your verison from https://drive.google.com/drive/folders/1xHpOkNSss2PM-cnLKGaCHLNfWvxl9hAX
+
+download .electron-eval folder and move it the root fo this project
+
+```bash
+npm install
+npm install -g promptfoo
+export ANTHROPIC_API_KEY=your_anthropic_api_key_here
+export OPENAI_API_KEY=your_openai_api_key_here
+node eval.mjs run-evaluation --eval-data-dir ./electron-eval
+```
+
 ## License
 
 MIT

data/lib/committer/commit_generator.rb

@@ -16,9 +16,8 @@ module Committer
     end
 
     def build_commit_prompt
-      format(template,
-             diff: @diff,
-             commit_context: @commit_context)
+      scopes = Committer::Config::Accessor.instance[:scopes] || []
+      Committer::PromptTemplates.build_prompt(diff, scopes, commit_context)
     end
 
     def template
@@ -55,8 +54,8 @@ module Committer
       end
     end
 
-    def prepare_commit_message
-      client = Clients::ClaudeClient.new
+    def prepare_commit_message(client_class = Clients::ClaudeClient)
+      client = client_class.new
 
       prompt = build_commit_prompt
       response = client.post(prompt)

data/lib/committer/config/accessor.rb

@@ -28,60 +28,52 @@ module Committer
 
       def load_config
         # Load configs from both locations and merge them
-        home_config = load_config_from_path(Committer::Config::Constants::CONFIG_FILE)
-        git_root_config = load_config_from_git_root
+
+        home_config = load_yml_file(read_file_from_home(Committer::Config::Constants::CONFIG_FILE_NAME))
+        git_root_config = load_yml_file(read_file_from_git_root(Committer::Config::Constants::CONFIG_FILE_NAME))
         raise Committer::ConfigErrors::NotSetup if home_config.empty? && git_root_config.empty?
 
+        unless home_config.is_a?(Hash) && git_root_config.is_a?(Hash)
+          raise Committer::ConfigErrors::FormatError,
+                'Config file must be a YAML hash'
+        end
+
         # Merge configs with git root taking precedence
         home_config.merge(git_root_config)
       end
 
-      def load_config_from_path(path)
-        return {} unless File.exist?(path)
-
-        result = YAML.load_file(path)
-        raise Committer::ConfigErrors::FormatError, 'Config file must be a YAML hash' unless result.is_a?(Hash)
-
-        result
+      def load_yml_file(contents)
+        YAML.safe_load(contents, permitted_classes: [Symbol, NilClass, String, Array]) || {}
       end
 
-      def load_file_from_path(path)
+      def read_file_from_path(path)
         return '' unless File.exist?(path)
 
         File.read(path)
       end
 
-      def load_formatting_rules
-        git_root = Committer::GitHelper.repo_root
-        unless git_root.empty?
-          formatting_rules_git_path = File.join(git_root, '.committer',
-                                                Committer::Config::Constants::FORMATTING_RULES_FILE_NAME)
-        end
-
-        git_path_contents = load_file_from_path(formatting_rules_git_path) if formatting_rules_git_path
+      def read_file_from_git_root(file_name)
+        read_file_from_path(File.join(Committer::GitHelper.repo_root, '.committer', file_name))
+      end
 
-        return git_path_contents unless git_path_contents.empty?
+      def read_file_from_home(file_name)
+        read_file_from_path(File.join(Committer::Config::Constants::CONFIG_DIR, file_name))
+      end
 
-        home_path = File.join(Committer::Config::Constants::CONFIG_DIR,
-                              Committer::Config::Constants::FORMATTING_RULES_FILE_NAME)
+      def load_default_file(file_name)
+        read_file_from_path(File.join(Committer::Config::Constants::DEFAULTS_PATH, file_name))
+      end
 
-        home_path_contents = load_file_from_path(home_path)
+      def read_path_prioritized_file(file_name)
+        git_path_contents = read_file_from_git_root(file_name)
 
-        return home_path_contents unless home_path_contents.empty?
+        return git_path_contents unless git_path_contents.empty?
 
-        default_path = File.join(Committer::Config::Constants::DEFAULT_PROMPT_PATH,
-                                 Committer::Config::Constants::FORMATTING_RULES_FILE_NAME)
-        load_file_from_path(default_path)
-      end
+        home_path_contents = read_file_from_home(file_name)
 
-      def load_config_from_git_root
-        git_root = Committer::GitHelper.repo_root
-        return {} if git_root.empty?
+        return home_path_contents unless home_path_contents.empty?
 
-        git_config_file = File.join(git_root, '.committer', 'config.yml')
-        load_config_from_path(git_config_file)
-      rescue StandardError
-        {}
+        load_default_file(file_name)
       end
 
       # Force reload configuration (useful for testing)

data/lib/committer/config/constants.rb

@@ -4,10 +4,12 @@ module Committer
   module Config
     module Constants
       CONFIG_DIR = File.join(Dir.home, '.committer')
-      CONFIG_FILE = File.join(CONFIG_DIR, 'config.yml')
-      FORMATTING_RULES_FILE_NAME = 'formatting_rules.txt'
-      DEFAULT_PROMPT_PATH = File.join(File.dirname(__FILE__), '..')
+      DEFAULTS_PATH = File.join(File.dirname(__FILE__), './defaults')
+
+      COMMIT_MESSAGE_ONLY_PROMPT_FILE_NAME = 'commit_message_only.prompt'
+      COMMIT_MESSAGE_AND_BODY_PROMPT_FILE_NAME = 'commit_message_and_body.prompt'
       CONFIG_FILE_NAME = 'config.yml'
+
       DEFAULT_CONFIG = {
         'api_key' => nil,
         'model' => 'claude-3-7-sonnet-20250219',

data/lib/committer/config/defaults/commit_message_and_body.prompt

@@ -0,0 +1,52 @@
+You are an experienced software developer tasked with creating a commit message based on a git diff. Your goal is to produce a clear, concise, and informative commit message.
+
+First, carefully analyze the following git diff:
+
+<git_diff>
+{{DIFF}}
+</git_diff>
+
+Here are the available scopes (if any):
+
+<scopes>
+{{SCOPES}}
+</scopes>
+
+<user_context>
+{{CONTEXT}}
+</user_context>
+
+Please follow these instructions to generate the commit message:
+
+1. Analyze the git diff and determine the most appropriate commit type from the following options:
+   - feat: A new feature
+   - fix: A bug fix
+   - docs: Documentation only changes
+   - style: Changes that do not affect the meaning of the code
+   - refactor: A code change that neither fixes a bug nor adds a feature
+   - perf: A code change that improves performance
+   - test: Adding missing tests or correcting existing tests
+   - chore: Changes to the build process or auxiliary tools and libraries
+
+
+2. Adhere to these message guidelines:
+   - Keep the summary under 70 characters
+   - Use imperative, present tense (e.g., "add" not "added" or "adds")
+   - Do not end the summary with a period
+   - Be concise but descriptive
+
+3. Format the commit message as follows:
+   - If a scope is available: <type>(<scope>): <description>
+   - If no scope is available: <type>: <description>
+
+4 Body Guidelines:
+   - Add a blank line between summary and body
+   - Use the body to explain why the change was made, incorporating the user's context, defined in <user_context>
+   - Wrap each line in the body at 80 characters maximum
+   - Break the body into multiple paragraphs if needed
+
+   [Your concise commit message in the specified format]
+   [blank line]
+   [Your detailed commit message body]
+
+Respond ONLY with the commit message text (message and body), nothing else.

data/lib/committer/config/defaults/commit_message_only.prompt

@@ -0,0 +1,49 @@
+You are an experienced software developer tasked with creating a commit message based on a git diff. Your goal is to produce a clear, concise, and informative commit message.
+
+First, carefully analyze the following git diff:
+
+<git_diff>
+{{DIFF}}
+</git_diff>
+
+Here are the available scopes (if any):
+
+<scopes>
+{{SCOPES}}
+</scopes>
+
+
+
+2. Adhere to these message guidelines:
+   - Keep the summary under 70 characters
+   - Use imperative, present tense (e.g., "add" not "added" or "adds")
+   - Do not end the summary with a period
+   - Be concise but descriptive
+
+3. Format the commit message as follows:
+   - If a scope is available: <type>(<scope>): <description>
+   - If no scope is available: <type>: <description>
+
+
+Please follow these instructions to generate the commit message:
+
+if such and such changes its docs b ut if commit also has such then its a feat
+
+1. Analyze the git diff and determine the most appropriate commit type from the following options:
+   - feat: A new feature these changes ARE NOT in the docs directory
+   - fix: A bug fix these changes ARE NOT in the docs directory
+   - docs: Documentation only changes for example changes that happen in docs directory
+   - style: Changes that do not affect the code execution (e.g., white-space, formatting, missing semi-colons, etc.) usually done by linters
+   - refactor: 
+      - A code change that does not change the behaviour of the code at all, just how the code is written executed, 
+      - modifing error messages is not a refactor
+      - if parts of the code is removed and no alternative in place IT IS NOT A REFACTOR
+   - perf: A code change that improves performance
+   - test: Adding missing tests or correcting existing tests
+   - build: Changes that affect the build system or external dependencies
+   - ci: 
+      - Changes to the CI configuration files and scripts
+      - these changes should not change the way that the code is built otherwise its build
+
+
+Respond ONLY with the commit message line, nothing else.

data/lib/committer/config/writer.rb

@@ -20,7 +20,6 @@ module Committer
 
       def setup
         create_default_config
-        create_sample_formatting_rules
       end
 
       def write_config_file(file_path, contents)
@@ -34,15 +33,6 @@ module Committer
         end
       end
 
-      def create_sample_formatting_rules
-        default_formatting_rules = File.read(File.join(Committer::Config::Constants::DEFAULT_PROMPT_PATH,
-                                                       Committer::Config::Constants::FORMATTING_RULES_FILE_NAME))
-        formatting_rules_file = File.join(@config_dir,
-                                          "#{Committer::Config::Constants::FORMATTING_RULES_FILE_NAME}.sample")
-        wrote_file = write_config_file(formatting_rules_file, default_formatting_rules)
-        nil unless wrote_file
-      end
-
       def create_default_config
         wrote_file = write_config_file(config_file, Committer::Config::Constants::DEFAULT_CONFIG.to_yaml)
         return unless wrote_file

data/lib/committer/git_helper.rb

@@ -22,7 +22,7 @@ module Committer
         stdout, stderr, status = Open3.capture3('git diff --staged')
         raise Committer::Error, "Failed to get git diff: #{stderr}" unless status.success?
 
-        stdout
+        stdout.dup.force_encoding('UTF-8').scrub
       end
     end
   end

data/lib/committer/prompt_templates.rb

@@ -2,12 +2,19 @@
 
 module Committer
   module PromptTemplates
-    def self.load_formatting_rules
-      Committer::Config::Accessor.instance.load_formatting_rules
+    def self.build_prompt(diff, scopes, commit_context)
+      prompt_template = if commit_context.nil? || commit_context.empty?
+                          Committer::PromptTemplates.build_prompt_summary_only
+                        else
+                          Committer::PromptTemplates.build_prompt_summary_and_body
+                        end
+      prompt_template
+        .gsub('{{DIFF}}', diff)
+        .gsub('{{SCOPES}}', build_scopes_list(scopes))
+        .gsub('{{CONTEXT}}', commit_context || '')
     end
 
-    def self.load_scopes
-      scopes = Committer::Config::Accessor.instance[:scopes] || []
+    def self.build_scopes_list(scopes)
       return 'DO NOT include a scope in your commit message' if scopes.empty?
 
       scope_list = "\nScopes:\n#{scopes.map { |s| "- #{s}" }.join("\n")}"
@@ -15,56 +22,16 @@ module Committer
       "- Choose an appropriate scope from the list above if relevant to the change \n#{scope_list}"
     end
 
-    def self.commit_message_guidelines
-      <<~PROMPT
-        #{load_formatting_rules}
-
-        # Formatting rules with body:
-        <message>
-
-        <blank line>
-        <body with more detailed explanation>
-
-        #{load_scopes}
-
-        # Message Guidelines:
-        - Keep the summary under 70 characters
-        - Use imperative, present tense (e.g., "add" not "added" or "adds")
-        - Do not end the summary with a period
-        - Be concise but descriptive in the summary
-
-        # Body Guidelines:
-        - Add a blank line between summary and body
-        - Use the body to explain why the change was made, incorporating the user's context
-        - Wrap each line in the body at 80 characters maximum
-        - Break the body into multiple paragraphs if needed
-
-        Git Diff:
-        ```
-        %<diff>s
-        ```
-      PROMPT
-    end
-
     def self.build_prompt_summary_only
-      <<~PROMPT
-        Below is a git diff of staged changes. Please analyze it and create a commit message following the formatting rules format with ONLY a message line (NO body):
-
-        #{commit_message_guidelines}
-
-        Respond ONLY with the commit message line, nothing else.
-      PROMPT
+      load_prompt(Committer::Config::Constants::COMMIT_MESSAGE_ONLY_PROMPT_FILE_NAME)
     end
 
     def self.build_prompt_summary_and_body
-      <<~PROMPT
-        Below is a git diff of staged changes. Please analyze it and create a commit message following the formatting rules format with a summary line and a detailed body:
-
-        #{commit_message_guidelines}
-        User's context for this change: %<commit_context>s
+      load_prompt(Committer::Config::Constants::COMMIT_MESSAGE_AND_BODY_PROMPT_FILE_NAME)
+    end
 
-        Respond ONLY with the commit message text (message and body), nothing else.
-      PROMPT
+    def self.load_prompt(file_name)
+      Committer::Config::Accessor.instance.read_path_prioritized_file(file_name)
     end
   end
 end

data/lib/committer/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Committer
-  VERSION = '0.5.0'
+  VERSION = '0.5.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: committer
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Sebastien Stettler
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-03-08 00:00:00.000000000 Z
+date: 2025-03-26 00:00:00.000000000 Z
 dependencies: []
 description: A tool that uses Claude API to generate conventional commit messages
   based on staged changes
@@ -32,8 +32,10 @@ files:
 - lib/committer/committer_errors.rb
 - lib/committer/config/accessor.rb
 - lib/committer/config/constants.rb
+- lib/committer/config/defaults/commit_message_and_body.prompt
+- lib/committer/config/defaults/commit_message_only.prompt
+- lib/committer/config/defaults/formatting_rules.txt
 - lib/committer/config/writer.rb
-- lib/committer/formatting_rules.txt
 - lib/committer/git_helper.rb
 - lib/committer/prepare-commit-msg
 - lib/committer/prompt_templates.rb