aireview 0.2.1 → 2.0.0

data/lib/aireview/candidate_checker.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+require 'logger'
+require 'set'
+
+module Aireview
+  # A mechanical check that a candidate points at the code, against the diff
+  # the model actually saw: file, line, quote. The anchoring is checked, not
+  # the bug itself: a quote that is not found is a reason for Critique to
+  # look closer, not proof of a fabrication. A file that is not among the MR
+  # changes is a different matter: there is nothing to check such a
+  # candidate against, it is dropped.
+  class CandidateChecker
+    HUNK_HEADER = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/
+    FILE_HEADER = %r{^diff --git a/(.+?) b/(.+)$}
+    NOTE_QUOTE_NOT_FOUND = 'quoted_code not found in the diff shown to the model'
+    NOTE_LINE_RESET = 'line was outside the shown hunks and has been reset to null'
+    NOTE_NOT_VERIFIED = 'file shown partially or without a diff, location not verified'
+
+    # One file of the context: the new-file line ranges of the shown hunks
+    # and the normalized text of every hunk for the quote search — the new
+    # side (context + added lines) and the old side (context + removed
+    # lines) separately, so that a multi-line quote from one version of the
+    # file is found whole. Hunks are not glued together: there is skipped
+    # code between them, and a quote across a hunk boundary is not a quote.
+    Section = Struct.new(:ranges, :hunks, :partial, keyword_init: true) do
+      def include?(quote)
+        hunks.any? { |hunk| hunk[:new_text].include?(quote) || hunk[:old_text].include?(quote) }
+      end
+    end
+
+    def initialize(changes:, diff_text:, coverage:, logger: Logger.new($stderr))
+      @mr_paths = changes.flat_map { |change| change.values_at('old_path', 'new_path') }.compact.to_set
+      @coverage = coverage
+      @logger = logger
+      @sections = parse_sections(diff_text.to_s)
+    end
+
+    # Returns the candidates with marks: a note for Critique, quote_missing
+    # for the report, line reset to null when it was not confirmed.
+    # Candidates with a file outside the MR are dropped.
+    def check(candidates)
+      candidates.filter_map do |candidate|
+        id = value(candidate, 'id')
+        path = resolve_path(value(candidate, 'file'))
+        unless path
+          @logger.warn("Candidate #{id} dropped: file #{value(candidate, 'file').inspect} is not in the merge request")
+          next
+        end
+
+        annotate(candidate.dup, id: id, section: @sections[path])
+      end
+    end
+
+    private
+
+    def annotate(candidate, id:, section:)
+      notes = []
+      if section.nil? || section.partial
+        @logger.debug("Candidate #{id}: file shown partially or not at all, location not verified")
+        notes << NOTE_NOT_VERIFIED
+      else
+        notes << check_line(candidate, id, section)
+        notes << check_quote(candidate, id, section)
+      end
+      notes.compact!
+      set(candidate, 'note', notes.join('; ')) unless notes.empty?
+      candidate
+    end
+
+    def check_line(candidate, id, section)
+      line = value(candidate, 'line')
+      return nil if line.nil?
+      return nil if line.is_a?(Integer) && section.ranges.any? { |range| range.cover?(line) }
+
+      @logger.warn("Candidate #{id}: line #{line} is outside the shown hunks, reset to null")
+      set(candidate, 'line', nil)
+      NOTE_LINE_RESET
+    end
+
+    def check_quote(candidate, id, section)
+      quote = normalize_text(value(candidate, 'quoted_code'))
+      return nil if quote.empty? || section.include?(quote)
+
+      @logger.warn("Candidate #{id}: quoted_code not found in the diff shown to the model")
+      set(candidate, 'quote_missing', true)
+      NOTE_QUOTE_NOT_FOUND
+    end
+
+    # The diff is already packed to the budget: a partially shown file may
+    # lack hunks, a file without a diff has none at all. The --- / +++
+    # headers occur only between the file header and the first @@; inside a
+    # hunk a "+++ x" line is the added code "++ x".
+    def parse_sections(diff_text)
+      sections = {}
+      section = nil
+      hunk = nil
+      diff_text.each_line do |line|
+        if (header = line.match(FILE_HEADER))
+          section = Section.new(ranges: [], hunks: [], partial: false)
+          hunk = nil
+          header.captures.map(&:strip).each { |path| sections[path] = section }
+        elsif section && (hunk_header = line.match(HUNK_HEADER))
+          hunk = start_hunk(section, hunk_header)
+        elsif hunk
+          add_line(hunk, line)
+        end
+      end
+      mark_partial(sections)
+      normalize_hunks(sections)
+    end
+
+    # A hunk without new lines (a deletion, @@ -1 +0,0 @@) gives no range:
+    # there is no line 0 in the new file.
+    def start_hunk(section, header)
+      start = header[1].to_i
+      length = header[2] ? header[2].to_i : 1
+      section.ranges << (start..(start + length - 1)) if length.positive?
+      hunk = {new_text: +'', old_text: +''}
+      section.hunks << hunk
+      hunk
+    end
+
+    def add_line(hunk, line)
+      if line.start_with?(' ')
+        hunk[:new_text] << line[1..]
+        hunk[:old_text] << line[1..]
+      elsif line.start_with?('+')
+        hunk[:new_text] << line[1..]
+      elsif line.start_with?('-')
+        hunk[:old_text] << line[1..]
+      end
+    end
+
+    def normalize_hunks(sections)
+      sections.each_value do |section|
+        section.hunks.each { |hunk| hunk.transform_values! { |text| normalize_text(text) } }
+      end
+    end
+
+    def mark_partial(sections)
+      partial = @coverage.files_partial.map { |file| file[:path] } + @coverage.files_unavailable
+      partial += @coverage.hunks_skipped.map { |skipped| skipped[:path] }
+      partial.each { |path| sections[path]&.partial = true }
+    end
+
+    # An exact match first: a/ and b/ directories can be real. The prefix
+    # from the diff header is stripped only when the exact path is not in
+    # the MR.
+    def resolve_path(path)
+      path = path.to_s.strip
+      return path if @mr_paths.include?(path)
+
+      stripped = path.sub(%r{\A(?:\./|[ab]/)}, '')
+      stripped if @mr_paths.include?(stripped)
+    end
+
+    def normalize_text(text)
+      text.to_s.gsub(/\s+/, ' ').strip
+    end
+
+    def value(hash, key)
+      hash[key] || hash[key.to_sym]
+    end
+
+    def set(hash, key, new_value)
+      hash[hash.key?(key.to_sym) ? key.to_sym : key] = new_value
+    end
+  end
+end

data/lib/aireview/cli.rb

@@ -1,8 +1,9 @@
 # frozen_string_literal: true
 require 'securerandom'
+require_relative 'model_checker'
 
 module Aireview
-  class CLI
+  class CLI # rubocop:disable Metrics/ClassLength
     def self.start(argv, out: $stdout, err: $stderr, env: ENV)
       new(argv, out: out, err: err, env: env).start
     end
@@ -24,6 +25,8 @@ module Aireview
       case command
       when 'review'
         run_review(@argv)
+      when 'models'
+        run_models(@argv)
       when '--help', '-h', nil
         @out.puts(help)
         0
@@ -41,6 +44,33 @@ module Aireview
 
     private
 
+    # aireview models check [--config PATH] [--verbose]: a probe request with
+    # the production schemas to every model of the chains; see ModelChecker.
+    def run_models(argv)
+      options = parse_models_options(argv)
+      raise ParseError, "Usage: aireview models check [options] (got: #{argv.join(' ')})" unless argv == ['check']
+
+      @logger.level = Logger::DEBUG if options[:verbose]
+      config = Config.load(config_path: options[:config], cwd: Dir.pwd, env: @env, logger: @logger)
+      config.warnings.each { |warning| @logger.warn(warning) }
+      ModelChecker.new(config: config, out: @out, logger: @logger, strict: options[:strict] == true).run
+    end
+
+    def parse_models_options(argv)
+      options = {}
+      OptionParser.new do |parser|
+        parser.banner = 'Usage: aireview models check [options]'
+        parser.on('--config PATH', 'Path to .aireview.yml') { |value| options[:config] = value }
+        parser.on('--strict', 'Treat a skipped model (unreachable Ollama) as a failure') { options[:strict] = true }
+        parser.on('--verbose', 'Enable debug logging') { options[:verbose] = true }
+        parser.on('-h', '--help', 'Show help') do
+          @out.puts(parser)
+          raise Aireview::HelpRequested
+        end
+      end.parse!(argv)
+      options
+    end
+
     def run_review(argv)
       options, mr_url = review_options_and_url(argv)
       parser_result = MrParser.parse(mr_url)
@@ -68,9 +98,11 @@ module Aireview
         generate_model: options[:generate_model],
         critique_model: options[:critique_model],
         generate_temperature: options[:generate_temperature],
-        critique_temperature: options[:critique_temperature]
+        critique_temperature: options[:critique_temperature],
+        no_fallbacks: options[:no_fallbacks] == true
       )
       config.require_llm_configuration!
+      config.warnings.each { |warning| @logger.warn(warning) }
       config
     end
 
@@ -102,8 +134,8 @@ module Aireview
       [merge_request, changes]
     end
 
-    # Дифф уходит дальше по файлам, а не одной строкой: бюджет контекста
-    # режет его по границам файлов и хунков.
+    # The diff travels on as files, not as one string: the context budget
+    # cuts it at file and hunk boundaries.
     def prepare_changes(changes, config)
       diff_fetcher = DiffFetcher.new(ignore_paths: config.ignore_paths, logger: @logger)
       filtered_changes = diff_fetcher.filter(changes)
@@ -140,8 +172,8 @@ module Aireview
         critique: !options[:no_critique]
       )
 
-      # Печатаем до публикации: если публикация не состоится, текст ревью
-      # останется хотя бы в логе джоба.
+      # Printed before publishing: if publishing fails, the review text at
+      # least stays in the job log.
       @out.puts(review)
 
       publish_review(review, context, publication) if publication
@@ -149,8 +181,8 @@ module Aireview
       0
     end
 
-    # Поиск прошлого ревью идёт до вызова LLM: иначе запросы тратятся впустую,
-    # даже когда публиковать нечего.
+    # The previous review is looked up before calling the LLM: otherwise the
+    # requests are wasted even when there is nothing to publish.
     def prepare_publication(pipeline, config, context, options)
       return nil unless options[:post]
 
@@ -204,10 +236,10 @@ module Aireview
       )
     end
 
-    # Пока работала LLM, MR мог уехать: новый коммит, перебазирование или смена
-    # целевой ветки. Публиковать ревью неактуального диффа хуже, чем не
-    # публиковать ничего, а ошибку проверки нельзя трактовать как «всё на
-    # месте», поэтому её не глушим.
+    # While the LLM was working the MR may have moved on: a new commit, a
+    # rebase or a target branch change. Publishing a review of a stale diff
+    # is worse than publishing nothing, and a failed check cannot be read as
+    # "all in place", so it is not swallowed.
     def merge_request_moved?(context)
       current = context[:gitlab_client].fetch_merge_request(
         context[:parser_result].project_id,
@@ -280,6 +312,10 @@ module Aireview
       parser.on('--no-critique', 'Skip critique pass and render Generate candidates directly') do
         options[:no_critique] = true
       end
+
+      parser.on('--no-fallbacks', 'Use only the primary model and the first API key of each stage') do
+        options[:no_fallbacks] = true
+      end
     end
 
     def add_publication_options(parser, options)
@@ -328,9 +364,12 @@ module Aireview
       <<~HELP
         Usage:
           aireview review <merge_request_url> [options]
+          aireview models check [--config PATH] [--strict] [--verbose]
 
         Commands:
-          review    Run review for a GitLab merge request URL
+          review        Run review for a GitLab merge request URL
+          models check  Send a probe request with the generate and critique schemas
+                        to every model of both stages; exit 1 if any fails
 
         Options:
           --post           Post review as a merge request note
@@ -349,6 +388,7 @@ module Aireview
           --no-jira        Disable Jira enrichment
           --dry-run        Print prompts without LLM calls
           --no-critique    Skip second LLM critique pass
+          --no-fallbacks   Use only the primary model and the first API key of each stage
           --verbose        Enable verbose logging
           -h, --help       Show help
       HELP

data/lib/aireview/config.rb

@@ -5,11 +5,18 @@ require 'yaml'
 require_relative 'errors'
 require_relative 'utils'
 require_relative 'config_limits'
+require_relative 'config_fallbacks'
+require_relative 'config_layers'
+require_relative 'config_loader'
 
 module Aireview
+  # Answers questions about the merged settings: values, their source
+  # (layer), the routing plan, provider keys. How settings are read from
+  # files and the environment is ConfigLoader's business.
   class Config
     include ConfigLimits
-    extend ConfigLimits::ClassMethods
+    include ConfigFallbacks
+    include ConfigLayers
 
     DEFAULT_SECRET_FILES = [
       '.env',
@@ -39,171 +46,57 @@ module Aireview
         'provider' => 'gemini',
         'temperature' => 0,
         'timeout' => 60,
-        'max_prompt_chars' => ConfigLimits::DEFAULT_MAX_PROMPT_CHARS
+        'max_prompt_chars' => ConfigLimits::DEFAULT_MAX_PROMPT_CHARS,
+        'time_budget' => ConfigFallbacks::DEFAULT_TIME_BUDGET,
+        'overloaded_quarantine' => ConfigFallbacks::DEFAULT_OVERLOADED_QUARANTINE
       },
       'context' => ConfigLimits::CONTEXT_DEFAULTS
     }.freeze
 
-    ENV_MAPPING = {
-      'gitlab_url' => 'GITLAB_URL',
-      'gitlab_token' => 'GITLAB_TOKEN',
-      'jira_url' => 'JIRA_URL',
-      'jira_login' => 'JIRA_LOGIN',
-      'jira_password' => 'JIRA_PASSWORD',
-      'review_language' => 'REVIEW_LANGUAGE',
-      'review_mode' => 'REVIEW_MODE',
-      'llm_api_base' => 'LLM_API_BASE',
-      'ollama_api_base' => 'OLLAMA_API_BASE',
-      'llm_http_proxy' => 'LLM_HTTP_PROXY'
-    }.freeze
-
-    PROVIDER_KEY_MAPPING = {
-      'gemini' => 'GEMINI_API_KEY',
-      'ollama' => nil
-    }.freeze
-
-    attr_reader :config_path
+    attr_reader :config_path, :layers
 
-    def self.load(config_path: nil, cwd: Dir.pwd, env: ENV, logger: Logger.new($stderr))
-      load_dotenv(cwd)
-
-      file_path = config_path ? File.expand_path(config_path, cwd) : discover_file(cwd, '.aireview.yml')
-      file_config = File.file?(file_path) ? normalize_hash(YAML.load_file(file_path) || {}) : {}
-
-      merged = deep_merge(DEFAULTS, file_config)
-      merged = deep_merge(merged, env_config(env))
-
-      new(merged, config_path: File.file?(file_path) ? file_path : nil, logger: logger)
+    def self.load(**options)
+      ConfigLoader.load(**options)
     end
 
-    def self.load_dotenv(cwd)
-      require 'dotenv'
-      dotenv_path = discover_file(cwd, '.env')
-      Dotenv.load(dotenv_path) if File.file?(dotenv_path)
-    rescue LoadError
-      nil
-    end
-
-    def self.discover_file(cwd, basename)
-      current = Pathname.new(cwd).expand_path
-
-      loop do
-        candidate = current.join(basename)
-        return candidate.to_s if candidate.file?
-
-        break if current.root?
-
-        current = current.parent
-      end
-
-      File.join(cwd, basename)
-    end
-
-    def self.env_config(env)
-      mapped_env_config(env)
-        .merge('llm' => llm_env_config(env))
-        .merge(context_env_config(env))
-        .merge(provider_key_env_config(env))
-        .merge(generic_api_key_env_config(env))
-    end
-
-    def self.mapped_env_config(env)
-      ENV_MAPPING.each_with_object({}) do |(key, env_key), config|
-        value = env[env_key]
-        config[key] = value unless Aireview::Utils.blank?(value)
-      end
-    end
-
-    def self.llm_env_config(env)
-      {
-        'provider' => env['LLM_PROVIDER'],
-        'temperature' => parse_float(env['LLM_TEMPERATURE']),
-        'timeout' => parse_float(env['LLM_TIMEOUT']),
-        'max_prompt_chars' => parse_integer(env['LLM_MAX_PROMPT_CHARS'], 'LLM_MAX_PROMPT_CHARS'),
-        'generate' => llm_stage_env_config(env, 'GENERATE'),
-        'critique' => llm_stage_env_config(env, 'CRITIQUE')
-      }.compact.reject { |key, value| %w[generate critique].include?(key) && value.empty? }
-    end
-
-    def self.llm_stage_env_config(env, stage)
-      {
-        'provider' => env["LLM_#{stage}_PROVIDER"],
-        'model' => env["LLM_#{stage}_MODEL"],
-        'temperature' => parse_float(env["LLM_#{stage}_TEMPERATURE"]),
-        'max_prompt_chars' => parse_integer(env["LLM_#{stage}_MAX_PROMPT_CHARS"], "LLM_#{stage}_MAX_PROMPT_CHARS")
-      }.compact
-    end
-
-    def self.provider_key_env_config(env)
-      PROVIDER_KEY_MAPPING.each_with_object({}) do |(provider, env_key), config|
-        next unless env_key
-
-        value = env[env_key]
-        config["#{provider}_api_key"] = value unless Aireview::Utils.blank?(value)
-      end
+    def self.env_names
+      ConfigLoader.env_names
     end
 
-    def self.generic_api_key_env_config(env)
-      api_key = env['LLM_API_KEY']
-      return {} if Aireview::Utils.blank?(api_key)
-
-      {'llm_api_key' => api_key}
-    end
-
-    def self.parse_float(value)
-      return nil if Aireview::Utils.blank?(value)
-
-      Float(value)
-    rescue ArgumentError
-      nil
-    end
-
-    def self.deep_merge(left, right)
-      left.merge(right) do |_, old_value, new_value|
-        if old_value.is_a?(Hash) && new_value.is_a?(Hash)
-          deep_merge(old_value, new_value)
-        else
-          new_value
-        end
-      end
-    end
-
-    def self.normalize_hash(value)
-      case value
-      when Hash
-        value.each_with_object({}) do |(key, inner_value), result|
-          result[key.to_s] = normalize_hash(inner_value)
-        end
-      when Array
-        value.map { |item| normalize_hash(item) }
-      else
-        value
-      end
-    end
-
-    def initialize(data, config_path:, logger:)
-      @data = self.class.normalize_hash(data)
+    # The layers are the single source of truth: the merged data is computed
+    # from them. A config built from a hash (without ConfigLoader) is one
+    # layer, otherwise the CLI layer from with_overrides would be the only
+    # one and the stage settings of the original hash would be lost.
+    def initialize(data = nil, config_path: nil, logger: Logger.new($stderr), layers: nil)
+      @layers = layers || [ConfigLayers::Layer.new(name: ConfigLayers::DATA_LAYER, data: Utils.normalize_hash(data))]
+      @data = @layers.map(&:data).reduce({}) { |merged, layer_data| Utils.deep_merge(merged, layer_data) }
       @config_path = config_path
       @logger = logger
     end
 
+    # CLI overrides change only the primary model of a stage, the reserves
+    # from the config stay; no_fallbacks leaves one model and one key.
     def with_overrides(
       generate_model: nil,
       critique_model: nil,
       generate_temperature: nil,
-      critique_temperature: nil
+      critique_temperature: nil,
+      no_fallbacks: false
     )
       llm_config = {
         'generate' => stage_overrides(model: generate_model, temperature: generate_temperature),
         'critique' => stage_overrides(model: critique_model, temperature: critique_temperature)
       }.reject { |_, overrides| overrides.empty? }
-      return self if llm_config.empty?
-
-      merged = self.class.deep_merge(
-        @data,
-        'llm' => llm_config
+      overrides = {}
+      overrides['llm'] = llm_config unless llm_config.empty?
+      overrides['fallbacks_disabled'] = true if no_fallbacks
+      return self if overrides.empty?
+
+      self.class.new(
+        config_path: config_path,
+        logger: @logger,
+        layers: @layers + [ConfigLayers::Layer.new(name: ConfigLayers::CLI_LAYER, data: overrides)]
       )
-      self.class.new(merged, config_path: config_path, logger: @logger)
     end
 
     def gitlab_url
@@ -238,28 +131,42 @@ module Aireview
       dig('llm', 'timeout') || DEFAULTS.dig('llm', 'timeout')
     end
 
+    # The primary model of a stage is the first in its chain: the start for
+    # Generate, the first of the pool for Critique. These go into the review key.
     def generate_model
-      dig('llm', 'generate', 'model')
+      routing.primary('generate').model
     end
 
     def critique_model
-      dig('llm', 'critique', 'model')
+      routing.primary('critique').model
     end
 
     def generate_provider
-      dig('llm', 'generate', 'provider') || llm_provider
+      routing.primary('generate').provider
+    end
+
+    # Everything besides the prompt that affects the review result goes into
+    # the note key (see ReviewMarker): provider, model and temperature of the
+    # stages, the shared pool with its critique policy. Reserves of per-stage
+    # chains do not change the result.
+    def result_signature
+      {
+        'generate' => [generate_provider, generate_model, generate_temperature],
+        'critique' => [critique_provider, critique_model, critique_temperature],
+        'pool' => routing.signature
+      }
     end
 
     def critique_provider
-      dig('llm', 'critique', 'provider') || llm_provider
+      routing.primary('critique').provider
     end
 
     def generate_temperature
-      dig('llm', 'generate', 'temperature') || llm_temperature
+      stage_setting('generate', 'temperature') || DEFAULTS.dig('llm', 'temperature')
     end
 
     def critique_temperature
-      dig('llm', 'critique', 'temperature') || llm_temperature
+      stage_setting('critique', 'temperature') || DEFAULTS.dig('llm', 'temperature')
     end
 
     def llm_api_base
@@ -278,8 +185,8 @@ module Aireview
       @data['review_language'] || DEFAULTS['review_language']
     end
 
-    # update — обновляем свою заметку, когда дифф или настройки изменились,
-    # once — ревьюим один раз автоматически; Retry джоба обновляет ревью при изменениях.
+    # update — our note is updated when the diff or the settings changed,
+    # once — one automatic review; a job Retry updates the review on changes.
     def review_mode
       mode = (@data['review_mode'] || DEFAULTS['review_mode']).to_s
       return mode if REVIEW_MODES.include?(mode)
@@ -324,41 +231,29 @@ module Aireview
       raise ConfigError, 'GITLAB_TOKEN is required'
     end
 
-    def require_models!
-      missing = []
-      missing << 'llm.generate.model (or LLM_GENERATE_MODEL)' if Aireview::Utils.blank?(generate_model)
-      missing << 'llm.critique.model (or LLM_CRITIQUE_MODEL)' if Aireview::Utils.blank?(critique_model)
-      raise ConfigError, "LLM models are required: #{missing.join(', ')}" unless missing.empty?
-    end
+    private
 
-    def require_llm_configuration!
-      require_models!
+    # Without a pool an override changes only the primary model of the stage,
+    # the reserves stay. With a pool the stage mode switches explicitly: a
+    # model from the pool becomes the start and the stage's own model and
+    # fallbacks are reset; a model outside the pool is a single chain, start
+    # and fallbacks are reset.
+    def stage_overrides(model:, temperature:)
+      overrides = {'temperature' => temperature}.compact
+      return overrides unless model
 
-      missing_keys = {
-        'generate' => generate_provider,
-        'critique' => critique_provider
-      }.filter_map do |stage, provider|
-        next if provider.to_s == 'ollama'
-        next if Aireview::Utils.present?(provider_api_key(provider))
+      items = Array(dig('llm', 'models'))
+      return overrides.merge('model' => model) if items.empty?
 
-        "#{stage}: API key is required for provider #{provider.inspect}"
+      if ModelPool.member?(items, llm_provider, model)
+        overrides.merge('start' => model, 'model' => nil, 'fallbacks' => nil)
+      else
+        overrides.merge('start' => nil, 'model' => model, 'fallbacks' => [])
       end
-      raise ConfigError, missing_keys.join(', ') unless missing_keys.empty?
-    end
-
-    private
-
-    def stage_overrides(model:, temperature:)
-      {
-        'model' => model,
-        'temperature' => temperature
-      }.compact
     end
 
     def dig(*keys)
-      keys.reduce(@data) do |accumulator, key|
-        accumulator.is_a?(Hash) ? accumulator[key] : nil
-      end
+      Utils.dig(@data, *keys)
     end
   end
 end