aireview 0.2.1 → 2.0.0

data/lib/aireview/review_pipeline.rb

@@ -1,15 +1,21 @@
 # frozen_string_literal: true
 require 'json'
 require_relative 'errors'
+require_relative 'stages'
 require_relative 'context_builder'
+require_relative 'candidate_checker'
+require_relative 'result_parser'
 require_relative 'review_renderer'
 require_relative 'review_schemas'
 require_relative 'reviewer'
 
 module Aireview
+  # A review run: context → Generate → anchoring check against the diff →
+  # Critique → report. Invalid JSON is repaired once by the same model; when
+  # the repair is invalid too, the stage restarts on another model with the
+  # original request.
   class ReviewPipeline
-    class SchemaError < StandardError
-    end
+    SchemaError = ResultParser::SchemaError
 
     REPAIR_SYSTEM_PROMPT = <<~PROMPT.strip.freeze
       You fix invalid JSON produced by another LLM call.
@@ -23,6 +29,7 @@ module Aireview
 
     def initialize(config:, reviewer: nil, context_builder: nil, logger: Logger.new($stderr))
       @config = config
+      @parser = ResultParser.new
       @reviewer = reviewer || Reviewer.new(config: config, logger: logger)
       @context_builder = context_builder || ContextBuilder.new(config: config, logger: logger)
       @logger = logger
@@ -37,29 +44,31 @@ module Aireview
       )
       generate_prompt = @context_builder.build_generate_prompt(context)
       @logger.info("Pipeline generate pass started (model=#{@config.generate_model})")
-      candidates_raw = @reviewer.generate(**generate_prompt)
-      generate_result = parse_with_repair(
-        raw: candidates_raw,
-        kind: 'generate result',
-        expected: :generate,
-        repair_stage: :generate
-      )
+      generate_result = run_stage('generate') do
+        parse_with_repair(
+          raw: @reviewer.generate(**generate_prompt),
+          kind: 'generate result',
+          expected: 'generate',
+          repair_stage: 'generate'
+        )
+      end
       summary = generate_result['summary']
       candidates = Array(generate_result['candidates'])
-      @logger.info("Pipeline generate pass completed with #{candidates.size} candidate(s)")
+      @logger.info("Pipeline generate pass completed with #{candidates.size} candidate(s) " \
+                   "(model=#{@reviewer.answered_model('generate')})")
+      candidates = check_candidates(context: context, changes: changes, candidates: candidates)
 
-      accepted = if critique
-                   @logger.info("Pipeline critique pass started (model=#{@config.critique_model})")
-                   critique_candidates(context: context, candidates: candidates)
-                 else
-                   @logger.info('Pipeline critique pass skipped')
-                   candidates
-                 end
+      accepted = critique ? maybe_critique(context: context, candidates: candidates) : skip_critique(candidates)
 
       @logger.info("Pipeline finished with #{accepted.size} accepted finding(s)")
 
-      renderer = ReviewRenderer.new(language: @config.review_language)
-      renderer.render(accepted, summary: summary, coverage: context.coverage)
+      ReviewRenderer.new(language: @config.review_language).render(
+        accepted,
+        summary: summary,
+        coverage: context.coverage,
+        fallback_models: @reviewer.fallback_models,
+        critique_weaker: critique && @reviewer.critique_weaker?
+      )
     end
 
     def dry_run_prompts(merge_request:, changes:, jira_issue: nil, critique: true)
@@ -83,6 +92,15 @@ module Aireview
         generate_temperature: @config.generate_temperature,
         critique_model: @config.critique_model,
         critique_temperature: @config.critique_temperature,
+        generate_fallbacks: @config.fallback_names('generate'),
+        critique_fallbacks: critique ? @config.fallback_names('critique') : [],
+        sources: setting_sources(critique),
+        config_paths: @config.layer_paths,
+        warnings: @config.warnings,
+        critique_rule: critique ? @config.routing.rule : nil,
+        api_keys: @config.api_key_counts(critique ? STAGES : ['generate']),
+        time_budget: @config.llm_time_budget,
+        overloaded_quarantine: @config.overloaded_quarantine,
         coverage: context.coverage,
         sizes: context.sizes
       }
@@ -90,20 +108,76 @@ module Aireview
 
     private
 
+    # Where the model, provider and reserves of a stage came from, for --dry-run.
+    def setting_sources(critique)
+      (critique ? %w[generate critique] : %w[generate]).to_h do |stage|
+        [stage.to_sym, {
+          model: @config.stage_model_source(stage),
+          provider: @config.stage_provider_source(stage),
+          fallbacks: @config.stage_fallbacks_source(stage)
+        }]
+      end
+    end
+
+    # A stage is a request, parsing and one repair by the same model. An
+    # invalid result after the repair, like a repair with no requests left,
+    # excludes the model for the stage, and the stage starts over on the next
+    # one — with the original request. API errors pass through: the router
+    # handles them with reserves, and exhausted routes are exhausted for a
+    # restart too.
+    def run_stage(stage)
+      loop do
+        return yield
+      rescue ParseError => e
+        reason = "#{e.is_a?(RepairImpossibleError) ? 'repair impossible' : 'invalid result'}: #{e.message}"
+        excluded = @reviewer.exclude_answered_model(stage: stage, reason: reason)
+        raise unless excluded
+
+        @logger.warn("Pipeline #{stage}: restarting on the next model after #{excluded} (#{e.message})")
+      end
+    end
+
+    # Anchoring is checked against the diff the model saw, before Critique:
+    # it gets the notes, the report gets the reset line and the missing-quote mark.
+    def check_candidates(context:, changes:, candidates:)
+      CandidateChecker.new(
+        changes: changes,
+        diff_text: context.diff_text,
+        coverage: context.coverage,
+        logger: @logger
+      ).check(candidates)
+    end
+
+    def skip_critique(candidates, reason = nil)
+      @logger.info(['Pipeline critique pass skipped', reason].compact.join(': '))
+      candidates
+    end
+
+    # Critique has nothing to filter without candidates: the LLM request
+    # would waste quota and time.
+    def maybe_critique(context:, candidates:)
+      return skip_critique(candidates, 'no candidates') if candidates.empty?
+
+      @logger.info("Pipeline critique pass started (model=#{@config.critique_model})")
+      critique_candidates(context: context, candidates: candidates)
+    end
+
     def critique_candidates(context:, candidates:)
       candidates_json = JSON.pretty_generate(candidates)
       candidates_by_id = index_candidates_by_id(candidates)
       critique_prompt = @context_builder.build_critique_prompt(context, candidates_json: candidates_json)
-      critique_raw = @reviewer.critique(**critique_prompt)
-      critique_result = parse_with_repair(
-        raw: critique_raw,
-        kind: 'critique result',
-        expected: :critique,
-        repair_stage: :critique,
-        critique_candidate_ids: candidates_by_id.keys
-      )
+      critique_result = run_stage('critique') do
+        parse_with_repair(
+          raw: @reviewer.critique(**critique_prompt),
+          kind: 'critique result',
+          expected: 'critique',
+          repair_stage: 'critique',
+          critique_candidate_ids: candidates_by_id.keys
+        )
+      end
       verdicts = Array(critique_result['verdicts'])
-      @logger.info("Pipeline critique pass completed with #{verdicts.size} verdict(s)")
+      @logger.info("Pipeline critique pass completed with #{verdicts.size} verdict(s) " \
+                   "(model=#{@reviewer.answered_model('critique')})")
       apply_critique_verdicts(
         verdicts: verdicts,
         candidates_by_id: candidates_by_id
@@ -156,108 +230,11 @@ module Aireview
     end
 
     def parse_expected_result(raw, expected, critique_candidate_ids: nil)
-      parsed = raw.is_a?(Hash) ? raw : JSON.parse(strip_code_fences(raw.to_s))
-
-      case expected
-      when :generate
-        parsed = normalize_generate_result(parsed)
-      when :critique
-        parsed = normalize_critique_result(parsed, critique_candidate_ids: critique_candidate_ids)
-      else
-        raise ArgumentError, "Unknown expected JSON schema: #{expected.inspect}"
-      end
-
-      parsed
-    end
-
-    def strip_code_fences(text)
-      stripped = text.to_s.strip
-      return stripped unless stripped.start_with?('```')
-
-      stripped
-        .sub(/\A```[[:alnum:]_-]*[ \t]*\r?\n?/, '')
-        .sub(/\r?\n?```[ \t]*\z/, '')
-        .strip
-    end
-
-    def normalize_generate_result(parsed)
-      parsed = {'summary' => nil, 'candidates' => parsed} if parsed.is_a?(Array)
-      validate_generate_result_shape!(parsed)
-      parsed['summary'] = nil unless parsed.key?('summary')
-      candidate_ids = parsed['candidates'].map { |candidate| normalize_id(value(candidate, 'id')) }
-      validate_identifiers!(
-        candidate_ids,
-        missing_message: 'each generate candidate must include a non-empty id',
-        duplicate_prefix: 'duplicate generate candidate ids'
-      )
-
-      parsed
-    end
-
-    def normalize_critique_result(parsed, critique_candidate_ids:)
-      validate_critique_result_shape!(parsed)
-      verdicts = parsed['verdicts']
-      verdict_ids = verdicts.map { |verdict| normalize_id(value(verdict, 'id')) }
-      validate_identifiers!(
-        verdict_ids,
-        missing_message: 'each verdict must include a non-empty id',
-        duplicate_prefix: 'duplicate verdict ids'
-      )
-      validate_expected_verdict_ids!(verdict_ids, critique_candidate_ids)
-      verdicts.each { |verdict| validate_verdict!(verdict) }
-
-      parsed
-    end
-
-    def validate_generate_result_shape!(parsed)
-      valid_shape = parsed.is_a?(Hash) && parsed['candidates'].is_a?(Array)
-      raise SchemaError, 'expected an object with summary and candidates array' unless valid_shape
-      raise SchemaError, 'each generate candidate must be an object' unless parsed['candidates'].all?(Hash)
-    end
-
-    def validate_critique_result_shape!(parsed)
-      valid_shape = parsed.is_a?(Hash) && parsed['verdicts'].is_a?(Array)
-      raise SchemaError, 'expected an object with verdicts array' unless valid_shape
-      raise SchemaError, 'each critique verdict must be an object' unless parsed['verdicts'].all?(Hash)
-    end
-
-    def validate_identifiers!(identifiers, missing_message:, duplicate_prefix:)
-      raise SchemaError, missing_message unless identifiers.all?
-
-      duplicate_ids = identifiers.group_by(&:itself).select { |_, ids| ids.size > 1 }.keys
-      return if duplicate_ids.empty?
-
-      raise SchemaError, "#{duplicate_prefix}: #{duplicate_ids.join(', ')}"
-    end
-
-    def validate_expected_verdict_ids!(verdict_ids, expected_ids)
-      return unless expected_ids
-
-      unknown_ids = verdict_ids - expected_ids
-      missing_ids = expected_ids - verdict_ids
-      raise SchemaError, "unknown verdict ids: #{unknown_ids.join(', ')}" unless unknown_ids.empty?
-      raise SchemaError, "missing verdict ids: #{missing_ids.join(', ')}" unless missing_ids.empty?
-    end
-
-    def validate_verdict!(verdict)
-      id = normalize_id(value(verdict, 'id'))
-      decision = normalize_decision(value(verdict, 'decision'))
-      raise SchemaError, "invalid verdict decision for #{id}" unless %w[keep reject].include?(decision)
-
-      refinement = value(verdict, 'refinement')
-      raise SchemaError, "reject verdict cannot include refinement for #{id}" if invalid_refinement?(verdict, decision)
-      return if refinement.nil?
-      return if refinement.is_a?(Hash)
-
-      raise SchemaError, "refinement must be an object for #{id}"
-    end
-
-    def invalid_refinement?(verdict, decision)
-      refinement_key?(verdict) && decision != 'keep'
+      @parser.parse(raw, expected: expected, critique_candidate_ids: critique_candidate_ids)
     end
 
     def repair_json(raw:, kind:, expected:, stage:, critique_candidate_ids: nil)
-      schema = expected == :critique ? ReviewSchemas.critique : ReviewSchemas.generate
+      schema = expected == 'critique' ? ReviewSchemas.critique : ReviewSchemas.generate
       user_prompt = <<~PROMPT
         The previous #{kind} response was invalid.
 
@@ -270,19 +247,28 @@ module Aireview
         Invalid response:
         #{raw}
       PROMPT
-      if stage == :critique && critique_candidate_ids
+      if stage == 'critique' && critique_candidate_ids
         user_prompt << "\nExpected candidate ids: #{critique_candidate_ids.join(', ')}\n"
       end
 
       @logger.info("Pipeline #{stage} repair started for #{kind}")
-      prompt = @context_builder.check_stage_size!(stage, REPAIR_SYSTEM_PROMPT, user_prompt)
-      if stage == :critique
-        @reviewer.critique(**prompt)
+      prompt = repair_prompt(stage, user_prompt)
+      if stage == 'critique'
+        @reviewer.critique(**prompt, pinned: true)
       else
-        @reviewer.generate(**prompt)
+        @reviewer.generate(**prompt, pinned: true)
       end
     end
 
+    # A repair that does not fit the stage limit is an invalid result of this
+    # model, not a size error of the original request: the stage moves to
+    # the next model with the original prompt.
+    def repair_prompt(stage, user_prompt)
+      @context_builder.check_stage_size!(stage, REPAIR_SYSTEM_PROMPT, user_prompt)
+    rescue ContextBudgetError => e
+      raise ParseError, "repair request does not fit the stage limit: #{e.message}"
+    end
+
     def index_candidates_by_id(candidates)
       candidates.each_with_object({}) do |candidate, result|
         next unless candidate.is_a?(Hash)
@@ -345,21 +331,14 @@ module Aireview
       hash[key] || hash[key.to_sym]
     end
 
-    def normalize_id(value)
-      presence(value)
-    end
-
     def normalize_decision(value)
       value.to_s.strip.downcase
     end
 
-    def refinement_key?(verdict)
-      verdict.key?('refinement') || verdict.key?(:refinement)
-    end
-
     def presence(value)
       string = value.to_s.strip
       string.empty? ? nil : string
     end
+    alias normalize_id presence
   end
 end

data/lib/aireview/review_renderer.rb

@@ -44,7 +44,10 @@ module Aireview
         sections_truncated: 'sections truncated',
         hunks_of: 'hunks shown',
         diff_unavailable: 'diff not available',
-        section_list: 'Truncated sections'
+        section_list: 'Truncated sections',
+        fallback_used: 'Fallback model used',
+        critique_weaker: 'Critique ran on a model weaker than Generate: the findings were checked less strictly.',
+        quote_missing: 'quote not found in the diff'
       },
       'ru' => {
         summary: 'Сводка',
@@ -68,7 +71,10 @@ module Aireview
         sections_truncated: 'секций усечено',
         hunks_of: 'хунков показано',
         diff_unavailable: 'дифф недоступен',
-        section_list: 'Усечённые секции'
+        section_list: 'Усечённые секции',
+        fallback_used: 'Использована резервная модель',
+        critique_weaker: 'Критика выполнена моделью слабее generate: замечания проверены менее строго.',
+        quote_missing: 'цитата не найдена в диффе'
       }
     }.freeze
 
@@ -76,14 +82,12 @@ module Aireview
       @labels = LABELS.fetch(language.to_s) { LABELS.fetch(DEFAULT_LANGUAGE) }
     end
 
-    # coverage: факты усечения контекста от пайплайна, не текст модели.
-    # result по-прежнему про найденные проблемы; неполнота покрытия
-    # дописывается рядом с ним, чтобы строка результата не читалась как
-    # «проверено всё».
-    def render(accepted, summary:, coverage: nil)
-      findings = sorted_findings(Array(accepted)).first(TOTAL_FINDINGS_LIMIT)
-      mismatches = findings.select { |finding| category(finding) == 'task_mismatch' }.first(MISMATCH_LIMIT)
-      important = findings.select { |finding| important_finding?(finding) }.first(IMPORTANT_LIMIT)
+    # coverage: the truncation facts from the pipeline, not the model's text.
+    # result is still about the findings; incomplete coverage is written
+    # next to it so that the result line does not read as "everything was
+    # checked".
+    def render(accepted, summary:, coverage: nil, fallback_models: {}, critique_weaker: false)
+      mismatches, important = select_findings(Array(accepted))
       result = mismatches.empty? && important.empty? ? 'ok' : 'needs attention'
 
       <<~MARKDOWN.rstrip
@@ -102,13 +106,25 @@ module Aireview
         ## #{label(:result)}
 
         #{result}#{partial_note(coverage)}
-        #{coverage_block(coverage)}
+        #{coverage_block(coverage)}#{fallback_note(fallback_models)}#{weaker_note(critique_weaker)}
         #{label(:disclaimer)}
       MARKDOWN
     end
 
     private
 
+    # First select what is shown at all, then the section limits and only
+    # then the total limit: a finding that cannot be shown because of its
+    # category or a section limit must not take a slot in the total.
+    def select_findings(accepted)
+      sorted = sorted_findings(accepted)
+      mismatches = sorted.select { |finding| category(finding) == 'task_mismatch' }.first(MISMATCH_LIMIT)
+      important = sorted.select { |finding| important_finding?(finding) }.first(IMPORTANT_LIMIT)
+      shown = sorted.select { |finding| mismatches.include?(finding) || important.include?(finding) }
+        .first(TOTAL_FINDINGS_LIMIT)
+      [mismatches & shown, important & shown]
+    end
+
     def sorted_findings(findings)
       findings
         .grep(Hash)
@@ -167,13 +183,28 @@ module Aireview
       "\n## #{label(:not_reviewed)}\n\n#{lines.join("\n")}\n"
     end
 
+    # A key switch stays in the logs; a model switch is visible to the
+    # reader, because a fallback model may review less well than the primary.
+    def weaker_note(critique_weaker)
+      return '' unless critique_weaker
+
+      "\n#{label(:critique_weaker)}\n"
+    end
+
+    def fallback_note(fallback_models)
+      return '' if fallback_models.nil? || fallback_models.empty?
+
+      used = fallback_models.map { |stage, model| "#{stage} — #{model}" }.join(', ')
+      "\n#{label(:fallback_used)}: #{used}.\n"
+    end
+
     def location(finding)
       file = presence(value(finding, 'file'))
       line = value(finding, 'line')
       return label(:not_specified) unless file
-      return file if line.nil? || line.to_s.empty?
 
-      "#{file}:#{line}"
+      location = line.to_s.empty? ? file : "#{file}:#{line}"
+      value(finding, 'quote_missing') ? "#{location} (#{label(:quote_missing)})" : location
     end
 
     def label(key)