aireview 0.2.1 → 2.0.0

data/lib/aireview/model_pool.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+require_relative 'errors'
+require_relative 'utils'
+require_relative 'model_candidate'
+require_relative 'stage_chains'
+
+module Aireview
+  # A routing plan from a shared pool: models in order of priority (the
+  # first is the preferred one for Critique). Generate walks the pool from
+  # its start model downwards and round again; Critique takes the first
+  # live model not below the one that answered in Generate
+  # (rank: not_below_generate), self-critique by the same model is the last
+  # permitted option, lower only with allow_weaker. A stage with a model of
+  # its own does not use the pool: its chain is independent and the rank
+  # rule does not apply, as with rank: any.
+  class ModelPool
+    CRITIQUE_RANKS = %w[not_below_generate any].freeze
+    DEFAULT_CRITIQUE_RANK = 'not_below_generate'
+
+    attr_reader :warnings
+
+    # Whether a model is in the raw llm.models list — without building the
+    # plan, so that a CLI override does not trip over an invalid start of
+    # the old plan.
+    def self.member?(items, provider, model)
+      return false if Aireview::Utils.blank?(model)
+
+      Array(items).each_with_index.any? do |item, index|
+        parsed = parse_item(item, index, provider)
+        parsed[:model] == model.to_s || "#{parsed[:provider]}/#{parsed[:model]}" == model.to_s
+      end
+    end
+
+    def self.parse_item(item, index, provider)
+      item = ModelCandidate.parse_item(item)
+      name = "llm.models[#{index}]"
+      raise ConfigError, "#{name} must be a model name or a hash with model" unless item.is_a?(Hash)
+      raise ConfigError, "#{name}.model is required" if Aireview::Utils.blank?(item['model'])
+
+      limit = item['max_prompt_chars']
+      {
+        provider: (item['provider'] || provider).to_s,
+        model: item['model'].to_s,
+        max_prompt_chars: limit.nil? ? nil : StageChains.positive_limit(limit, name)
+      }
+    end
+
+    # items — the raw llm.models; starts — the start per stage (a model name
+    # or nil); inherited_starts — stages whose start came from a layer below
+    # the pool (image defaults versus the project's LLM_MODELS): such a start
+    # missing from the pool is replaced by the first model with a warning,
+    # an explicit start outside the pool is a configuration error;
+    # own_chains — stages with a chain of their own.
+    # Nine named settings read better than a struct for its own sake.
+    def initialize(items:, provider:, limits:, starts: {}, inherited_starts: [], rank: nil, allow_weaker: false, # rubocop:disable Metrics/ParameterLists
+                   own_chains: nil, only_primary: false)
+      @items = parse_items(items, provider)
+      @limits = limits.transform_keys(&:to_s)
+      @rank = validate_rank(rank)
+      @allow_weaker = allow_weaker == true
+      @own_chains = own_chains || StageChains.new({})
+      @only_primary = only_primary
+      @warnings = []
+      @starts = resolve_starts(starts.transform_keys(&:to_s), inherited_starts.map(&:to_s))
+    end
+
+    def pool(stage = 'generate')
+      limit = @limits.fetch(stage.to_s)
+      @items.map do |item|
+        ModelCandidate.new(provider: item[:provider], model: item[:model],
+                           max_prompt_chars: item[:max_prompt_chars] || limit)
+      end
+    end
+
+    def chain(stage)
+      stage = stage.to_s
+      return @own_chains.chain(stage) unless pool_stage?(stage)
+
+      trim(pool(stage).rotate(index_of(@starts[stage] || @items.first[:model])))
+    end
+
+    # The models not below the one that answered in Generate, in pool order
+    # (including that one), then — only with allow_weaker — the rest. An
+    # explicit critique start goes first only when it is permitted itself:
+    # Generate walks round the pool and may answer with a model above its
+    # own start, so this cannot be checked statically; a start that is not
+    # permitted does not bypass the ban on a weaker critique, it is skipped.
+    def critique_chain(after:)
+      return chain('critique') unless rank_applies?(after)
+
+      models = pool('critique')
+      limit = index_of(after)
+      allowed = models.first(limit + 1)
+      allowed += models.drop(limit + 1) if @allow_weaker
+      trim(with_start_first(allowed, @starts['critique']))
+    end
+
+    def primary(stage)
+      chain(stage).first
+    end
+
+    def weaker?(critique_candidate, generate_candidate)
+      return false unless pool_member?(critique_candidate) && pool_member?(generate_candidate)
+
+      index_of(critique_candidate) > index_of(generate_candidate)
+    end
+
+    # The order and policy of the pool go into the review key: they decide
+    # which model checks the findings. nil when a stage is outside the pool.
+    def signature
+      return nil unless both_stages_in_pool?
+
+      {
+        'models' => pool.map(&:to_s),
+        'generate_start' => @starts['generate'],
+        'critique_start' => @starts['critique'],
+        'rank' => @rank,
+        'allow_weaker' => @allow_weaker
+      }
+    end
+
+    # The critique selection rule in words, for --dry-run.
+    def rule
+      return nil unless both_stages_in_pool?
+      return @rank if @rank == 'any' || !@allow_weaker
+
+      "#{@rank}, weaker allowed"
+    end
+
+    def pool?
+      true
+    end
+
+    def pool_stage?(stage)
+      !@own_chains.stage?(stage)
+    end
+
+    def pool_member?(model)
+      return false if model.nil?
+
+      @items.any? { |item| match?(item, model) }
+    end
+
+    def start_used?(stage)
+      !@starts[stage.to_s].nil?
+    end
+
+    private
+
+    def both_stages_in_pool?
+      STAGES.all? { |stage| pool_stage?(stage) }
+    end
+
+    def rank_applies?(after)
+      both_stages_in_pool? && @rank != 'any' && pool_member?(after)
+    end
+
+    def trim(chain)
+      @only_primary ? chain.first(1) : chain
+    end
+
+    def with_start_first(chain, start)
+      return chain unless start
+
+      head = chain.find { |candidate| match_candidate?(candidate, start) }
+      unless head
+        @warnings << "llm.critique.start #{start} is below the model that answered in generate and allow_weaker " \
+                     'is off, ignoring it'
+        return chain
+      end
+
+      [head, *chain.reject { |candidate| candidate.equal?(head) }]
+    end
+
+    # The start of a stage with its own chain is not checked: it is outside the pool.
+    def resolve_starts(starts, inherited)
+      STAGES.to_h do |stage|
+        start = starts[stage]
+        next [stage, nil] if Aireview::Utils.blank?(start) || !pool_stage?(stage)
+        next [stage, start] if pool_member?(start)
+        raise ConfigError, "#{start} is not in llm.models: #{pool.join(', ')}" unless inherited.include?(stage)
+
+        @warnings << "llm.#{stage}.start #{start} is not in the overriding llm.models, starting from #{pool.first}"
+        [stage, nil]
+      end
+    end
+
+    def index_of(model)
+      index = @items.index { |item| match?(item, model) }
+      return index if index
+
+      raise ConfigError, "#{model} is not in llm.models: #{pool.join(', ')}"
+    end
+
+    # A model is given by name or as "provider/name"; a candidate matches by provider and name.
+    def match?(item, model)
+      return "#{item[:provider]}/#{item[:model]}" == model.to_s if model.is_a?(ModelCandidate)
+
+      item[:model] == model.to_s || "#{item[:provider]}/#{item[:model]}" == model.to_s
+    end
+
+    def match_candidate?(candidate, model)
+      candidate.model == model.to_s || candidate.to_s == model.to_s
+    end
+
+    def validate_rank(rank)
+      rank = (rank || DEFAULT_CRITIQUE_RANK).to_s
+      return rank if CRITIQUE_RANKS.include?(rank)
+
+      raise ConfigError, "llm.critique.rank must be one of #{CRITIQUE_RANKS.join(', ')}, got #{rank.inspect}"
+    end
+
+    def parse_items(items, provider)
+      parsed = Array(items).each_with_index.map { |item, index| self.class.parse_item(item, index, provider) }
+      raise ConfigError, 'llm.models must not be empty' if parsed.empty?
+
+      names = parsed.map { |item| "#{item[:provider]}/#{item[:model]}" }
+      duplicates = names.tally.select { |_, count| count > 1 }.keys
+      raise ConfigError, "llm.models has duplicates: #{duplicates.join(', ')}" unless duplicates.empty?
+
+      parsed
+    end
+  end
+end

data/lib/aireview/model_state.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+require 'set'
+
+module Aireview
+  # What the router learned about a model during the run. Three things with
+  # different lifetimes: sent requests are counted per stage, a quarantine
+  # lasts until a moment in time, an exclusion lasts until the end of the
+  # run (the provider has no such model, every key is out of quota) or the
+  # end of the stage (an invalid result). Keys with an exhausted daily quota
+  # are remembered separately: a quota is a property of "key + model", a
+  # quarantine a property of the model.
+  class ModelState
+    # Every request sent, the short retry and the JSON repair included,
+    # spends an attempt regardless of its outcome; the keys of a model share
+    # one counter, a quarantine does not reset it.
+    MAX_REQUESTS_PER_STAGE = 3
+
+    attr_reader :excluded_reason
+
+    def initialize(limit: MAX_REQUESTS_PER_STAGE)
+      @limit = limit
+      @sent = Hash.new(0)
+      @quarantined_until = nil
+      @excluded_reason = nil
+      @stage_exclusions = {}
+      @exhausted_keys = Set.new
+    end
+
+    def sent(stage)
+      @sent[stage.to_s]
+    end
+
+    def record_request(stage)
+      @sent[stage.to_s] += 1
+    end
+
+    def requests_left?(stage)
+      sent(stage) < @limit
+    end
+
+    def quarantine(until_time)
+      @quarantined_until = until_time
+    end
+
+    # A model that answered is not overloaded, whichever key answered.
+    def lift_quarantine
+      @quarantined_until = nil
+    end
+
+    def quarantine_left(now)
+      return 0 unless @quarantined_until
+
+      [@quarantined_until - now, 0].max
+    end
+
+    def exclude(reason)
+      @excluded_reason = reason
+    end
+
+    def exclude_for_stage(stage, reason)
+      @stage_exclusions[stage.to_s] = reason
+    end
+
+    def exhaust_key(key_index)
+      @exhausted_keys << key_index
+    end
+
+    def key_exhausted?(key_index)
+      @exhausted_keys.include?(key_index)
+    end
+
+    # Why the model cannot be tried in the stage; nil — it can (the
+    # quarantine is checked separately: it is a wait, not a ban).
+    def skip_reason(stage)
+      return @excluded_reason if @excluded_reason
+      return "excluded for this stage: #{@stage_exclusions[stage.to_s]}" if @stage_exclusions.key?(stage.to_s)
+      return "attempt limit of #{@limit} reached" unless requests_left?(stage)
+
+      nil
+    end
+  end
+end

data/lib/aireview/prompts/critique.txt

@@ -48,6 +48,13 @@ and reject the rest as duplicates.
 
 Do not add new findings. Do not change id.
 Do not change file, line, quoted_code. They do not need to be reinvented.
+
+A candidate may carry a note field: the result of a mechanical check of its
+link to the diff (the quote was not found in the shown diff, the line was
+reset to null, the file was shown partially). It means the link could not be
+confirmed, not that the finding is made up; take the note into account when
+deciding keep/reject and check such a candidate against the diff more
+carefully.
 Use refinement only when it makes a keep finding more precise.
 
 The answer must be a valid JSON object only, without markdown and without any

data/lib/aireview/publisher.rb

@@ -10,9 +10,9 @@ module Aireview
       @logger = logger
     end
 
-    # Собственная заметка с ревью: {id:, key:} или nil. Метки недостаточно —
-    # её может процитировать кто угодно, поэтому сверяем ещё и автора. Старый
-    # формат без метки подхватываем только если заметки с меткой нет.
+    # Our own review note: {id:, key:} or nil. The marker alone is not
+    # enough — anyone can quote it, so the author is checked too. The old
+    # format without a marker is picked up only when no marked note exists.
     def existing_review(project_id:, iid:)
       author_id = current_user_id
       legacy = nil
@@ -49,9 +49,9 @@ module Aireview
       "#{ReviewMarker.build(key)}\n#{PREFIX}\n\n#{review_body}"
     end
 
-    # Без надёжного автора матчинг по одной метке небезопасен: процитировать её
-    # может кто угодно, и тогда чужая заметка либо отменит ревью, либо будет
-    # перезаписана. Поэтому ошибку не глушим.
+    # Without a reliable author, matching by the marker alone is unsafe:
+    # anyone can quote it, and then someone else's note would either cancel
+    # the review or be overwritten. So the error is not swallowed.
     def current_user_id
       return @current_user_id if defined?(@current_user_id)
 

data/lib/aireview/result_parser.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+require 'json'
+require_relative 'utils'
+
+module Aireview
+  # Parses and validates the shape of an LLM answer: JSON (code fences
+  # included) or a ready structure by the schema → a hash with string keys
+  # and checked ids. A shape error is SchemaError; whether to repair the
+  # answer with another request is the pipeline's decision. The same parser
+  # lets `aireview models check` judge whether a model holds the schema.
+  class ResultParser
+    class SchemaError < StandardError
+    end
+
+    DECISIONS = %w[keep reject].freeze
+
+    # expected — :generate or :critique; critique_candidate_ids — the
+    # candidate ids Critique must give a verdict on, each of them.
+    def parse(raw, expected:, critique_candidate_ids: nil)
+      parsed = Utils.normalize_hash(raw.is_a?(Hash) ? raw : JSON.parse(strip_code_fences(raw.to_s)))
+
+      case expected.to_sym
+      when :generate then generate_result(parsed)
+      when :critique then critique_result(parsed, critique_candidate_ids)
+      else raise ArgumentError, "Unknown expected JSON schema: #{expected.inspect}"
+      end
+    end
+
+    private
+
+    def strip_code_fences(text)
+      stripped = text.strip
+      return stripped unless stripped.start_with?('```')
+
+      stripped
+        .sub(/\A```[[:alnum:]_-]*[ \t]*\r?\n?/, '')
+        .sub(/\r?\n?```[ \t]*\z/, '')
+        .strip
+    end
+
+    def generate_result(parsed)
+      parsed = {'summary' => nil, 'candidates' => parsed} if parsed.is_a?(Array)
+      valid_shape = parsed.is_a?(Hash) && parsed['candidates'].is_a?(Array)
+      raise SchemaError, 'expected an object with summary and candidates array' unless valid_shape
+      raise SchemaError, 'each generate candidate must be an object' unless parsed['candidates'].all?(Hash)
+
+      parsed['summary'] = nil unless parsed.key?('summary')
+      identifiers!(parsed['candidates'].map { |candidate| Utils.presence(candidate['id']) },
+                   missing: 'each generate candidate must include a non-empty id',
+                   duplicates: 'duplicate generate candidate ids')
+      parsed
+    end
+
+    def critique_result(parsed, expected_ids)
+      valid_shape = parsed.is_a?(Hash) && parsed['verdicts'].is_a?(Array)
+      raise SchemaError, 'expected an object with verdicts array' unless valid_shape
+      raise SchemaError, 'each critique verdict must be an object' unless parsed['verdicts'].all?(Hash)
+
+      verdict_ids = parsed['verdicts'].map { |verdict| Utils.presence(verdict['id']) }
+      identifiers!(verdict_ids, missing: 'each verdict must include a non-empty id',
+                                duplicates: 'duplicate verdict ids')
+      expected_verdict_ids!(verdict_ids, expected_ids)
+      parsed['verdicts'].each { |verdict| verdict!(verdict) }
+      parsed
+    end
+
+    def identifiers!(identifiers, missing:, duplicates:)
+      raise SchemaError, missing unless identifiers.all?
+
+      duplicate_ids = identifiers.tally.select { |_, count| count > 1 }.keys
+      raise SchemaError, "#{duplicates}: #{duplicate_ids.join(', ')}" unless duplicate_ids.empty?
+    end
+
+    def expected_verdict_ids!(verdict_ids, expected_ids)
+      return unless expected_ids
+
+      unknown_ids = verdict_ids - expected_ids
+      missing_ids = expected_ids - verdict_ids
+      raise SchemaError, "unknown verdict ids: #{unknown_ids.join(', ')}" unless unknown_ids.empty?
+      raise SchemaError, "missing verdict ids: #{missing_ids.join(', ')}" unless missing_ids.empty?
+    end
+
+    def verdict!(verdict)
+      id = Utils.presence(verdict['id'])
+      decision = verdict['decision'].to_s.strip.downcase
+      raise SchemaError, "invalid verdict decision for #{id}" unless DECISIONS.include?(decision)
+      if verdict.key?('refinement') && decision != 'keep'
+        raise SchemaError,
+              "reject verdict cannot include refinement for #{id}"
+      end
+
+      refinement = verdict['refinement']
+      return if refinement.nil? || refinement.is_a?(Hash)
+
+      raise SchemaError, "refinement must be an object for #{id}"
+    end
+  end
+end

data/lib/aireview/review_marker.rb

@@ -3,8 +3,8 @@ require 'digest'
 require 'json'
 
 module Aireview
-  # Скрытая метка в теле заметки: по ней ревью находит собственный комментарий
-  # и понимает, менялось ли с прошлого раза то, что влияет на результат.
+  # A hidden marker in the note body: it lets the review find its own
+  # comment and tell whether anything that affects the result changed.
   module ReviewMarker
     PATTERN = /<!--\s*aireview:key=([0-9a-f]+)\s*-->/
 
@@ -19,28 +19,27 @@ module Aireview
       match && match[1]
     end
 
-    # Ключ считается от готовых промптов, а не от одного SHA: так в него сами
-    # собой попадают дифф, описание MR, контекст Jira, инструкции ревью и
-    # ignore_paths. Модели и провайдеры добавляются рядом — на промпт они не
-    # влияют, но на результат влияют.
+    # The key is computed from the assembled prompts, not from a single SHA:
+    # that way the diff, the MR description, the Jira context, the review
+    # instructions and ignore_paths enter it by themselves. What else affects
+    # the result — provider, model and temperature of the stages, the shared
+    # pool with its critique policy — is known by Config#result_signature.
+    # Without a pool the key is the same as before.
     def key(prompts:, config:)
+      signature = config.result_signature
       source = {
-        'generate' => [
-          config.generate_provider,
-          prompts[:generate_model],
-          prompts[:generate_temperature],
-          prompts[:generate_prompt]
-        ],
-        'critique' => critique_source(prompts, config)
+        'generate' => [*signature['generate'], prompts[:generate_prompt]],
+        'critique' => prompts[:critique_prompt] ? [*signature['critique'], prompts[:critique_prompt]] : nil
       }
+      source['pool'] = signature['pool'] if signature['pool']
 
       Digest::SHA256.hexdigest(JSON.generate(source))[0, 16]
     end
 
-    # То, что делает результат ревью устаревшим: новый коммит, смена целевой
-    # ветки, перебазирование со сдвигом базы сравнения, а также правка
-    # заголовка или описания — из них берутся требования, с которыми ревью
-    # сверяет код.
+    # What makes a review result stale: a new commit, a target branch change,
+    # a rebase that moves the comparison base, and an edit of the title or
+    # the description — the requirements the review checks the code against
+    # come from those.
     def state(merge_request)
       {
         'sha' => merge_request['sha'],
@@ -50,16 +49,5 @@ module Aireview
         'description' => merge_request['description']
       }
     end
-
-    def critique_source(prompts, config)
-      return nil unless prompts[:critique_prompt]
-
-      [
-        config.critique_provider,
-        prompts[:critique_model],
-        prompts[:critique_temperature],
-        prompts[:critique_prompt]
-      ]
-    end
   end
 end