aireview 2.1.0 → 2.2.1

data/lib/aireview/config_loader.rb

@@ -26,10 +26,21 @@ module Aireview
       'review_mode' => 'REVIEW_MODE',
       'llm_api_base' => 'LLM_API_BASE',
       'ollama_api_base' => 'OLLAMA_API_BASE',
-      'llm_http_proxy' => 'LLM_HTTP_PROXY'
+      'llm_http_proxy' => 'LLM_HTTP_PROXY',
+      'jev_api_key' => 'JEV_API_KEY'
+    }.freeze
+    PROVIDER_KEY_MAPPING = {
+      'gemini' => 'GEMINI_API_KEY',
+      'openai' => 'OPENAI_API_KEY',
+      'anthropic' => 'ANTHROPIC_API_KEY',
+      'openrouter' => 'OPENROUTER_API_KEY'
+    }.freeze
+    PROVIDER_KEYS_MAPPING = {
+      'gemini' => 'GEMINI_API_KEYS',
+      'openai' => 'OPENAI_API_KEYS',
+      'anthropic' => 'ANTHROPIC_API_KEYS',
+      'openrouter' => 'OPENROUTER_API_KEYS'
     }.freeze
-    PROVIDER_KEY_MAPPING = {'gemini' => 'GEMINI_API_KEY'}.freeze
-    PROVIDER_KEYS_MAPPING = {'gemini' => 'GEMINI_API_KEYS'}.freeze
     CONTEXT_ENV = {
       'max_diff_chars' => 'MAX_DIFF_CHARS',
       'max_mr_description_chars' => 'MAX_MR_DESCRIPTION_CHARS',
@@ -38,7 +49,8 @@ module Aireview
     }.freeze
     LLM_ENV = %w[
       LLM_PROVIDER LLM_TEMPERATURE LLM_TIMEOUT LLM_MAX_PROMPT_CHARS LLM_TIME_BUDGET LLM_OVERLOADED_QUARANTINE
-      LLM_MODELS LLM_CRITIQUE_RANK LLM_CRITIQUE_ALLOW_WEAKER
+      LLM_MODELS LLM_CRITIQUE_RANK LLM_CRITIQUE_ALLOW_WEAKER LLM_CRITIQUE_ENGINE
+      LLM_JEV_SHADOW LLM_JEV_MODEL LLM_JEV_FALLBACK LLM_JEV_KEEP_ABOVE
     ].freeze
     LLM_STAGE_ENV_SUFFIXES = %w[PROVIDER MODEL TEMPERATURE MAX_PROMPT_CHARS FALLBACK_MODEL START].freeze
     IMAGE_DEFAULTS_ENV = 'AIREVIEW_DEFAULTS'
@@ -141,8 +153,21 @@ module Aireview
         'time_budget' => parse_integer(env['LLM_TIME_BUDGET'], 'LLM_TIME_BUDGET'),
         'overloaded_quarantine' => parse_integer(env['LLM_OVERLOADED_QUARANTINE'], 'LLM_OVERLOADED_QUARANTINE'),
         'generate' => llm_stage_env_config(env, 'GENERATE'),
-        'critique' => llm_stage_env_config(env, 'CRITIQUE')
-      }.compact.reject { |key, value| %w[generate critique].include?(key) && value.empty? }
+        'critique' => llm_stage_env_config(env, 'CRITIQUE'),
+        'jev' => jev_env_config(env)
+      }.compact.reject { |key, value| %w[generate critique jev].include?(key) && value.empty? }
+    end
+
+    # LLM_JEV_SHADOW=true|false, LLM_JEV_MODEL — a pinned Jev version,
+    # LLM_JEV_FALLBACK=model|fail, LLM_JEV_KEEP_ABOVE — the keep threshold.
+    def jev_env_config(env)
+      {
+        'shadow' => parse_boolean(env['LLM_JEV_SHADOW'], 'LLM_JEV_SHADOW'),
+        'model' => Aireview::Utils.presence(env['LLM_JEV_MODEL']),
+        'fallback' => Aireview::Utils.presence(env['LLM_JEV_FALLBACK']),
+        # Validated by ConfigJev: a typo must fail, not fall back to the default.
+        'keep_above' => Aireview::Utils.presence(env['LLM_JEV_KEEP_ABOVE'])
+      }.compact
     end
 
     def llm_stage_env_config(env, stage)
@@ -176,6 +201,7 @@ module Aireview
         'generate' => {'start' => env['LLM_GENERATE_START']}.compact,
         'critique' => {
           'start' => env['LLM_CRITIQUE_START'],
+          'engine' => Aireview::Utils.presence(env['LLM_CRITIQUE_ENGINE']),
           'rank' => env['LLM_CRITIQUE_RANK'],
           'allow_weaker' => parse_boolean(env['LLM_CRITIQUE_ALLOW_WEAKER'], 'LLM_CRITIQUE_ALLOW_WEAKER')
         }.compact

data/lib/aireview/context_builder.rb

@@ -22,8 +22,9 @@ module Aireview
     CANDIDATES_RESERVE_CHARS = 4_500
 
     # The context of one run: both stages get the same MR, Jira and diff,
-    # truncated once for the tightest of the stages.
-    Context = Struct.new(:user_prompt, :diff_text, :coverage, :sizes, keyword_init: true)
+    # truncated once for the tightest of the stages. sections — the MR and
+    # Jira part without the diff, for Jev.
+    Context = Struct.new(:user_prompt, :sections, :diff_text, :coverage, :sizes, keyword_init: true)
 
     def initialize(config:, logger: Logger.new($stderr))
       @config = config
@@ -49,7 +50,8 @@ module Aireview
 
       sizes = context_sizes(fixed: fixed, packed: packed, budget: budget, diff_budget: diff_budget, critique: critique)
       log_sizes(sizes)
-      Context.new(user_prompt: fixed + packed.text, diff_text: packed.text, coverage: coverage, sizes: sizes)
+      Context.new(user_prompt: fixed + packed.text, sections: sections.join("\n\n"), diff_text: packed.text,
+                  coverage: coverage, sizes: sizes)
     end
 
     def build_generate_prompt(context)
@@ -88,12 +90,17 @@ module Aireview
       {system_prompt: system, user_prompt: user}
     end
 
+    def scrub_text(text)
+      @secret_scrubber.scrub_text(text.to_s)
+    end
+
     private
 
     # The minimum over the stages: the context is one per run, so it must fit
-    # into each of them together with its system prompt and reserve.
+    # into each of them together with its system prompt and reserve. Only the
+    # stages that go to an LLM count: Jev as the critic has limits of its own.
     def context_budget(critique:)
-      stages = critique ? STAGES : ['generate']
+      stages = @config.llm_stages(critique: critique)
       budgets = stages.to_h { |stage| [stage, stage_budget(stage)] }
       stage, budget = budgets.min_by { |_, value| value }
       return budget if budget.positive?
@@ -153,7 +160,7 @@ module Aireview
     end
 
     def context_sizes(fixed:, packed:, budget:, diff_budget:, critique:)
-      stages = critique ? STAGES : ['generate']
+      stages = @config.llm_stages(critique: critique)
       {
         context_budget: budget,
         diff_budget: diff_budget,
@@ -187,10 +194,6 @@ module Aireview
       Aireview::Utils.presence(scrubbed) || '(empty)'
     end
 
-    def scrub_text(text)
-      @secret_scrubber.scrub_text(text.to_s)
-    end
-
     def language_name(code)
       LANGUAGE_NAMES.fetch(code.to_s, code.to_s)
     end

data/lib/aireview/dry_run_prompts.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+require 'json'
+require 'logger'
+require_relative 'utils'
+require_relative 'jev_critic'
+
+module Aireview
+  # The prompts of a run without calling a model, and everything --dry-run
+  # shows. The review key is computed from them too (ReviewMarker): the LLM
+  # Critique prompt is built only when an LLM Critique can run, the Jev
+  # question templates only when Jev decides.
+  class DryRunPrompts
+    CANDIDATES_JSON = '[{"id":"C1","file":"path/from/diff.rb","line":1,' \
+                      '"quoted_code":"...","problem":"...","why":"...","suggestion":"...",' \
+                      '"category":"bug","severity":"major"}]'
+
+    def initialize(config:, context_builder:, logger: Logger.new($stderr))
+      @config = config
+      @context_builder = context_builder
+      @logger = logger
+    end
+
+    def build(merge_request:, changes:, jira_issue: nil, critique: true)
+      @config.require_models!(critique: critique)
+      stages = @config.llm_stages(critique: critique)
+      context = @context_builder.prepare(
+        merge_request: merge_request,
+        changes: changes,
+        jira_issue: jira_issue,
+        critique: critique
+      )
+
+      prompts(context, stages, critique).merge(settings(context, stages, critique))
+    end
+
+    private
+
+    def prompts(context, stages, critique)
+      jev = @config.jev_critique?(critique: critique)
+      critique_prompt = if stages.include?('critique')
+                          @context_builder.build_critique_prompt(context, candidates_json: CANDIDATES_JSON)
+                        end
+      {
+        generate_prompt: @context_builder.build_generate_prompt(context),
+        critique_prompt: critique_prompt,
+        jev_questions: jev ? JevCritic.decision_templates : nil,
+        jev_critique: jev ? jev_critique_settings(context) : nil,
+        jev_shadow: critique && !jev ? jev_shadow_settings : nil
+      }
+    end
+
+    def settings(context, stages, critique)
+      llm_critique = stages.include?('critique')
+      {
+        generate_model: @config.generate_model,
+        generate_temperature: @config.generate_temperature,
+        critique_model: @config.critique_model,
+        critique_temperature: @config.critique_temperature,
+        generate_fallbacks: @config.fallback_names('generate'),
+        critique_fallbacks: llm_critique ? @config.fallback_names('critique') : [],
+        sources: setting_sources(stages),
+        config_paths: @config.layer_paths,
+        warnings: @config.warnings,
+        critique_rule: llm_critique ? @config.routing.rule : nil,
+        api_keys: @config.api_key_counts(stages),
+        time_budget: @config.llm_time_budget,
+        overloaded_quarantine: @config.overloaded_quarantine,
+        coverage: context.coverage,
+        sizes: context.sizes
+      }
+    end
+
+    # The Jev request as it would go for the stub candidate; nothing is sent,
+    # so the critic needs no client. Only whether the key is set: its value
+    # never leaves.
+    def jev_critique_settings(context)
+      critic = JevCritic.new(client: nil, thresholds: @config.jev_thresholds,
+                             review_instructions: @config.review_instructions,
+                             scrub: @context_builder.method(:scrub_text), logger: @logger)
+      request = critic.preview(context: context, candidates: JSON.parse(CANDIDATES_JSON))
+      {model: @config.jev_model, key: Aireview::Utils.present?(@config.jev_api_key),
+       thresholds: @config.jev_thresholds, fallback: @config.jev_fallback,
+       state: request.state, questions: request.questions}
+    end
+
+    # Only whether the key is set: its value never leaves.
+    def jev_shadow_settings
+      return nil unless @config.jev_shadow?
+
+      {model: @config.jev_model, key: Aireview::Utils.present?(@config.jev_api_key), thresholds: @config.jev_thresholds}
+    end
+
+    # Where the model, provider and reserves of a stage came from, for --dry-run.
+    def setting_sources(stages)
+      stages.to_h do |stage|
+        [stage.to_sym, {
+          model: @config.stage_model_source(stage),
+          provider: @config.stage_provider_source(stage),
+          fallbacks: @config.stage_fallbacks_source(stage)
+        }]
+      end
+    end
+  end
+end

data/lib/aireview/dry_run_report.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require 'json'
 
 module Aireview
   # The --dry-run output: settings, the context summary and the prompts of both stages.
@@ -19,32 +20,63 @@ module Aireview
       @out.puts
       @out.puts('=== GENERATE USER PROMPT ===')
       @out.puts(dry_run.dig(:generate_prompt, :user_prompt))
-      return unless dry_run[:critique_prompt]
+      render_critique_prompt(dry_run[:critique_prompt]) if dry_run[:critique_prompt]
+      render_jev_request(dry_run[:jev_critique]) if dry_run[:jev_critique]
+    end
+
+    private
 
+    def render_critique_prompt(prompt)
       @out.puts
       @out.puts('=== CRITIQUE SYSTEM PROMPT ===')
-      @out.puts(dry_run.dig(:critique_prompt, :system_prompt))
+      @out.puts(prompt[:system_prompt])
       @out.puts
       @out.puts('=== CRITIQUE USER PROMPT ===')
-      @out.puts(dry_run.dig(:critique_prompt, :user_prompt))
+      @out.puts(prompt[:user_prompt])
     end
 
-    private
+    # The request Jev would get for the stub candidate.
+    def render_jev_request(jev)
+      @out.puts
+      @out.puts('=== JEV STATE ===')
+      @out.puts(JSON.pretty_generate(jev[:state]))
+      @out.puts
+      @out.puts('=== JEV QUESTIONS ===')
+      @out.puts(JSON.pretty_generate(jev[:questions]))
+    end
 
+    # With Jev as the engine the LLM Critique line is its fallback.
     def render_settings(dry_run)
       @out.puts('=== LLM SETTINGS ===')
       render_config_paths(dry_run[:config_paths])
       render_stage(dry_run, :generate)
+      render_jev_critique(dry_run[:jev_critique])
       if dry_run[:critique_prompt]
-        render_stage(dry_run, :critique)
-      else
+        render_stage(dry_run, :critique, title: dry_run[:jev_critique] ? 'Critique fallback' : 'Critique')
+      elsif !dry_run[:jev_critique]
         @out.puts('Critique: disabled')
       end
       @out.puts("Critique rule: #{dry_run[:critique_rule]}") if dry_run[:critique_rule]
+      render_jev_shadow(dry_run[:jev_shadow])
       render_reserves(dry_run)
       list('warnings', dry_run[:warnings], separator: "\n  ")
     end
 
+    def render_jev_critique(jev)
+      return unless jev
+
+      thresholds = jev[:thresholds].map { |name, value| "#{name}=#{value}" }.join(' ')
+      @out.puts("Critique: jev #{jev[:model]} (key #{jev[:key] ? 'set' : 'missing'}; " \
+                "fallback: #{jev[:fallback]}; #{thresholds})")
+    end
+
+    def render_jev_shadow(jev)
+      return unless jev
+
+      thresholds = jev[:thresholds].map { |name, value| "#{name}=#{value}" }.join(' ')
+      @out.puts("Jev shadow: #{jev[:model]} (key #{jev[:key] ? 'set' : 'missing'}; log only, #{thresholds})")
+    end
+
     def render_config_paths(paths)
       return if paths.nil? || paths.empty?
 
@@ -53,9 +85,9 @@ module Aireview
 
     # The source of every setting is the layer it came from: built-in, image
     # defaults, .aireview.yml, env or cli.
-    def render_stage(dry_run, stage)
+    def render_stage(dry_run, stage, title: stage.capitalize)
       sources = dry_run.dig(:sources, stage) || {}
-      @out.puts("#{stage.capitalize}: #{dry_run[:"#{stage}_model"]} " \
+      @out.puts("#{title}: #{dry_run[:"#{stage}_model"]} " \
                 "temperature=#{dry_run[:"#{stage}_temperature"]}#{origin(sources, :model, :provider)}")
       fallbacks = dry_run[:"#{stage}_fallbacks"]
       list('fallbacks', fallbacks, separator: ' -> ', suffix: origin(sources, :fallbacks))

data/lib/aireview/errors.rb

@@ -14,4 +14,15 @@ module Aireview
   class RouteExhaustedError < ApiError; end
   class ContextBudgetError < Error; end
   class HelpRequested < Error; end
+
+  # A Jev request failed: network, HTTP status, an answer of the wrong shape.
+  # status is the HTTP status when the server answered.
+  class JevError < Error
+    attr_reader :status
+
+    def initialize(message, status: nil)
+      super(message)
+      @status = status
+    end
+  end
 end

data/lib/aireview/jev_client.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+require 'json'
+require 'logger'
+require_relative 'errors'
+require_relative 'utils'
+
+module Aireview
+  # One evaluation request to Jev (TypeSafe, POST /v1/systemone): a state and
+  # named questions in, one answer per question out. RubyLLM does not know
+  # Jev and the router is built for text models, so Jev has its own small
+  # client: one retry on 429/529, no key rotation, no reserves. A failure is
+  # a JevError; the caller decides whether it matters.
+  class JevClient
+    API_URL = 'https://api.typesafe.ai/v1/'
+    OPEN_TIMEOUT = 5
+    RETRY_STATUSES = [429, 529].freeze
+    RETRY_DELAY = 2
+    # retry-after beyond this is not worth waiting for in a review job.
+    MAX_RETRY_DELAY = 10
+
+    Result = Struct.new(:model, :answers, :usage, keyword_init: true)
+
+    # Jev goes out the same way as the LLM providers, through
+    # LLM_HTTP_PROXY when it is set. dependencies — connection: and sleeper:
+    # for tests.
+    def initialize(config:, logger: Logger.new($stderr), **dependencies)
+      require 'faraday'
+
+      raise ConfigError, 'JEV_API_KEY is required' if Aireview::Utils.blank?(config.jev_api_key)
+
+      @api_key = config.jev_api_key
+      @model = config.jev_model
+      @logger = logger
+      @connection = dependencies[:connection] || build_connection(config.jev_timeout, config.llm_http_proxy)
+      @sleeper = dependencies[:sleeper] || ->(seconds) { sleep(seconds) }
+    end
+
+    # questions — {key => question}; returns Result with answers under the
+    # same keys, each checked against the type of its question.
+    def evaluate(state:, questions:)
+      body = JSON.generate(model: @model, state: state, questions: questions)
+      @logger.info("Jev request started (model=#{@model}, questions=#{questions.size})")
+      response = post_with_one_retry(body)
+      result = parse(response, questions)
+      log_completed(result)
+      result
+    end
+
+    private
+
+    def build_connection(timeout, proxy)
+      Faraday.new(url: API_URL, proxy: Aireview::Utils.presence(proxy)) do |builder|
+        builder.options.open_timeout = OPEN_TIMEOUT
+        builder.options.timeout = timeout
+        builder.adapter Faraday.default_adapter
+      end
+    end
+
+    def post_with_one_retry(body)
+      response = post(body)
+      return response unless RETRY_STATUSES.include?(response.status.to_i)
+
+      delay = retry_delay(response)
+      @logger.warn("Jev answered #{response.status}, retrying once in #{delay}s")
+      @sleeper.call(delay)
+      post(body)
+    end
+
+    def post(body)
+      @connection.post('systemone') do |request|
+        request.headers['Authorization'] = "Bearer #{@api_key}"
+        request.headers['Content-Type'] = 'application/json'
+        request.body = body
+      end
+    rescue Faraday::Error => e
+      raise JevError, "Jev request failed: #{e.class}: #{e.message}"
+    end
+
+    def retry_delay(response)
+      seconds = Integer(response.headers['retry-after'].to_s, exception: false)
+      seconds&.positive? ? [seconds, MAX_RETRY_DELAY].min : RETRY_DELAY
+    end
+
+    def parse(response, questions)
+      status = response.status.to_i
+      unless status.between?(200, 299)
+        raise JevError.new("Jev API error #{status}: #{response.body.to_s[0, 500]}", status: status)
+      end
+
+      payload = JSON.parse(response.body.to_s)
+      answers = payload['answers'] if payload.is_a?(Hash)
+      raise JevError, 'Jev returned no answers' unless answers.is_a?(Hash)
+
+      questions.each { |key, question| check_answer!(key, question, answers[key.to_s]) }
+      Result.new(model: payload['model'], answers: answers, usage: payload['usage'])
+    rescue JSON::ParserError
+      raise JevError, "Jev returned invalid JSON: #{response.body.to_s[0, 500]}"
+    end
+
+    def check_answer!(key, question, answer)
+      type = question[:type] || question['type']
+      return if answer.is_a?(Hash) && answer['type'] == type && valid_value?(type, answer)
+
+      raise JevError, "Jev returned an invalid #{type} answer for #{key}: #{answer.inspect}"
+    end
+
+    def valid_value?(type, answer)
+      case type
+      when 'noul' then probability?(answer['noul'])
+      when 'choice' then answer['choice'].is_a?(String) && probability?(answer['confidence'])
+      else true
+      end
+    end
+
+    def probability?(value)
+      value.is_a?(Numeric) && value.between?(0, 1)
+    end
+
+    # The answering version is logged: an alias resolves on the server, and
+    # a pinned version that answers as another one breaks the thresholds.
+    def log_completed(result)
+      tokens = result.usage.is_a?(Hash) ? ", tokens: input=#{result.usage['input_tokens']}" : ''
+      @logger.info("Jev request completed (model=#{result.model}#{tokens})")
+      return if result.model == @model || !@model.match?(/\Ajev-\d/)
+
+      @logger.warn("Jev answered as #{result.model.inspect}, not the requested #{@model}")
+    end
+  end
+end