aireview 0.3.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -0
- data/README.md +277 -28
- data/config/defaults.yml +47 -0
- data/lib/aireview/candidate_checker.rb +24 -22
- data/lib/aireview/cli.rb +46 -12
- data/lib/aireview/config.rb +65 -180
- data/lib/aireview/config_fallbacks.rb +79 -68
- data/lib/aireview/config_layers.rb +110 -0
- data/lib/aireview/config_limits.rb +10 -35
- data/lib/aireview/config_loader.rb +240 -0
- data/lib/aireview/context_budget.rb +22 -20
- data/lib/aireview/context_builder.rb +18 -17
- data/lib/aireview/diff_fetcher.rb +12 -11
- data/lib/aireview/dry_run_report.rb +33 -8
- data/lib/aireview/errors.rb +8 -0
- data/lib/aireview/gitlab_client.rb +10 -9
- data/lib/aireview/llm_client.rb +113 -0
- data/lib/aireview/llm_failure.rb +36 -19
- data/lib/aireview/llm_router.rb +315 -158
- data/lib/aireview/model_candidate.rb +28 -0
- data/lib/aireview/model_checker.rb +148 -0
- data/lib/aireview/model_pool.rb +224 -0
- data/lib/aireview/model_state.rb +82 -0
- data/lib/aireview/publisher.rb +6 -6
- data/lib/aireview/result_parser.rb +98 -0
- data/lib/aireview/review_marker.rb +16 -28
- data/lib/aireview/review_pipeline.rb +88 -86
- data/lib/aireview/review_renderer.rb +19 -11
- data/lib/aireview/reviewer.rb +37 -109
- data/lib/aireview/stage_chains.rb +113 -0
- data/lib/aireview/stages.rb +7 -0
- data/lib/aireview/utils.rb +29 -0
- data/lib/aireview/version.rb +1 -1
- data/lib/aireview.rb +1 -0
- metadata +19 -3
- data/lib/aireview/result_validation.rb +0 -65
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Aireview
|
|
4
|
-
#
|
|
4
|
+
# The --dry-run output: settings, the context summary and the prompts of both stages.
|
|
5
5
|
class DryRunReport
|
|
6
6
|
def initialize(out)
|
|
7
7
|
@out = out
|
|
@@ -33,15 +33,37 @@ module Aireview
|
|
|
33
33
|
|
|
34
34
|
def render_settings(dry_run)
|
|
35
35
|
@out.puts('=== LLM SETTINGS ===')
|
|
36
|
-
|
|
37
|
-
|
|
36
|
+
render_config_paths(dry_run[:config_paths])
|
|
37
|
+
render_stage(dry_run, :generate)
|
|
38
38
|
if dry_run[:critique_prompt]
|
|
39
|
-
|
|
40
|
-
list('fallbacks', dry_run[:critique_fallbacks], separator: ' -> ')
|
|
39
|
+
render_stage(dry_run, :critique)
|
|
41
40
|
else
|
|
42
41
|
@out.puts('Critique: disabled')
|
|
43
42
|
end
|
|
43
|
+
@out.puts("Critique rule: #{dry_run[:critique_rule]}") if dry_run[:critique_rule]
|
|
44
44
|
render_reserves(dry_run)
|
|
45
|
+
list('warnings', dry_run[:warnings], separator: "\n ")
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def render_config_paths(paths)
|
|
49
|
+
return if paths.nil? || paths.empty?
|
|
50
|
+
|
|
51
|
+
@out.puts("Config: #{paths.map { |name, path| "#{name} #{path}" }.join(', ')}")
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# The source of every setting is the layer it came from: built-in, image
|
|
55
|
+
# defaults, .aireview.yml, env or cli.
|
|
56
|
+
def render_stage(dry_run, stage)
|
|
57
|
+
sources = dry_run.dig(:sources, stage) || {}
|
|
58
|
+
@out.puts("#{stage.capitalize}: #{dry_run[:"#{stage}_model"]} " \
|
|
59
|
+
"temperature=#{dry_run[:"#{stage}_temperature"]}#{origin(sources, :model, :provider)}")
|
|
60
|
+
fallbacks = dry_run[:"#{stage}_fallbacks"]
|
|
61
|
+
list('fallbacks', fallbacks, separator: ' -> ', suffix: origin(sources, :fallbacks))
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def origin(sources, *keys)
|
|
65
|
+
parts = keys.filter_map { |key| "#{key} from #{sources[key]}" if sources[key] }
|
|
66
|
+
parts.empty? ? '' : " (#{parts.join(', ')})"
|
|
45
67
|
end
|
|
46
68
|
|
|
47
69
|
def render_context_sizes(sizes)
|
|
@@ -67,12 +89,15 @@ module Aireview
|
|
|
67
89
|
def render_reserves(dry_run)
|
|
68
90
|
keys = Array(dry_run[:api_keys]).map { |provider, count| "#{provider} #{count}" }
|
|
69
91
|
@out.puts("API keys: #{keys.join(', ')}") unless keys.empty?
|
|
70
|
-
|
|
92
|
+
return unless dry_run[:time_budget]
|
|
93
|
+
|
|
94
|
+
quarantine = dry_run[:overloaded_quarantine]
|
|
95
|
+
@out.puts("Time budget: #{dry_run[:time_budget]}s#{", overloaded quarantine: #{quarantine}s" if quarantine}")
|
|
71
96
|
end
|
|
72
97
|
|
|
73
|
-
def list(title, items, separator: ', ')
|
|
98
|
+
def list(title, items, separator: ', ', suffix: '')
|
|
74
99
|
items = Array(items)
|
|
75
|
-
@out.puts(" #{title}: #{items.join(separator)}") unless items.empty?
|
|
100
|
+
@out.puts(" #{title}: #{items.join(separator)}#{suffix}") unless items.empty?
|
|
76
101
|
end
|
|
77
102
|
end
|
|
78
103
|
end
|
data/lib/aireview/errors.rb
CHANGED
|
@@ -3,7 +3,15 @@ module Aireview
|
|
|
3
3
|
class Error < StandardError; end
|
|
4
4
|
class ConfigError < Error; end
|
|
5
5
|
class ParseError < Error; end
|
|
6
|
+
# Repairing invalid JSON is impossible: the model that answered is out of
|
|
7
|
+
# attempts or quarantined with no budget to wait. For the pipeline it is
|
|
8
|
+
# the same as an invalid result — the stage restarts on another model.
|
|
9
|
+
class RepairImpossibleError < ParseError; end
|
|
6
10
|
class ApiError < Error; end
|
|
11
|
+
# A pinned route (the JSON repair by the same model) could not answer: out
|
|
12
|
+
# of attempts, excluded, or quarantined with no budget to wait. Not fatal
|
|
13
|
+
# for the run — the stage restarts on another model.
|
|
14
|
+
class RouteExhaustedError < ApiError; end
|
|
7
15
|
class ContextBudgetError < Error; end
|
|
8
16
|
class HelpRequested < Error; end
|
|
9
17
|
end
|
|
@@ -35,8 +35,9 @@ module Aireview
|
|
|
35
35
|
get_json('user')
|
|
36
36
|
end
|
|
37
37
|
|
|
38
|
-
# Retry
|
|
39
|
-
#
|
|
38
|
+
# A Retry creates a new job in the same pipeline. Earlier attempts are
|
|
39
|
+
# visible only with include_retried; a new CI_JOB_ID alone happens after an
|
|
40
|
+
# ordinary push too.
|
|
40
41
|
def retried_job?(project_id, job_id)
|
|
41
42
|
current = get_json("projects/#{project_id}/jobs/#{job_id}")
|
|
42
43
|
validate_retry_job!(current, pipeline: true)
|
|
@@ -54,13 +55,13 @@ module Aireview
|
|
|
54
55
|
raise ApiError, 'Too many pipeline jobs to determine whether this job is a retry'
|
|
55
56
|
end
|
|
56
57
|
|
|
57
|
-
#
|
|
58
|
-
#
|
|
59
|
-
#
|
|
60
|
-
#
|
|
61
|
-
#
|
|
62
|
-
#
|
|
63
|
-
#
|
|
58
|
+
# Notes come in pages sorted by creation time, and updating a comment
|
|
59
|
+
# does not move it up in that order: on a long MR our review is far from
|
|
60
|
+
# the first page. Stopping the walk silently is not an option — with an
|
|
61
|
+
# incomplete list the review would decide the note is missing and create
|
|
62
|
+
# a second one, so the limit fails with an error. The order is explicit:
|
|
63
|
+
# it decides which old unmarked note the review picks up, and the API
|
|
64
|
+
# default is not to be relied on here.
|
|
64
65
|
def fetch_merge_request_notes(project_id, iid)
|
|
65
66
|
notes = []
|
|
66
67
|
page = 1
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require 'timeout'
|
|
3
|
+
require_relative 'errors'
|
|
4
|
+
require_relative 'utils'
|
|
5
|
+
|
|
6
|
+
module Aireview
|
|
7
|
+
# One request to one model with one key through RubyLLM. Retries, keys and
|
|
8
|
+
# reserves belong to LlmRouter: the built-in RubyLLM/Faraday retries (3 by
|
|
9
|
+
# default) are off, otherwise every router attempt would turn into four
|
|
10
|
+
# HTTP requests and burn quota before the error reaches the classifier.
|
|
11
|
+
class LlmClient
|
|
12
|
+
# What a stage sends to the model; the same for every route of the stage.
|
|
13
|
+
Prompt = Struct.new(:stage, :system, :user, :temperature, :schema, keyword_init: true) do
|
|
14
|
+
def chars
|
|
15
|
+
system.length + user.length
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def initialize(config:, logger: Logger.new($stderr))
|
|
20
|
+
@config = config
|
|
21
|
+
@logger = logger
|
|
22
|
+
@contexts = {}
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Returns the RubyLLM answer (content is text or a structure by the
|
|
26
|
+
# schema). A request error is re-raised as is — LlmFailure classifies it.
|
|
27
|
+
def request(prompt, candidate:, key:, timeout:, key_index: 0)
|
|
28
|
+
load_ruby_llm
|
|
29
|
+
stage = prompt.stage.to_s
|
|
30
|
+
model = candidate.model
|
|
31
|
+
@logger.info("LLM #{stage} request started (model=#{model}, temperature=#{prompt.temperature})")
|
|
32
|
+
chat = build_chat(context: context(stage, candidate.provider, key, key_index), stage: stage,
|
|
33
|
+
model: model, provider: candidate.provider)
|
|
34
|
+
chat = configure_reasoning(chat: chat, model: model, provider: candidate.provider)
|
|
35
|
+
.with_temperature(prompt.temperature.to_f)
|
|
36
|
+
.with_schema(prompt.schema)
|
|
37
|
+
chat.with_instructions(prompt.system)
|
|
38
|
+
response = Timeout.timeout(timeout) { chat.ask(prompt.user) }
|
|
39
|
+
@logger.info("LLM #{stage} request completed (model=#{model})")
|
|
40
|
+
response
|
|
41
|
+
rescue Timeout::Error
|
|
42
|
+
@logger.warn("LLM #{stage} request timed out after #{timeout.round} seconds (model=#{model})")
|
|
43
|
+
raise
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def load_ruby_llm
|
|
49
|
+
require 'ruby_llm'
|
|
50
|
+
rescue LoadError => e
|
|
51
|
+
@logger.error("LLM setup failed: #{e.message}")
|
|
52
|
+
raise ConfigError, "Missing dependency: #{e.message}"
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def configure_reasoning(chat:, model:, provider:)
|
|
56
|
+
return chat unless provider == 'ollama' && model.start_with?('gpt-oss:')
|
|
57
|
+
|
|
58
|
+
chat.with_thinking(effort: :low)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def build_chat(context:, stage:, model:, provider:)
|
|
62
|
+
context.chat(model: model, provider: provider.to_sym)
|
|
63
|
+
rescue RubyLLM::ModelNotFoundError
|
|
64
|
+
@logger.warn(
|
|
65
|
+
"LLM #{stage}: model not found in RubyLLM registry; " \
|
|
66
|
+
"using fallback with incomplete model metadata " \
|
|
67
|
+
"(model=#{model}, provider=#{provider})"
|
|
68
|
+
)
|
|
69
|
+
context.chat(model: model, provider: provider.to_sym, assume_model_exists: true)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# A RubyLLM context per stage, provider and key index: switching the key
|
|
73
|
+
# is another context, not an edit of the global config.
|
|
74
|
+
def context(stage, provider, key, key_index)
|
|
75
|
+
@contexts[[stage, provider, key_index]] ||= build_context(provider.to_s, key)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def build_context(provider, api_key)
|
|
79
|
+
RubyLLM.context do |ruby_config|
|
|
80
|
+
configure_http_proxy(ruby_config)
|
|
81
|
+
ruby_config.request_timeout = @config.llm_timeout.to_f
|
|
82
|
+
ruby_config.max_retries = 0
|
|
83
|
+
configure_provider(ruby_config, provider, api_key)
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def configure_http_proxy(ruby_config)
|
|
88
|
+
return unless Aireview::Utils.present?(@config.llm_http_proxy)
|
|
89
|
+
|
|
90
|
+
ruby_config.http_proxy = @config.llm_http_proxy
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def configure_provider(ruby_config, provider, api_key)
|
|
94
|
+
case provider
|
|
95
|
+
when 'gemini', 'openai', 'openrouter'
|
|
96
|
+
configure_remote_provider(ruby_config, provider, api_key)
|
|
97
|
+
when 'anthropic'
|
|
98
|
+
ruby_config.anthropic_api_key = api_key
|
|
99
|
+
when 'ollama'
|
|
100
|
+
ruby_config.ollama_api_base = @config.ollama_api_base
|
|
101
|
+
else
|
|
102
|
+
raise ConfigError, "Unsupported LLM provider: #{provider.inspect}"
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def configure_remote_provider(ruby_config, provider, api_key)
|
|
107
|
+
ruby_config.public_send("#{provider}_api_key=", api_key)
|
|
108
|
+
return unless Aireview::Utils.present?(@config.llm_api_base)
|
|
109
|
+
|
|
110
|
+
ruby_config.public_send("#{provider}_api_base=", @config.llm_api_base)
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
data/lib/aireview/llm_failure.rb
CHANGED
|
@@ -3,17 +3,27 @@ require 'json'
|
|
|
3
3
|
require 'timeout'
|
|
4
4
|
|
|
5
5
|
module Aireview
|
|
6
|
-
#
|
|
7
|
-
#
|
|
6
|
+
# Classifies an LLM request error. Answers only "what is it"; the decision
|
|
7
|
+
# to retry, switch the key or the model belongs to LlmRouter.
|
|
8
8
|
#
|
|
9
|
-
# :daily_quota —
|
|
10
|
-
# :rate_limit —
|
|
11
|
-
# :overloaded — 503
|
|
12
|
-
# :timeout —
|
|
13
|
-
# :
|
|
14
|
-
# :
|
|
9
|
+
# :daily_quota — the project's daily quota for the model, retries are useless;
|
|
10
|
+
# :rate_limit — a per-minute limit, passes after the hinted time;
|
|
11
|
+
# :overloaded — 503 / "high demand" on the model;
|
|
12
|
+
# :timeout — no answer for longer than LLM_TIMEOUT;
|
|
13
|
+
# :unavailable — the provider has no such model: retired, a typo, not pulled into Ollama;
|
|
14
|
+
# :fatal — an API error that reserves do not cure;
|
|
15
|
+
# :unhandled — not a provider error, re-raised as is.
|
|
15
16
|
module LlmFailure
|
|
16
|
-
KINDS = %i[daily_quota rate_limit overloaded timeout fatal unhandled].freeze
|
|
17
|
+
KINDS = %i[daily_quota rate_limit overloaded timeout unavailable fatal unhandled].freeze
|
|
18
|
+
# Only by the provider's text about the model: a bare 404 is also what a
|
|
19
|
+
# wrong LLM_API_BASE or proxy answers, and the next model will not help.
|
|
20
|
+
# Ollama: `model 'x' not found` (through /v1) and `model "x" not found,
|
|
21
|
+
# try pulling it first` (older versions and /api).
|
|
22
|
+
UNAVAILABLE_MODEL_TEXT = Regexp.union(
|
|
23
|
+
/\bmodels?\/[\w.:-]+ is not found\b/i,
|
|
24
|
+
/\bis not supported for generateContent\b/i,
|
|
25
|
+
/\bmodel ['"][^'"]+['"] not found\b/i
|
|
26
|
+
)
|
|
17
27
|
QUOTA_FAILURE_TYPE = 'type.googleapis.com/google.rpc.QuotaFailure'
|
|
18
28
|
DAILY_QUOTA_ID = /PerDay/i
|
|
19
29
|
DAILY_QUOTA_TEXT = /\bper\s+day\b|\bdaily\b/i
|
|
@@ -21,18 +31,22 @@ module Aireview
|
|
|
21
31
|
|
|
22
32
|
module_function
|
|
23
33
|
|
|
24
|
-
#
|
|
25
|
-
# 429
|
|
26
|
-
#
|
|
34
|
+
# Quota details are checked before the exception class: RubyLLM turns a
|
|
35
|
+
# 429 mentioning input_token into ContextLengthExceededError, although
|
|
36
|
+
# it is an exhausted token quota, not an oversized request.
|
|
27
37
|
def classify(error)
|
|
28
38
|
return :timeout if transport_timeout?(error)
|
|
29
39
|
return :unhandled unless ruby_llm_error?(error)
|
|
30
40
|
|
|
31
|
-
quota_kind(error) || (overloaded?(error) ? :overloaded : :fatal)
|
|
41
|
+
quota_kind(error) || model_kind(error) || (overloaded?(error) ? :overloaded : :fatal)
|
|
32
42
|
end
|
|
33
43
|
|
|
34
|
-
|
|
35
|
-
|
|
44
|
+
def model_kind(error)
|
|
45
|
+
:unavailable if error.message.to_s.match?(UNAVAILABLE_MODEL_TEXT)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# The outer Timeout.timeout and Faraday's transport timeouts: the latter
|
|
49
|
+
# inherit neither Timeout::Error nor RubyLLM::Error.
|
|
36
50
|
def transport_timeout?(error)
|
|
37
51
|
return true if error.is_a?(Timeout::Error) || error.is_a?(Errno::ETIMEDOUT)
|
|
38
52
|
|
|
@@ -47,10 +61,10 @@ module Aireview
|
|
|
47
61
|
defined?(RubyLLM::Error) && error.is_a?(RubyLLM::Error)
|
|
48
62
|
end
|
|
49
63
|
|
|
50
|
-
# Google
|
|
51
|
-
# (GenerateRequestsPerDay… / …PerMinute…).
|
|
52
|
-
#
|
|
53
|
-
#
|
|
64
|
+
# Google puts the kind of quota into QuotaFailure.violations[].quotaId
|
|
65
|
+
# (GenerateRequestsPerDay… / …PerMinute…). The free_tier_requests metric
|
|
66
|
+
# is the same for both, it cannot tell them apart. The message text is
|
|
67
|
+
# the fallback signal when there is no response body.
|
|
54
68
|
def quota_kind(error)
|
|
55
69
|
ids = quota_ids(error)
|
|
56
70
|
return daily_quota_id?(ids) ? :daily_quota : :rate_limit unless ids.empty?
|
|
@@ -86,5 +100,8 @@ module Aireview
|
|
|
86
100
|
match = message.to_s.match(RETRY_AFTER)
|
|
87
101
|
match[1].to_f if match
|
|
88
102
|
end
|
|
103
|
+
|
|
104
|
+
private_class_method :model_kind, :transport_timeout?, :overloaded?, :ruby_llm_error?, :quota_kind,
|
|
105
|
+
:daily_quota_id?, :quota_ids, :response_body
|
|
89
106
|
end
|
|
90
107
|
end
|