aireview 0.3.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -0
- data/README.md +277 -28
- data/config/defaults.yml +47 -0
- data/lib/aireview/candidate_checker.rb +24 -22
- data/lib/aireview/cli.rb +46 -12
- data/lib/aireview/config.rb +65 -180
- data/lib/aireview/config_fallbacks.rb +79 -68
- data/lib/aireview/config_layers.rb +110 -0
- data/lib/aireview/config_limits.rb +10 -35
- data/lib/aireview/config_loader.rb +240 -0
- data/lib/aireview/context_budget.rb +22 -20
- data/lib/aireview/context_builder.rb +18 -17
- data/lib/aireview/diff_fetcher.rb +12 -11
- data/lib/aireview/dry_run_report.rb +33 -8
- data/lib/aireview/errors.rb +8 -0
- data/lib/aireview/gitlab_client.rb +10 -9
- data/lib/aireview/llm_client.rb +113 -0
- data/lib/aireview/llm_failure.rb +36 -19
- data/lib/aireview/llm_router.rb +315 -158
- data/lib/aireview/model_candidate.rb +28 -0
- data/lib/aireview/model_checker.rb +148 -0
- data/lib/aireview/model_pool.rb +224 -0
- data/lib/aireview/model_state.rb +82 -0
- data/lib/aireview/publisher.rb +6 -6
- data/lib/aireview/result_parser.rb +98 -0
- data/lib/aireview/review_marker.rb +16 -28
- data/lib/aireview/review_pipeline.rb +88 -86
- data/lib/aireview/review_renderer.rb +19 -11
- data/lib/aireview/reviewer.rb +37 -109
- data/lib/aireview/stage_chains.rb +113 -0
- data/lib/aireview/stages.rb +7 -0
- data/lib/aireview/utils.rb +29 -0
- data/lib/aireview/version.rb +1 -1
- data/lib/aireview.rb +1 -0
- metadata +19 -3
- data/lib/aireview/result_validation.rb +0 -65
|
@@ -1,18 +1,21 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
require 'json'
|
|
3
3
|
require_relative 'errors'
|
|
4
|
+
require_relative 'stages'
|
|
4
5
|
require_relative 'context_builder'
|
|
5
6
|
require_relative 'candidate_checker'
|
|
6
|
-
require_relative '
|
|
7
|
+
require_relative 'result_parser'
|
|
7
8
|
require_relative 'review_renderer'
|
|
8
9
|
require_relative 'review_schemas'
|
|
9
10
|
require_relative 'reviewer'
|
|
10
11
|
|
|
11
12
|
module Aireview
|
|
13
|
+
# A review run: context → Generate → anchoring check against the diff →
|
|
14
|
+
# Critique → report. Invalid JSON is repaired once by the same model; when
|
|
15
|
+
# the repair is invalid too, the stage restarts on another model with the
|
|
16
|
+
# original request.
|
|
12
17
|
class ReviewPipeline
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
SchemaError = ResultValidation::SchemaError
|
|
18
|
+
SchemaError = ResultParser::SchemaError
|
|
16
19
|
|
|
17
20
|
REPAIR_SYSTEM_PROMPT = <<~PROMPT.strip.freeze
|
|
18
21
|
You fix invalid JSON produced by another LLM call.
|
|
@@ -26,6 +29,7 @@ module Aireview
|
|
|
26
29
|
|
|
27
30
|
def initialize(config:, reviewer: nil, context_builder: nil, logger: Logger.new($stderr))
|
|
28
31
|
@config = config
|
|
32
|
+
@parser = ResultParser.new
|
|
29
33
|
@reviewer = reviewer || Reviewer.new(config: config, logger: logger)
|
|
30
34
|
@context_builder = context_builder || ContextBuilder.new(config: config, logger: logger)
|
|
31
35
|
@logger = logger
|
|
@@ -40,16 +44,18 @@ module Aireview
|
|
|
40
44
|
)
|
|
41
45
|
generate_prompt = @context_builder.build_generate_prompt(context)
|
|
42
46
|
@logger.info("Pipeline generate pass started (model=#{@config.generate_model})")
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
47
|
+
generate_result = run_stage('generate') do
|
|
48
|
+
parse_with_repair(
|
|
49
|
+
raw: @reviewer.generate(**generate_prompt),
|
|
50
|
+
kind: 'generate result',
|
|
51
|
+
expected: 'generate',
|
|
52
|
+
repair_stage: 'generate'
|
|
53
|
+
)
|
|
54
|
+
end
|
|
50
55
|
summary = generate_result['summary']
|
|
51
56
|
candidates = Array(generate_result['candidates'])
|
|
52
|
-
@logger.info("Pipeline generate pass completed with #{candidates.size} candidate(s)"
|
|
57
|
+
@logger.info("Pipeline generate pass completed with #{candidates.size} candidate(s) " \
|
|
58
|
+
"(model=#{@reviewer.answered_model('generate')})")
|
|
53
59
|
candidates = check_candidates(context: context, changes: changes, candidates: candidates)
|
|
54
60
|
|
|
55
61
|
accepted = critique ? maybe_critique(context: context, candidates: candidates) : skip_critique(candidates)
|
|
@@ -60,7 +66,8 @@ module Aireview
|
|
|
60
66
|
accepted,
|
|
61
67
|
summary: summary,
|
|
62
68
|
coverage: context.coverage,
|
|
63
|
-
fallback_models: @reviewer.fallback_models
|
|
69
|
+
fallback_models: @reviewer.fallback_models,
|
|
70
|
+
critique_weaker: critique && @reviewer.critique_weaker?
|
|
64
71
|
)
|
|
65
72
|
end
|
|
66
73
|
|
|
@@ -85,10 +92,15 @@ module Aireview
|
|
|
85
92
|
generate_temperature: @config.generate_temperature,
|
|
86
93
|
critique_model: @config.critique_model,
|
|
87
94
|
critique_temperature: @config.critique_temperature,
|
|
88
|
-
generate_fallbacks: @config.fallback_names(
|
|
89
|
-
critique_fallbacks: critique ? @config.fallback_names(
|
|
90
|
-
|
|
95
|
+
generate_fallbacks: @config.fallback_names('generate'),
|
|
96
|
+
critique_fallbacks: critique ? @config.fallback_names('critique') : [],
|
|
97
|
+
sources: setting_sources(critique),
|
|
98
|
+
config_paths: @config.layer_paths,
|
|
99
|
+
warnings: @config.warnings,
|
|
100
|
+
critique_rule: critique ? @config.routing.rule : nil,
|
|
101
|
+
api_keys: @config.api_key_counts(critique ? STAGES : ['generate']),
|
|
91
102
|
time_budget: @config.llm_time_budget,
|
|
103
|
+
overloaded_quarantine: @config.overloaded_quarantine,
|
|
92
104
|
coverage: context.coverage,
|
|
93
105
|
sizes: context.sizes
|
|
94
106
|
}
|
|
@@ -96,8 +108,37 @@ module Aireview
|
|
|
96
108
|
|
|
97
109
|
private
|
|
98
110
|
|
|
99
|
-
#
|
|
100
|
-
|
|
111
|
+
# Where the model, provider and reserves of a stage came from, for --dry-run.
|
|
112
|
+
def setting_sources(critique)
|
|
113
|
+
(critique ? %w[generate critique] : %w[generate]).to_h do |stage|
|
|
114
|
+
[stage.to_sym, {
|
|
115
|
+
model: @config.stage_model_source(stage),
|
|
116
|
+
provider: @config.stage_provider_source(stage),
|
|
117
|
+
fallbacks: @config.stage_fallbacks_source(stage)
|
|
118
|
+
}]
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# A stage is a request, parsing and one repair by the same model. An
|
|
123
|
+
# invalid result after the repair, like a repair with no requests left,
|
|
124
|
+
# excludes the model for the stage, and the stage starts over on the next
|
|
125
|
+
# one — with the original request. API errors pass through: the router
|
|
126
|
+
# handles them with reserves, and exhausted routes are exhausted for a
|
|
127
|
+
# restart too.
|
|
128
|
+
def run_stage(stage)
|
|
129
|
+
loop do
|
|
130
|
+
return yield
|
|
131
|
+
rescue ParseError => e
|
|
132
|
+
reason = "#{e.is_a?(RepairImpossibleError) ? 'repair impossible' : 'invalid result'}: #{e.message}"
|
|
133
|
+
excluded = @reviewer.exclude_answered_model(stage: stage, reason: reason)
|
|
134
|
+
raise unless excluded
|
|
135
|
+
|
|
136
|
+
@logger.warn("Pipeline #{stage}: restarting on the next model after #{excluded} (#{e.message})")
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Anchoring is checked against the diff the model saw, before Critique:
|
|
141
|
+
# it gets the notes, the report gets the reset line and the missing-quote mark.
|
|
101
142
|
def check_candidates(context:, changes:, candidates:)
|
|
102
143
|
CandidateChecker.new(
|
|
103
144
|
changes: changes,
|
|
@@ -112,8 +153,8 @@ module Aireview
|
|
|
112
153
|
candidates
|
|
113
154
|
end
|
|
114
155
|
|
|
115
|
-
#
|
|
116
|
-
#
|
|
156
|
+
# Critique has nothing to filter without candidates: the LLM request
|
|
157
|
+
# would waste quota and time.
|
|
117
158
|
def maybe_critique(context:, candidates:)
|
|
118
159
|
return skip_critique(candidates, 'no candidates') if candidates.empty?
|
|
119
160
|
|
|
@@ -125,16 +166,18 @@ module Aireview
|
|
|
125
166
|
candidates_json = JSON.pretty_generate(candidates)
|
|
126
167
|
candidates_by_id = index_candidates_by_id(candidates)
|
|
127
168
|
critique_prompt = @context_builder.build_critique_prompt(context, candidates_json: candidates_json)
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
169
|
+
critique_result = run_stage('critique') do
|
|
170
|
+
parse_with_repair(
|
|
171
|
+
raw: @reviewer.critique(**critique_prompt),
|
|
172
|
+
kind: 'critique result',
|
|
173
|
+
expected: 'critique',
|
|
174
|
+
repair_stage: 'critique',
|
|
175
|
+
critique_candidate_ids: candidates_by_id.keys
|
|
176
|
+
)
|
|
177
|
+
end
|
|
136
178
|
verdicts = Array(critique_result['verdicts'])
|
|
137
|
-
@logger.info("Pipeline critique pass completed with #{verdicts.size} verdict(s)"
|
|
179
|
+
@logger.info("Pipeline critique pass completed with #{verdicts.size} verdict(s) " \
|
|
180
|
+
"(model=#{@reviewer.answered_model('critique')})")
|
|
138
181
|
apply_critique_verdicts(
|
|
139
182
|
verdicts: verdicts,
|
|
140
183
|
candidates_by_id: candidates_by_id
|
|
@@ -187,61 +230,11 @@ module Aireview
|
|
|
187
230
|
end
|
|
188
231
|
|
|
189
232
|
def parse_expected_result(raw, expected, critique_candidate_ids: nil)
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
case expected
|
|
193
|
-
when :generate
|
|
194
|
-
parsed = normalize_generate_result(parsed)
|
|
195
|
-
when :critique
|
|
196
|
-
parsed = normalize_critique_result(parsed, critique_candidate_ids: critique_candidate_ids)
|
|
197
|
-
else
|
|
198
|
-
raise ArgumentError, "Unknown expected JSON schema: #{expected.inspect}"
|
|
199
|
-
end
|
|
200
|
-
|
|
201
|
-
parsed
|
|
202
|
-
end
|
|
203
|
-
|
|
204
|
-
def strip_code_fences(text)
|
|
205
|
-
stripped = text.to_s.strip
|
|
206
|
-
return stripped unless stripped.start_with?('```')
|
|
207
|
-
|
|
208
|
-
stripped
|
|
209
|
-
.sub(/\A```[[:alnum:]_-]*[ \t]*\r?\n?/, '')
|
|
210
|
-
.sub(/\r?\n?```[ \t]*\z/, '')
|
|
211
|
-
.strip
|
|
212
|
-
end
|
|
213
|
-
|
|
214
|
-
def normalize_generate_result(parsed)
|
|
215
|
-
parsed = {'summary' => nil, 'candidates' => parsed} if parsed.is_a?(Array)
|
|
216
|
-
validate_generate_result_shape!(parsed)
|
|
217
|
-
parsed['summary'] = nil unless parsed.key?('summary')
|
|
218
|
-
candidate_ids = parsed['candidates'].map { |candidate| normalize_id(value(candidate, 'id')) }
|
|
219
|
-
validate_identifiers!(
|
|
220
|
-
candidate_ids,
|
|
221
|
-
missing_message: 'each generate candidate must include a non-empty id',
|
|
222
|
-
duplicate_prefix: 'duplicate generate candidate ids'
|
|
223
|
-
)
|
|
224
|
-
|
|
225
|
-
parsed
|
|
226
|
-
end
|
|
227
|
-
|
|
228
|
-
def normalize_critique_result(parsed, critique_candidate_ids:)
|
|
229
|
-
validate_critique_result_shape!(parsed)
|
|
230
|
-
verdicts = parsed['verdicts']
|
|
231
|
-
verdict_ids = verdicts.map { |verdict| normalize_id(value(verdict, 'id')) }
|
|
232
|
-
validate_identifiers!(
|
|
233
|
-
verdict_ids,
|
|
234
|
-
missing_message: 'each verdict must include a non-empty id',
|
|
235
|
-
duplicate_prefix: 'duplicate verdict ids'
|
|
236
|
-
)
|
|
237
|
-
validate_expected_verdict_ids!(verdict_ids, critique_candidate_ids)
|
|
238
|
-
verdicts.each { |verdict| validate_verdict!(verdict) }
|
|
239
|
-
|
|
240
|
-
parsed
|
|
233
|
+
@parser.parse(raw, expected: expected, critique_candidate_ids: critique_candidate_ids)
|
|
241
234
|
end
|
|
242
235
|
|
|
243
236
|
def repair_json(raw:, kind:, expected:, stage:, critique_candidate_ids: nil)
|
|
244
|
-
schema = expected ==
|
|
237
|
+
schema = expected == 'critique' ? ReviewSchemas.critique : ReviewSchemas.generate
|
|
245
238
|
user_prompt = <<~PROMPT
|
|
246
239
|
The previous #{kind} response was invalid.
|
|
247
240
|
|
|
@@ -254,19 +247,28 @@ module Aireview
|
|
|
254
247
|
Invalid response:
|
|
255
248
|
#{raw}
|
|
256
249
|
PROMPT
|
|
257
|
-
if stage ==
|
|
250
|
+
if stage == 'critique' && critique_candidate_ids
|
|
258
251
|
user_prompt << "\nExpected candidate ids: #{critique_candidate_ids.join(', ')}\n"
|
|
259
252
|
end
|
|
260
253
|
|
|
261
254
|
@logger.info("Pipeline #{stage} repair started for #{kind}")
|
|
262
|
-
prompt =
|
|
263
|
-
if stage ==
|
|
264
|
-
@reviewer.critique(**prompt)
|
|
255
|
+
prompt = repair_prompt(stage, user_prompt)
|
|
256
|
+
if stage == 'critique'
|
|
257
|
+
@reviewer.critique(**prompt, pinned: true)
|
|
265
258
|
else
|
|
266
|
-
@reviewer.generate(**prompt)
|
|
259
|
+
@reviewer.generate(**prompt, pinned: true)
|
|
267
260
|
end
|
|
268
261
|
end
|
|
269
262
|
|
|
263
|
+
# A repair that does not fit the stage limit is an invalid result of this
|
|
264
|
+
# model, not a size error of the original request: the stage moves to
|
|
265
|
+
# the next model with the original prompt.
|
|
266
|
+
def repair_prompt(stage, user_prompt)
|
|
267
|
+
@context_builder.check_stage_size!(stage, REPAIR_SYSTEM_PROMPT, user_prompt)
|
|
268
|
+
rescue ContextBudgetError => e
|
|
269
|
+
raise ParseError, "repair request does not fit the stage limit: #{e.message}"
|
|
270
|
+
end
|
|
271
|
+
|
|
270
272
|
def index_candidates_by_id(candidates)
|
|
271
273
|
candidates.each_with_object({}) do |candidate, result|
|
|
272
274
|
next unless candidate.is_a?(Hash)
|
|
@@ -46,6 +46,7 @@ module Aireview
|
|
|
46
46
|
diff_unavailable: 'diff not available',
|
|
47
47
|
section_list: 'Truncated sections',
|
|
48
48
|
fallback_used: 'Fallback model used',
|
|
49
|
+
critique_weaker: 'Critique ran on a model weaker than Generate: the findings were checked less strictly.',
|
|
49
50
|
quote_missing: 'quote not found in the diff'
|
|
50
51
|
},
|
|
51
52
|
'ru' => {
|
|
@@ -72,6 +73,7 @@ module Aireview
|
|
|
72
73
|
diff_unavailable: 'дифф недоступен',
|
|
73
74
|
section_list: 'Усечённые секции',
|
|
74
75
|
fallback_used: 'Использована резервная модель',
|
|
76
|
+
critique_weaker: 'Критика выполнена моделью слабее generate: замечания проверены менее строго.',
|
|
75
77
|
quote_missing: 'цитата не найдена в диффе'
|
|
76
78
|
}
|
|
77
79
|
}.freeze
|
|
@@ -80,11 +82,11 @@ module Aireview
|
|
|
80
82
|
@labels = LABELS.fetch(language.to_s) { LABELS.fetch(DEFAULT_LANGUAGE) }
|
|
81
83
|
end
|
|
82
84
|
|
|
83
|
-
# coverage:
|
|
84
|
-
# result
|
|
85
|
-
#
|
|
86
|
-
#
|
|
87
|
-
def render(accepted, summary:, coverage: nil, fallback_models: {})
|
|
85
|
+
# coverage: the truncation facts from the pipeline, not the model's text.
|
|
86
|
+
# result is still about the findings; incomplete coverage is written
|
|
87
|
+
# next to it so that the result line does not read as "everything was
|
|
88
|
+
# checked".
|
|
89
|
+
def render(accepted, summary:, coverage: nil, fallback_models: {}, critique_weaker: false)
|
|
88
90
|
mismatches, important = select_findings(Array(accepted))
|
|
89
91
|
result = mismatches.empty? && important.empty? ? 'ok' : 'needs attention'
|
|
90
92
|
|
|
@@ -104,16 +106,16 @@ module Aireview
|
|
|
104
106
|
## #{label(:result)}
|
|
105
107
|
|
|
106
108
|
#{result}#{partial_note(coverage)}
|
|
107
|
-
#{coverage_block(coverage)}#{fallback_note(fallback_models)}
|
|
109
|
+
#{coverage_block(coverage)}#{fallback_note(fallback_models)}#{weaker_note(critique_weaker)}
|
|
108
110
|
#{label(:disclaimer)}
|
|
109
111
|
MARKDOWN
|
|
110
112
|
end
|
|
111
113
|
|
|
112
114
|
private
|
|
113
115
|
|
|
114
|
-
#
|
|
115
|
-
#
|
|
116
|
-
#
|
|
116
|
+
# First select what is shown at all, then the section limits and only
|
|
117
|
+
# then the total limit: a finding that cannot be shown because of its
|
|
118
|
+
# category or a section limit must not take a slot in the total.
|
|
117
119
|
def select_findings(accepted)
|
|
118
120
|
sorted = sorted_findings(accepted)
|
|
119
121
|
mismatches = sorted.select { |finding| category(finding) == 'task_mismatch' }.first(MISMATCH_LIMIT)
|
|
@@ -181,8 +183,14 @@ module Aireview
|
|
|
181
183
|
"\n## #{label(:not_reviewed)}\n\n#{lines.join("\n")}\n"
|
|
182
184
|
end
|
|
183
185
|
|
|
184
|
-
#
|
|
185
|
-
#
|
|
186
|
+
# A key switch stays in the logs; a model switch is visible to the
|
|
187
|
+
# reader, because a fallback model may review less well than the primary.
|
|
188
|
+
def weaker_note(critique_weaker)
|
|
189
|
+
return '' unless critique_weaker
|
|
190
|
+
|
|
191
|
+
"\n#{label(:critique_weaker)}\n"
|
|
192
|
+
end
|
|
193
|
+
|
|
186
194
|
def fallback_note(fallback_models)
|
|
187
195
|
return '' if fallback_models.nil? || fallback_models.empty?
|
|
188
196
|
|
data/lib/aireview/reviewer.rb
CHANGED
|
@@ -1,142 +1,70 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
require_relative 'errors'
|
|
3
3
|
require_relative 'output_schemas'
|
|
4
|
-
require_relative 'utils'
|
|
5
4
|
require_relative 'llm_router'
|
|
6
|
-
|
|
5
|
+
require_relative 'llm_client'
|
|
7
6
|
|
|
8
7
|
module Aireview
|
|
8
|
+
# The review stages on top of the router: Generate and Critique with their
|
|
9
|
+
# own schemas and temperatures. LlmClient makes the request, LlmRouter
|
|
10
|
+
# walks the models.
|
|
9
11
|
class Reviewer
|
|
10
12
|
attr_reader :router
|
|
11
13
|
|
|
12
|
-
def initialize(config:, logger: Logger.new($stderr), router: nil)
|
|
14
|
+
def initialize(config:, logger: Logger.new($stderr), router: nil, client: nil)
|
|
13
15
|
@config = config
|
|
14
16
|
@logger = logger
|
|
15
17
|
@router = router || LlmRouter.new(config: config, logger: logger)
|
|
16
|
-
@
|
|
18
|
+
@client = client || LlmClient.new(config: config, logger: logger)
|
|
17
19
|
end
|
|
18
20
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
)
|
|
21
|
+
# pinned — a request only to the model that answered last in the stage
|
|
22
|
+
# (the repair of its own JSON): its failure is RepairImpossibleError.
|
|
23
|
+
def generate(system_prompt:, user_prompt:, pinned: false)
|
|
24
|
+
prompt = LlmClient::Prompt.new(stage: 'generate', system: system_prompt, user: user_prompt,
|
|
25
|
+
temperature: @config.generate_temperature, schema: GenerateOutputSchema)
|
|
26
|
+
call_llm(prompt, pinned: pinned)
|
|
26
27
|
end
|
|
27
28
|
|
|
28
|
-
def critique(system_prompt:, user_prompt:)
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
user: user_prompt,
|
|
33
|
-
options: {temperature: @config.critique_temperature, schema: CritiqueOutputSchema}
|
|
34
|
-
)
|
|
29
|
+
def critique(system_prompt:, user_prompt:, pinned: false)
|
|
30
|
+
prompt = LlmClient::Prompt.new(stage: 'critique', system: system_prompt, user: user_prompt,
|
|
31
|
+
temperature: @config.critique_temperature, schema: CritiqueOutputSchema)
|
|
32
|
+
call_llm(prompt, pinned: pinned)
|
|
35
33
|
end
|
|
36
34
|
|
|
37
|
-
#
|
|
35
|
+
# Stages answered by a fallback model, for the report line.
|
|
38
36
|
def fallback_models
|
|
39
37
|
@router.fallback_models
|
|
40
38
|
end
|
|
41
39
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
require 'ruby_llm'
|
|
46
|
-
|
|
47
|
-
@config.require_llm_configuration!
|
|
48
|
-
response = @router.call(stage: stage, request_chars: system.length + user.length) do |route, timeout|
|
|
49
|
-
perform_llm_request(
|
|
50
|
-
context: llm_context(stage, route),
|
|
51
|
-
stage: stage,
|
|
52
|
-
system: system,
|
|
53
|
-
user: user,
|
|
54
|
-
options: options.merge(model: route.candidate.model, provider: route.candidate.provider, timeout: timeout)
|
|
55
|
-
)
|
|
56
|
-
end
|
|
57
|
-
response.content
|
|
58
|
-
rescue LoadError => e
|
|
59
|
-
@logger.error("LLM #{stage} setup failed: #{e.message}")
|
|
60
|
-
raise ConfigError, "Missing dependency: #{e.message}"
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
def perform_llm_request(context:, stage:, system:, user:, options:)
|
|
64
|
-
model = options[:model]
|
|
65
|
-
temperature = options[:temperature]
|
|
66
|
-
@logger.info("LLM #{stage} request started (model=#{model}, temperature=#{temperature})")
|
|
67
|
-
chat = build_chat(context: context, stage: stage, model: model, provider: options[:provider])
|
|
68
|
-
chat = configure_reasoning(chat: chat, model: model, provider: options[:provider])
|
|
69
|
-
.with_temperature(temperature.to_f)
|
|
70
|
-
.with_schema(options[:schema])
|
|
71
|
-
chat.with_instructions(system)
|
|
72
|
-
response = Timeout.timeout(options[:timeout]) { chat.ask(user) }
|
|
73
|
-
@logger.info("LLM #{stage} request completed (model=#{model})")
|
|
74
|
-
response
|
|
75
|
-
rescue Timeout::Error
|
|
76
|
-
@logger.warn("LLM #{stage} request timed out after #{options[:timeout].round} seconds (model=#{model})")
|
|
77
|
-
raise
|
|
78
|
-
end
|
|
79
|
-
|
|
80
|
-
def configure_reasoning(chat:, model:, provider:)
|
|
81
|
-
return chat unless provider == 'ollama' && model.start_with?('gpt-oss:')
|
|
82
|
-
|
|
83
|
-
chat.with_thinking(effort: :low)
|
|
40
|
+
# The model that answered last in the stage with its place in the chain, for the log.
|
|
41
|
+
def answered_model(stage)
|
|
42
|
+
@router.answered(stage)
|
|
84
43
|
end
|
|
85
44
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
@logger.warn(
|
|
90
|
-
"LLM #{stage}: model not found in RubyLLM registry; " \
|
|
91
|
-
"using fallback with incomplete model metadata " \
|
|
92
|
-
"(model=#{model}, provider=#{provider})"
|
|
93
|
-
)
|
|
94
|
-
context.chat(model: model, provider: provider.to_sym, assume_model_exists: true)
|
|
45
|
+
# Critique ran on a model below Generate in the pool (allow_weaker).
|
|
46
|
+
def critique_weaker?
|
|
47
|
+
@router.critique_weaker?
|
|
95
48
|
end
|
|
96
49
|
|
|
97
|
-
#
|
|
98
|
-
#
|
|
99
|
-
def
|
|
100
|
-
|
|
101
|
-
@llm_contexts[cache_key] ||= build_llm_context(route.candidate.provider.to_s, route.key)
|
|
50
|
+
# An invalid result: the model is excluded for the stage, the next
|
|
51
|
+
# request of the stage goes to another. Returns the excluded model or nil.
|
|
52
|
+
def exclude_answered_model(stage:, reason:)
|
|
53
|
+
@router.exclude_answered(stage: stage, reason: reason)
|
|
102
54
|
end
|
|
103
55
|
|
|
104
|
-
|
|
105
|
-
# умолчанию 3) превратили бы каждую нашу попытку в четыре HTTP-запроса и
|
|
106
|
-
# жгли бы квоту до того, как ошибка дойдёт до классификатора.
|
|
107
|
-
def build_llm_context(provider, api_key)
|
|
108
|
-
RubyLLM.context do |ruby_config|
|
|
109
|
-
configure_http_proxy(ruby_config)
|
|
110
|
-
ruby_config.request_timeout = @config.llm_timeout.to_f
|
|
111
|
-
ruby_config.max_retries = 0
|
|
112
|
-
configure_provider(ruby_config, provider, api_key)
|
|
113
|
-
end
|
|
114
|
-
end
|
|
115
|
-
|
|
116
|
-
def configure_http_proxy(ruby_config)
|
|
117
|
-
return unless Aireview::Utils.present?(@config.llm_http_proxy)
|
|
118
|
-
|
|
119
|
-
ruby_config.http_proxy = @config.llm_http_proxy
|
|
120
|
-
end
|
|
56
|
+
private
|
|
121
57
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
when 'ollama'
|
|
129
|
-
ruby_config.ollama_api_base = @config.ollama_api_base
|
|
130
|
-
else
|
|
131
|
-
raise ConfigError, "Unsupported LLM provider: #{provider.inspect}"
|
|
58
|
+
# A pinned route giving up is not an API error for the pipeline but
|
|
59
|
+
# "repair impossible": the same fate as an invalid result.
|
|
60
|
+
def call_llm(prompt, pinned:)
|
|
61
|
+
response = @router.call(stage: prompt.stage, request_chars: prompt.chars, pinned: pinned) do |route, timeout|
|
|
62
|
+
@client.request(prompt, candidate: route.candidate, key: route.key, key_index: route.key_index,
|
|
63
|
+
timeout: timeout)
|
|
132
64
|
end
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
ruby_config.public_send("#{provider}_api_key=", api_key)
|
|
137
|
-
return unless Aireview::Utils.present?(@config.llm_api_base)
|
|
138
|
-
|
|
139
|
-
ruby_config.public_send("#{provider}_api_base=", @config.llm_api_base)
|
|
65
|
+
response.content
|
|
66
|
+
rescue RouteExhaustedError => e
|
|
67
|
+
raise RepairImpossibleError, e.message
|
|
140
68
|
end
|
|
141
69
|
end
|
|
142
70
|
end
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require_relative 'errors'
|
|
3
|
+
require_relative 'utils'
|
|
4
|
+
require_relative 'model_candidate'
|
|
5
|
+
require_relative 'stages'
|
|
6
|
+
|
|
7
|
+
module Aireview
|
|
8
|
+
# A routing plan of independent chains: every stage has its own primary
|
|
9
|
+
# model and reserves in walking order. Critique does not depend on which
|
|
10
|
+
# model answered in Generate.
|
|
11
|
+
#
|
|
12
|
+
# The plan interface (ModelPool implements it too): chain(stage),
|
|
13
|
+
# critique_chain(after:), primary(stage), weaker?, signature, rule,
|
|
14
|
+
# pool?, pool_stage?, pool_member?, start_used?, warnings.
|
|
15
|
+
class StageChains
|
|
16
|
+
# settings — per stage: provider, model, fallbacks (the raw list from the
|
|
17
|
+
# config), max_prompt_chars. only_primary keeps one model.
|
|
18
|
+
def self.build(settings, only_primary: false)
|
|
19
|
+
chains = settings.to_h do |stage, stage_settings|
|
|
20
|
+
[stage.to_s, stage_chain(stage.to_s, stage_settings, only_primary: only_primary)]
|
|
21
|
+
end
|
|
22
|
+
new(chains)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def self.stage_chain(stage, settings, only_primary:)
|
|
26
|
+
primary = ModelCandidate.new(
|
|
27
|
+
provider: settings.fetch(:provider).to_s,
|
|
28
|
+
model: settings[:model],
|
|
29
|
+
max_prompt_chars: settings.fetch(:max_prompt_chars)
|
|
30
|
+
)
|
|
31
|
+
return [primary] if only_primary
|
|
32
|
+
|
|
33
|
+
[primary, *fallbacks(stage, Array(settings[:fallbacks]), primary)]
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# A reserve without a provider inherits the stage provider, without a
|
|
37
|
+
# limit its limit.
|
|
38
|
+
def self.fallbacks(stage, items, primary)
|
|
39
|
+
items.each_with_index.map do |item, index|
|
|
40
|
+
item = ModelCandidate.parse_item(item)
|
|
41
|
+
name = "llm.#{stage}.fallbacks[#{index}]"
|
|
42
|
+
raise ConfigError, "#{name} must be a model name or a hash with model" unless item.is_a?(Hash)
|
|
43
|
+
raise ConfigError, "#{name}.model is required" if Aireview::Utils.blank?(item['model'])
|
|
44
|
+
|
|
45
|
+
limit = item['max_prompt_chars']
|
|
46
|
+
ModelCandidate.new(
|
|
47
|
+
provider: (item['provider'] || primary.provider).to_s,
|
|
48
|
+
model: item['model'].to_s,
|
|
49
|
+
max_prompt_chars: limit.nil? ? primary.max_prompt_chars : positive_limit(limit, name)
|
|
50
|
+
)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def self.positive_limit(value, name)
|
|
55
|
+
integer = Integer(value, exception: false)
|
|
56
|
+
return integer if integer&.positive?
|
|
57
|
+
|
|
58
|
+
raise ConfigError, "#{name}.max_prompt_chars must be a positive integer, got #{value.inspect}"
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def initialize(chains)
|
|
62
|
+
@chains = chains
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def chain(stage)
|
|
66
|
+
@chains.fetch(stage.to_s) { raise ArgumentError, "unknown LLM stage #{stage.inspect}" }
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def stage?(stage)
|
|
70
|
+
@chains.key?(stage.to_s)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def critique_chain(after:)
|
|
74
|
+
chain('critique')
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def primary(stage)
|
|
78
|
+
chain(stage).first
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def weaker?(_critique_candidate, _generate_candidate)
|
|
82
|
+
false
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def signature
|
|
86
|
+
nil
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def rule
|
|
90
|
+
nil
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def pool?
|
|
94
|
+
false
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def pool_stage?(_stage)
|
|
98
|
+
false
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def pool_member?(_model)
|
|
102
|
+
false
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def start_used?(_stage)
|
|
106
|
+
false
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def warnings
|
|
110
|
+
[]
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
data/lib/aireview/utils.rb
CHANGED
|
@@ -14,5 +14,34 @@ module Aireview
|
|
|
14
14
|
def presence(value)
|
|
15
15
|
present?(value) ? value.to_s.strip : nil
|
|
16
16
|
end
|
|
17
|
+
|
|
18
|
+
# Merges settings by key: nested hashes merge, everything else (arrays
|
|
19
|
+
# included) is replaced by the right side as a whole.
|
|
20
|
+
def deep_merge(left, right)
|
|
21
|
+
left.merge(right) do |_, old_value, new_value|
|
|
22
|
+
if old_value.is_a?(Hash) && new_value.is_a?(Hash)
|
|
23
|
+
deep_merge(old_value, new_value)
|
|
24
|
+
else
|
|
25
|
+
new_value
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Hash keys are strings at any depth: YAML, env and LLM answers arrive in
|
|
31
|
+
# different shapes, everything downstream works with one.
|
|
32
|
+
def normalize_hash(value)
|
|
33
|
+
case value
|
|
34
|
+
when Hash
|
|
35
|
+
value.each_with_object({}) { |(key, inner), result| result[key.to_s] = normalize_hash(inner) }
|
|
36
|
+
when Array
|
|
37
|
+
value.map { |item| normalize_hash(item) }
|
|
38
|
+
else
|
|
39
|
+
value
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def dig(data, *keys)
|
|
44
|
+
keys.reduce(data) { |accumulator, key| accumulator.is_a?(Hash) ? accumulator[key] : nil }
|
|
45
|
+
end
|
|
17
46
|
end
|
|
18
47
|
end
|