lex-assessor 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +12 -0
- data/lex-assessor.gemspec +36 -0
- data/lib/legion/extensions/assessor/actors/assessor.rb +25 -0
- data/lib/legion/extensions/assessor/helpers/classifier.rb +100 -0
- data/lib/legion/extensions/assessor/helpers/config_templates.rb +86 -0
- data/lib/legion/extensions/assessor/helpers/dedup.rb +55 -0
- data/lib/legion/extensions/assessor/runners/assessor.rb +175 -0
- data/lib/legion/extensions/assessor/transport/exchanges/assessor.rb +21 -0
- data/lib/legion/extensions/assessor/transport/queues/assessor.rb +29 -0
- data/lib/legion/extensions/assessor/version.rb +9 -0
- data/lib/legion/extensions/assessor.rb +26 -0
- metadata +155 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: ee974378d5cc8bd9b93e356c359aeb0437d5c31614561706590c9d11c57e1ab4
|
|
4
|
+
data.tar.gz: 657ee8494e2ecb23cb8d6f7863eed2813f9634872b31edb186c36344768bd24a
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: c3486bb38f5412de35cf35403e0b61af44c5593d8d5e945a8fa1ae23672d7eb41634ad96d5b361bfc114b6ce79937aca2a78d059a55ed7378829be70c13ed522
|
|
7
|
+
data.tar.gz: 035cc91d9340c17de9d0c1abb11fcd4961c6e1882c10d7bed8541d4c9b92f22cc5036b87fb3fa6f920d14807bcb549183e6b5974d9ae36a49307edf5a6542e41
|
data/Gemfile
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
source 'https://rubygems.org'
|
|
4
|
+
|
|
5
|
+
gemspec
|
|
6
|
+
|
|
7
|
+
gem 'rspec', '~> 3.13'
|
|
8
|
+
gem 'rspec_junit_formatter'
|
|
9
|
+
gem 'rubocop', '~> 1.75', require: false
|
|
10
|
+
gem 'rubocop-legion', '~> 0.1', require: false
|
|
11
|
+
gem 'rubocop-rspec', require: false
|
|
12
|
+
gem 'simplecov', require: false
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'lib/legion/extensions/assessor/version'
|
|
4
|
+
|
|
5
|
+
Gem::Specification.new do |spec|
|
|
6
|
+
spec.name = 'lex-assessor'
|
|
7
|
+
spec.version = Legion::Extensions::Assessor::VERSION
|
|
8
|
+
spec.authors = ['Esity']
|
|
9
|
+
spec.email = ['matthewdiverson@gmail.com']
|
|
10
|
+
|
|
11
|
+
spec.summary = 'Fleet pipeline intake for LegionIO'
|
|
12
|
+
spec.description = 'Fleet pipeline intake: classify, deduplicate, and route work items'
|
|
13
|
+
spec.homepage = 'https://github.com/LegionIO/lex-assessor'
|
|
14
|
+
spec.license = 'MIT'
|
|
15
|
+
spec.required_ruby_version = '>= 3.4'
|
|
16
|
+
|
|
17
|
+
spec.metadata['homepage_uri'] = spec.homepage
|
|
18
|
+
spec.metadata['source_code_uri'] = 'https://github.com/LegionIO/lex-assessor'
|
|
19
|
+
spec.metadata['changelog_uri'] = 'https://github.com/LegionIO/lex-assessor'
|
|
20
|
+
spec.metadata['documentation_uri'] = 'https://github.com/LegionIO/lex-assessor'
|
|
21
|
+
spec.metadata['bug_tracker_uri'] = 'https://github.com/LegionIO/lex-assessor/issues'
|
|
22
|
+
spec.metadata['rubygems_mfa_required'] = 'true'
|
|
23
|
+
|
|
24
|
+
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
|
25
|
+
Dir.glob('{lib}/**/*') + %w[lex-assessor.gemspec Gemfile]
|
|
26
|
+
end
|
|
27
|
+
spec.require_paths = ['lib']
|
|
28
|
+
|
|
29
|
+
spec.add_dependency 'legion-cache', '>= 1.3.11'
|
|
30
|
+
spec.add_dependency 'legion-crypt', '>= 1.4.9'
|
|
31
|
+
spec.add_dependency 'legion-data', '>= 1.4.17'
|
|
32
|
+
spec.add_dependency 'legion-json', '>= 1.2.1'
|
|
33
|
+
spec.add_dependency 'legion-logging', '>= 1.3.2'
|
|
34
|
+
spec.add_dependency 'legion-settings', '>= 1.3.14'
|
|
35
|
+
spec.add_dependency 'legion-transport', '>= 1.3.9'
|
|
36
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
return unless defined?(Legion::Extensions::Actors::Subscription)
|
|
4
|
+
|
|
5
|
+
module Legion
|
|
6
|
+
module Extensions
|
|
7
|
+
module Assessor
|
|
8
|
+
module Actor
|
|
9
|
+
class Assessor < Legion::Extensions::Actors::Subscription
|
|
10
|
+
def runner_function
|
|
11
|
+
'assess'
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def check_subtask?
|
|
15
|
+
true
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def generate_task?
|
|
19
|
+
false
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Legion
|
|
4
|
+
module Extensions
|
|
5
|
+
module Assessor
|
|
6
|
+
module Helpers
|
|
7
|
+
module Classifier
|
|
8
|
+
extend self
|
|
9
|
+
|
|
10
|
+
CLASSIFICATION_SCHEMA = {
|
|
11
|
+
type: :object,
|
|
12
|
+
properties: {
|
|
13
|
+
priority: { type: :string, enum: %w[low medium high critical] },
|
|
14
|
+
complexity: {
|
|
15
|
+
type: :string,
|
|
16
|
+
enum: %w[trivial_fix simple_bug moderate_feature complex_feature critical_production background]
|
|
17
|
+
},
|
|
18
|
+
work_type: { type: :string, enum: %w[bug_fix feature refactor test docs dependency security] },
|
|
19
|
+
language: { type: :string },
|
|
20
|
+
estimated_difficulty: { type: :number, minimum: 0.0, maximum: 1.0 }
|
|
21
|
+
},
|
|
22
|
+
required: %w[priority complexity work_type estimated_difficulty]
|
|
23
|
+
}.freeze
|
|
24
|
+
|
|
25
|
+
def classify(work_item:)
|
|
26
|
+
prompt = build_prompt(work_item: work_item)
|
|
27
|
+
result = Legion::LLM::Prompt.extract(
|
|
28
|
+
prompt,
|
|
29
|
+
schema: classification_schema,
|
|
30
|
+
tools: [],
|
|
31
|
+
intent: { capability: :moderate },
|
|
32
|
+
caller: { extension: 'lex-assessor', operation: 'classify' },
|
|
33
|
+
agent: {
|
|
34
|
+
id: 'fleet:assessor',
|
|
35
|
+
name: 'Fleet Assessor',
|
|
36
|
+
type: :autonomous,
|
|
37
|
+
goal: work_item[:title]
|
|
38
|
+
},
|
|
39
|
+
tracing: {
|
|
40
|
+
trace_id: work_item[:work_item_id],
|
|
41
|
+
correlation_id: work_item[:source_ref]
|
|
42
|
+
}
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
normalize_result(json_parse(result.message[:content]))
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def classification_schema
|
|
49
|
+
CLASSIFICATION_SCHEMA
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def build_prompt(work_item:)
|
|
53
|
+
<<~PROMPT
|
|
54
|
+
Classify this work item for an autonomous coding pipeline.
|
|
55
|
+
|
|
56
|
+
Source: #{work_item[:source]}
|
|
57
|
+
Reference: #{work_item[:source_ref]}
|
|
58
|
+
Title: #{work_item[:title]}
|
|
59
|
+
Description: #{work_item[:description]}
|
|
60
|
+
|
|
61
|
+
Classify the complexity, priority, work type, primary language, and estimated difficulty (0.0 to 1.0).
|
|
62
|
+
|
|
63
|
+
Complexity levels:
|
|
64
|
+
- trivial_fix: one-line change, obvious fix
|
|
65
|
+
- simple_bug: clear bug, single file, straightforward
|
|
66
|
+
- moderate_feature: multi-file change, needs planning
|
|
67
|
+
- complex_feature: architectural change, multiple subsystems
|
|
68
|
+
- critical_production: production outage, needs immediate fix with max capability
|
|
69
|
+
- background: low-priority chore, minimal review needed
|
|
70
|
+
PROMPT
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def difficulty_to_capability(difficulty)
|
|
74
|
+
case difficulty
|
|
75
|
+
when 0.0...0.3 then :basic
|
|
76
|
+
when 0.3...0.6 then :moderate
|
|
77
|
+
else :reasoning
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
private
|
|
82
|
+
|
|
83
|
+
def json_parse(content)
|
|
84
|
+
Legion::JSON.load(content) # rubocop:disable Legion/HelperMigration/DirectJson
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def normalize_result(result)
|
|
88
|
+
{
|
|
89
|
+
priority: (result[:priority] || :medium).to_sym,
|
|
90
|
+
complexity: (result[:complexity] || :moderate_feature).to_sym,
|
|
91
|
+
work_type: (result[:work_type] || :bug_fix).to_sym,
|
|
92
|
+
language: (result[:language] || :ruby).to_sym,
|
|
93
|
+
estimated_difficulty: (result[:estimated_difficulty] || 0.5).to_f.clamp(0.0, 1.0)
|
|
94
|
+
}
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Legion
|
|
4
|
+
module Extensions
|
|
5
|
+
module Assessor
|
|
6
|
+
module Helpers
|
|
7
|
+
module ConfigTemplates
|
|
8
|
+
extend self
|
|
9
|
+
|
|
10
|
+
TEMPLATES = {
|
|
11
|
+
trivial_fix: {
|
|
12
|
+
planning: { enabled: false, solvers: 1, validators: 0, max_iterations: 1 }.freeze,
|
|
13
|
+
implementation: { solvers: 1, validators: 0, max_iterations: 1, models: nil }.freeze,
|
|
14
|
+
validation: { enabled: false, run_tests: true, run_lint: true, security_scan: false, adversarial_review: false, reviewer_models: nil }.freeze,
|
|
15
|
+
feedback: { drain_enabled: false, max_drain_rounds: 1, summarize_after: 1 }.freeze
|
|
16
|
+
}.freeze,
|
|
17
|
+
simple_bug: {
|
|
18
|
+
planning: { enabled: false, solvers: 1, validators: 0, max_iterations: 1 }.freeze,
|
|
19
|
+
implementation: { solvers: 1, validators: 1, max_iterations: 3, models: nil }.freeze,
|
|
20
|
+
validation: { enabled: true, run_tests: true, run_lint: true, security_scan: true, adversarial_review: true, reviewer_models: nil }.freeze,
|
|
21
|
+
feedback: { drain_enabled: true, max_drain_rounds: 2, summarize_after: 2 }.freeze
|
|
22
|
+
}.freeze,
|
|
23
|
+
moderate_feature: {
|
|
24
|
+
planning: { enabled: true, solvers: 1, validators: 1, max_iterations: 2 }.freeze,
|
|
25
|
+
implementation: { solvers: 1, validators: 3, max_iterations: 5, models: nil }.freeze,
|
|
26
|
+
validation: { enabled: true, run_tests: true, run_lint: true, security_scan: true, adversarial_review: true, reviewer_models: nil }.freeze,
|
|
27
|
+
feedback: { drain_enabled: true, max_drain_rounds: 3, summarize_after: 2 }.freeze
|
|
28
|
+
}.freeze,
|
|
29
|
+
complex_feature: {
|
|
30
|
+
planning: { enabled: true, solvers: 2, validators: 2, max_iterations: 3 }.freeze,
|
|
31
|
+
implementation: { solvers: 2, validators: 3, max_iterations: 8, models: nil }.freeze,
|
|
32
|
+
validation: { enabled: true, run_tests: true, run_lint: true, security_scan: true, adversarial_review: true, reviewer_models: nil }.freeze,
|
|
33
|
+
feedback: { drain_enabled: true, max_drain_rounds: 3, summarize_after: 2 }.freeze
|
|
34
|
+
}.freeze,
|
|
35
|
+
critical_production: {
|
|
36
|
+
# planning skipped intentionally: critical items need immediate action, not planning overhead
|
|
37
|
+
planning: { enabled: false, solvers: 1, validators: 0, max_iterations: 1 }.freeze,
|
|
38
|
+
implementation: { solvers: 3, validators: 3, max_iterations: 10, models: nil }.freeze,
|
|
39
|
+
validation: { enabled: true, run_tests: true, run_lint: true, security_scan: true, adversarial_review: true, reviewer_models: nil }.freeze,
|
|
40
|
+
feedback: { drain_enabled: true, max_drain_rounds: 3, summarize_after: 2 }.freeze
|
|
41
|
+
}.freeze,
|
|
42
|
+
background: {
|
|
43
|
+
# validation.enabled=false is the gate; run_tests/run_lint pre-configured if re-enabled via override
|
|
44
|
+
planning: { enabled: false, solvers: 1, validators: 0, max_iterations: 1 }.freeze,
|
|
45
|
+
implementation: { solvers: 1, validators: 0, max_iterations: 2, models: nil }.freeze,
|
|
46
|
+
validation: { enabled: false, run_tests: true, run_lint: true, security_scan: false, adversarial_review: false, reviewer_models: nil }.freeze,
|
|
47
|
+
feedback: { drain_enabled: false, max_drain_rounds: 1, summarize_after: 1 }.freeze
|
|
48
|
+
}.freeze
|
|
49
|
+
}.freeze
|
|
50
|
+
|
|
51
|
+
DEFAULT_TEMPLATE = :moderate_feature
|
|
52
|
+
|
|
53
|
+
def for_classification(complexity:)
|
|
54
|
+
key = complexity.to_sym
|
|
55
|
+
template = TEMPLATES.fetch(key) { TEMPLATES[DEFAULT_TEMPLATE] }
|
|
56
|
+
deep_dup(template)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def merge_with_overrides(template:, overrides:)
|
|
60
|
+
return deep_dup(template) if overrides.nil? || overrides.empty?
|
|
61
|
+
|
|
62
|
+
deep_merge(template, overrides)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
private
|
|
66
|
+
|
|
67
|
+
def deep_dup(hash)
|
|
68
|
+
hash.each_with_object({}) do |(k, v), result|
|
|
69
|
+
result[k] = v.is_a?(Hash) ? deep_dup(v) : v
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def deep_merge(base, overlay)
|
|
74
|
+
base.merge(overlay) do |_key, old_val, new_val|
|
|
75
|
+
if old_val.is_a?(Hash) && new_val.is_a?(Hash)
|
|
76
|
+
deep_merge(old_val, new_val)
|
|
77
|
+
else
|
|
78
|
+
new_val
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'digest'
|
|
4
|
+
|
|
5
|
+
module Legion
|
|
6
|
+
module Extensions
|
|
7
|
+
module Assessor
|
|
8
|
+
module Helpers
|
|
9
|
+
module Dedup
|
|
10
|
+
extend self
|
|
11
|
+
|
|
12
|
+
CACHE_PREFIX = 'fleet:active:'
|
|
13
|
+
|
|
14
|
+
def fingerprint(source:, source_ref:, title:)
|
|
15
|
+
Digest::SHA256.hexdigest("#{source}:#{source_ref}:#{title}")
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def duplicate?(fingerprint:)
|
|
19
|
+
!cache_get(cache_key(fingerprint)).nil?
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def default_ttl
|
|
23
|
+
Legion::Settings.dig(:fleet, :cache, :dedup_ttl_seconds)&.to_i || 86_400
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def claim!(fingerprint:, ttl: default_ttl)
|
|
27
|
+
cache_set_nx(cache_key(fingerprint), '1', ttl: ttl)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def release!(fingerprint:)
|
|
31
|
+
cache_delete(cache_key(fingerprint))
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def cache_key(fingerprint)
|
|
35
|
+
"#{CACHE_PREFIX}#{fingerprint}"
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
def cache_get(key)
|
|
41
|
+
Legion::Cache.get(key) # rubocop:disable Legion/HelperMigration/DirectCache
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def cache_set_nx(key, value, ttl:)
|
|
45
|
+
Legion::Cache.set_nx(key, value, ttl: ttl)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def cache_delete(key)
|
|
49
|
+
Legion::Cache.delete(key) # rubocop:disable Legion/HelperMigration/DirectCache
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Legion
|
|
4
|
+
module Extensions
|
|
5
|
+
module Assessor
|
|
6
|
+
module Runners
|
|
7
|
+
module Assessor
|
|
8
|
+
extend self
|
|
9
|
+
|
|
10
|
+
def assess(results: nil, work_item: nil, args: nil, **)
|
|
11
|
+
results = json_load_string(results) if results.is_a?(String)
|
|
12
|
+
work_item ||= results&.dig(:work_item) || args&.dig(:work_item)
|
|
13
|
+
raise ArgumentError, "work_item is nil in #{__method__}" if work_item.nil?
|
|
14
|
+
|
|
15
|
+
fingerprint = Helpers::Dedup.fingerprint(
|
|
16
|
+
source: work_item[:source],
|
|
17
|
+
source_ref: work_item[:source_ref],
|
|
18
|
+
title: work_item[:title]
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
return { success: false, reason: :duplicate, work_item_id: work_item[:work_item_id] } unless Helpers::Dedup.claim!(fingerprint: fingerprint)
|
|
22
|
+
|
|
23
|
+
classification = Helpers::Classifier.classify(work_item: work_item)
|
|
24
|
+
template = Helpers::ConfigTemplates.for_classification(complexity: classification[:complexity])
|
|
25
|
+
source_overrides = extract_source_overrides(work_item[:config])
|
|
26
|
+
merged_config = Helpers::ConfigTemplates.merge_with_overrides(template: template, overrides: source_overrides)
|
|
27
|
+
|
|
28
|
+
work_item = work_item.merge(
|
|
29
|
+
config: work_item[:config].merge(merged_config).merge(
|
|
30
|
+
priority: classification[:priority],
|
|
31
|
+
complexity: classification[:complexity],
|
|
32
|
+
estimated_difficulty: classification[:estimated_difficulty]
|
|
33
|
+
),
|
|
34
|
+
resolved_max_iterations: merged_config.dig(:implementation, :max_iterations) || 5,
|
|
35
|
+
pipeline: work_item[:pipeline].merge(
|
|
36
|
+
stage: 'assessed',
|
|
37
|
+
trace: work_item[:pipeline][:trace] + [build_trace_entry]
|
|
38
|
+
)
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
{ success: true, work_item: work_item }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def escalate(results: nil, work_item: nil, args: nil, **)
|
|
45
|
+
results = json_load_string(results) if results.is_a?(String)
|
|
46
|
+
work_item ||= results&.dig(:work_item) || args&.dig(:work_item)
|
|
47
|
+
raise ArgumentError, "work_item is nil in #{__method__}" if work_item.nil?
|
|
48
|
+
|
|
49
|
+
notify_external_systems(work_item)
|
|
50
|
+
|
|
51
|
+
# Step 4: Clear dedup cache key
|
|
52
|
+
fingerprint = Helpers::Dedup.fingerprint(
|
|
53
|
+
source: work_item[:source],
|
|
54
|
+
source_ref: work_item[:source_ref],
|
|
55
|
+
title: work_item[:title]
|
|
56
|
+
)
|
|
57
|
+
Helpers::Dedup.release!(fingerprint: fingerprint)
|
|
58
|
+
|
|
59
|
+
# Step 5: Clear Redis refs (payload, context, worktree)
|
|
60
|
+
cleanup_redis_refs(work_item[:work_item_id])
|
|
61
|
+
|
|
62
|
+
# Step 6: Return escalated work item
|
|
63
|
+
work_item = work_item.merge(
|
|
64
|
+
pipeline: work_item[:pipeline].merge(
|
|
65
|
+
stage: 'escalated',
|
|
66
|
+
trace: work_item[:pipeline][:trace] + [build_trace_entry(stage: 'escalation')]
|
|
67
|
+
)
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
{ success: true, work_item: work_item }
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
private
|
|
74
|
+
|
|
75
|
+
def json_load_string(str)
|
|
76
|
+
Legion::JSON.load(str) # rubocop:disable Legion/HelperMigration/DirectJson
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def extract_source_overrides(config)
|
|
80
|
+
return nil if config.nil?
|
|
81
|
+
|
|
82
|
+
overrides = {}
|
|
83
|
+
%i[planning implementation validation feedback workspace context tracing safety selection escalation].each do |key|
|
|
84
|
+
overrides[key] = config[key] if config[key].is_a?(Hash) && !config[key].values.all?(&:nil?)
|
|
85
|
+
end
|
|
86
|
+
overrides.empty? ? nil : overrides
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def build_trace_entry(stage: 'assessor')
|
|
90
|
+
{
|
|
91
|
+
stage: stage,
|
|
92
|
+
node: node_name,
|
|
93
|
+
started_at: ::Time.now.utc.iso8601,
|
|
94
|
+
completed_at: ::Time.now.utc.iso8601,
|
|
95
|
+
token_usage: nil
|
|
96
|
+
}
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def notify_external_systems(work_item)
|
|
100
|
+
owner = work_item.dig(:repo, :owner)
|
|
101
|
+
repo = work_item.dig(:repo, :name)
|
|
102
|
+
issue_number = work_item[:source_ref].to_s.scan(/\d+/).last&.to_i
|
|
103
|
+
|
|
104
|
+
if defined?(Legion::Extensions::Github::Runners::Labels)
|
|
105
|
+
Legion::Extensions::Github::Runners::Labels.add_labels_to_issue(
|
|
106
|
+
owner: owner, repo: repo, issue_number: issue_number, labels: ['fleet:escalated']
|
|
107
|
+
)
|
|
108
|
+
else
|
|
109
|
+
Legion::Logging.warn('fleet:assessor: lex-github not loaded; skipping issue label') # rubocop:disable Legion/HelperMigration/DirectLogging
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
if defined?(Legion::Extensions::Github::Runners::Comments)
|
|
113
|
+
Legion::Extensions::Github::Runners::Comments.create_issue_comment(
|
|
114
|
+
owner: owner, repo: repo, issue_number: issue_number,
|
|
115
|
+
body: build_escalation_summary(work_item)
|
|
116
|
+
)
|
|
117
|
+
else
|
|
118
|
+
Legion::Logging.warn('fleet:assessor: lex-github not loaded; skipping escalation comment') # rubocop:disable Legion/HelperMigration/DirectLogging
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
submit_to_approval_queue(work_item)
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def submit_to_approval_queue(work_item)
|
|
125
|
+
if defined?(Legion::Extensions::Audit::Runners::ApprovalQueue)
|
|
126
|
+
Legion::Extensions::Audit::Runners::ApprovalQueue.submit(
|
|
127
|
+
approval_type: 'fleet.escalation',
|
|
128
|
+
payload: { work_item: work_item.merge(pipeline: work_item[:pipeline].merge(attempt: 0, resumed: true)) },
|
|
129
|
+
requester_id: 'fleet:assessor',
|
|
130
|
+
resume_routing_key: 'lex.developer.runners.developer.incorporate_feedback',
|
|
131
|
+
resume_exchange: 'lex.developer'
|
|
132
|
+
)
|
|
133
|
+
else
|
|
134
|
+
Legion::Logging.warn('fleet:assessor: lex-audit not loaded; skipping ApprovalQueue submission') # rubocop:disable Legion/HelperMigration/DirectLogging
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def build_escalation_summary(work_item)
|
|
139
|
+
attempt = work_item.dig(:pipeline, :attempt) || 0
|
|
140
|
+
review = work_item.dig(:pipeline, :review_result) || {}
|
|
141
|
+
feedback = Array(work_item.dig(:pipeline, :feedback_history))
|
|
142
|
+
|
|
143
|
+
<<~SUMMARY
|
|
144
|
+
## Fleet Pipeline -- Escalation
|
|
145
|
+
|
|
146
|
+
**Work Item**: #{work_item[:work_item_id]}
|
|
147
|
+
**Title**: #{work_item[:title]}
|
|
148
|
+
**Attempts**: #{attempt}
|
|
149
|
+
**Last Verdict**: #{review[:verdict] || 'N/A'}
|
|
150
|
+
|
|
151
|
+
### Feedback History
|
|
152
|
+
#{feedback.map { |f| "- Round #{f[:round]}: #{Array(f[:issues]).join(', ')}" }.join("\n")}
|
|
153
|
+
|
|
154
|
+
This work item has been escalated for human review after exceeding the maximum iteration count.
|
|
155
|
+
SUMMARY
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def cleanup_redis_refs(work_item_id)
|
|
159
|
+
cache_delete("fleet:payload:#{work_item_id}")
|
|
160
|
+
cache_delete("fleet:context:#{work_item_id}")
|
|
161
|
+
cache_delete("fleet:worktree:#{work_item_id}")
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def cache_delete(key)
|
|
165
|
+
Legion::Cache.delete(key) # rubocop:disable Legion/HelperMigration/DirectCache
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def node_name
|
|
169
|
+
Legion::Settings.dig(:node, :name) || 'unknown'
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Legion
|
|
4
|
+
module Extensions
|
|
5
|
+
module Assessor
|
|
6
|
+
module Transport
|
|
7
|
+
module Exchanges
|
|
8
|
+
class Assessor < Legion::Transport::Exchange
|
|
9
|
+
def exchange_name
|
|
10
|
+
'lex.assessor'
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def exchange_options
|
|
14
|
+
{ type: 'topic', durable: true }
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Legion
|
|
4
|
+
module Extensions
|
|
5
|
+
module Assessor
|
|
6
|
+
module Transport
|
|
7
|
+
module Queues
|
|
8
|
+
class Assessor < Legion::Transport::Queue
|
|
9
|
+
def queue_name
|
|
10
|
+
'lex.assessor.runners.assessor'
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def queue_options
|
|
14
|
+
{ durable: true }
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def routing_key
|
|
18
|
+
'lex.assessor.runners.assessor.#'
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def exchange
|
|
22
|
+
Exchanges::Assessor
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'assessor/version'
|
|
4
|
+
require_relative 'assessor/helpers/config_templates'
|
|
5
|
+
require_relative 'assessor/helpers/dedup'
|
|
6
|
+
require_relative 'assessor/helpers/classifier'
|
|
7
|
+
require_relative 'assessor/runners/assessor'
|
|
8
|
+
|
|
9
|
+
if defined?(Legion::Transport::Exchange)
|
|
10
|
+
require_relative 'assessor/transport/exchanges/assessor'
|
|
11
|
+
require_relative 'assessor/transport/queues/assessor'
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
require_relative 'assessor/actors/assessor'
|
|
15
|
+
|
|
16
|
+
module Legion
|
|
17
|
+
module Extensions
|
|
18
|
+
module Assessor
|
|
19
|
+
extend Legion::Extensions::Core if defined?(Legion::Extensions::Core)
|
|
20
|
+
|
|
21
|
+
def self.llm_required?
|
|
22
|
+
true
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: lex-assessor
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Esity
|
|
8
|
+
bindir: bin
|
|
9
|
+
cert_chain: []
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
+
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: legion-cache
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - ">="
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: 1.3.11
|
|
19
|
+
type: :runtime
|
|
20
|
+
prerelease: false
|
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
22
|
+
requirements:
|
|
23
|
+
- - ">="
|
|
24
|
+
- !ruby/object:Gem::Version
|
|
25
|
+
version: 1.3.11
|
|
26
|
+
- !ruby/object:Gem::Dependency
|
|
27
|
+
name: legion-crypt
|
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
|
29
|
+
requirements:
|
|
30
|
+
- - ">="
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: 1.4.9
|
|
33
|
+
type: :runtime
|
|
34
|
+
prerelease: false
|
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - ">="
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: 1.4.9
|
|
40
|
+
- !ruby/object:Gem::Dependency
|
|
41
|
+
name: legion-data
|
|
42
|
+
requirement: !ruby/object:Gem::Requirement
|
|
43
|
+
requirements:
|
|
44
|
+
- - ">="
|
|
45
|
+
- !ruby/object:Gem::Version
|
|
46
|
+
version: 1.4.17
|
|
47
|
+
type: :runtime
|
|
48
|
+
prerelease: false
|
|
49
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
50
|
+
requirements:
|
|
51
|
+
- - ">="
|
|
52
|
+
- !ruby/object:Gem::Version
|
|
53
|
+
version: 1.4.17
|
|
54
|
+
- !ruby/object:Gem::Dependency
|
|
55
|
+
name: legion-json
|
|
56
|
+
requirement: !ruby/object:Gem::Requirement
|
|
57
|
+
requirements:
|
|
58
|
+
- - ">="
|
|
59
|
+
- !ruby/object:Gem::Version
|
|
60
|
+
version: 1.2.1
|
|
61
|
+
type: :runtime
|
|
62
|
+
prerelease: false
|
|
63
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
64
|
+
requirements:
|
|
65
|
+
- - ">="
|
|
66
|
+
- !ruby/object:Gem::Version
|
|
67
|
+
version: 1.2.1
|
|
68
|
+
- !ruby/object:Gem::Dependency
|
|
69
|
+
name: legion-logging
|
|
70
|
+
requirement: !ruby/object:Gem::Requirement
|
|
71
|
+
requirements:
|
|
72
|
+
- - ">="
|
|
73
|
+
- !ruby/object:Gem::Version
|
|
74
|
+
version: 1.3.2
|
|
75
|
+
type: :runtime
|
|
76
|
+
prerelease: false
|
|
77
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
78
|
+
requirements:
|
|
79
|
+
- - ">="
|
|
80
|
+
- !ruby/object:Gem::Version
|
|
81
|
+
version: 1.3.2
|
|
82
|
+
- !ruby/object:Gem::Dependency
|
|
83
|
+
name: legion-settings
|
|
84
|
+
requirement: !ruby/object:Gem::Requirement
|
|
85
|
+
requirements:
|
|
86
|
+
- - ">="
|
|
87
|
+
- !ruby/object:Gem::Version
|
|
88
|
+
version: 1.3.14
|
|
89
|
+
type: :runtime
|
|
90
|
+
prerelease: false
|
|
91
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
92
|
+
requirements:
|
|
93
|
+
- - ">="
|
|
94
|
+
- !ruby/object:Gem::Version
|
|
95
|
+
version: 1.3.14
|
|
96
|
+
- !ruby/object:Gem::Dependency
|
|
97
|
+
name: legion-transport
|
|
98
|
+
requirement: !ruby/object:Gem::Requirement
|
|
99
|
+
requirements:
|
|
100
|
+
- - ">="
|
|
101
|
+
- !ruby/object:Gem::Version
|
|
102
|
+
version: 1.3.9
|
|
103
|
+
type: :runtime
|
|
104
|
+
prerelease: false
|
|
105
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
106
|
+
requirements:
|
|
107
|
+
- - ">="
|
|
108
|
+
- !ruby/object:Gem::Version
|
|
109
|
+
version: 1.3.9
|
|
110
|
+
description: 'Fleet pipeline intake: classify, deduplicate, and route work items'
|
|
111
|
+
email:
|
|
112
|
+
- matthewdiverson@gmail.com
|
|
113
|
+
executables: []
|
|
114
|
+
extensions: []
|
|
115
|
+
extra_rdoc_files: []
|
|
116
|
+
files:
|
|
117
|
+
- Gemfile
|
|
118
|
+
- lex-assessor.gemspec
|
|
119
|
+
- lib/legion/extensions/assessor.rb
|
|
120
|
+
- lib/legion/extensions/assessor/actors/assessor.rb
|
|
121
|
+
- lib/legion/extensions/assessor/helpers/classifier.rb
|
|
122
|
+
- lib/legion/extensions/assessor/helpers/config_templates.rb
|
|
123
|
+
- lib/legion/extensions/assessor/helpers/dedup.rb
|
|
124
|
+
- lib/legion/extensions/assessor/runners/assessor.rb
|
|
125
|
+
- lib/legion/extensions/assessor/transport/exchanges/assessor.rb
|
|
126
|
+
- lib/legion/extensions/assessor/transport/queues/assessor.rb
|
|
127
|
+
- lib/legion/extensions/assessor/version.rb
|
|
128
|
+
homepage: https://github.com/LegionIO/lex-assessor
|
|
129
|
+
licenses:
|
|
130
|
+
- MIT
|
|
131
|
+
metadata:
|
|
132
|
+
homepage_uri: https://github.com/LegionIO/lex-assessor
|
|
133
|
+
source_code_uri: https://github.com/LegionIO/lex-assessor
|
|
134
|
+
changelog_uri: https://github.com/LegionIO/lex-assessor
|
|
135
|
+
documentation_uri: https://github.com/LegionIO/lex-assessor
|
|
136
|
+
bug_tracker_uri: https://github.com/LegionIO/lex-assessor/issues
|
|
137
|
+
rubygems_mfa_required: 'true'
|
|
138
|
+
rdoc_options: []
|
|
139
|
+
require_paths:
|
|
140
|
+
- lib
|
|
141
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
142
|
+
requirements:
|
|
143
|
+
- - ">="
|
|
144
|
+
- !ruby/object:Gem::Version
|
|
145
|
+
version: '3.4'
|
|
146
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
147
|
+
requirements:
|
|
148
|
+
- - ">="
|
|
149
|
+
- !ruby/object:Gem::Version
|
|
150
|
+
version: '0'
|
|
151
|
+
requirements: []
|
|
152
|
+
rubygems_version: 3.6.9
|
|
153
|
+
specification_version: 4
|
|
154
|
+
summary: Fleet pipeline intake for LegionIO
|
|
155
|
+
test_files: []
|