legionio 1.4.86 → 1.4.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -0
- data/CHANGELOG.md +17 -0
- data/lib/legion/alerts.rb +12 -1
- data/lib/legion/api/acp.rb +101 -0
- data/lib/legion/api.rb +2 -0
- data/lib/legion/extensions/actors/every.rb +3 -1
- data/lib/legion/extensions/actors/fingerprint.rb +45 -0
- data/lib/legion/extensions/actors/poll.rb +37 -28
- data/lib/legion/extensions/actors/subscription.rb +21 -6
- data/lib/legion/ingress.rb +15 -7
- data/lib/legion/service.rb +11 -1
- data/lib/legion/telemetry/open_inference.rb +177 -0
- data/lib/legion/telemetry/safety_metrics.rb +166 -0
- data/lib/legion/telemetry.rb +3 -0
- data/lib/legion/version.rb +1 -1
- metadata +5 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a9ecd48fa743747aea34dee8175be57e1280ca6f6b65b84687ec315506694263
|
|
4
|
+
data.tar.gz: 5301e5709b614c062d71ef04ef27fc582490dd7b7ce83e8f3fe4d6a3509801d2
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2032e446fc6e104e56db67b9afb116b3587d56da1414bddc42e23c4f3562901825f36fad0fcf74014f48f3c3f0a6438de0ecc4cf0ca38601d3ef088baf534f7b
|
|
7
|
+
data.tar.gz: a609212ab03b9ff708f7f805f8d312c6b0e724d939bbdc52d5abde40127f525c1c2a47fed10ad2289010c2643a393c68ea9c6155ed2cda72b2f0f8f754bdb16b
|
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
# Legion Changelog
|
|
2
2
|
|
|
3
|
+
## [1.4.87] - 2026-03-20
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
- OpenInference OTel span instrumentation (Ingress TOOL spans, Subscription CHAIN spans)
|
|
7
|
+
- SafetyMetrics sliding window module with 4 default alert rules
|
|
8
|
+
- Fingerprint mixin for actor skip-if-unchanged optimization
|
|
9
|
+
|
|
3
10
|
## [1.4.86] - 2026-03-20
|
|
4
11
|
|
|
5
12
|
### Added
|
|
@@ -44,6 +51,10 @@
|
|
|
44
51
|
## [1.4.81] - 2026-03-20
|
|
45
52
|
|
|
46
53
|
### Added
|
|
54
|
+
- Fingerprint mixin for actor skip-if-unchanged optimization (`Legion::Extensions::Actors::Fingerprint`)
|
|
55
|
+
- SHA256-based `skip_or_run` gate: skips execution when `fingerprint_source` is stable
|
|
56
|
+
- Fingerprint integrated into `Every` and `Poll` actors via `include Fingerprint`
|
|
57
|
+
- Extracted `poll_cycle` method from Poll actor for clean separation of timer vs logic
|
|
47
58
|
- `legion eval experiments` subcommand: list all experiment runs with status and summary
|
|
48
59
|
- `legion eval promote --experiment NAME --tag TAG` subcommand: tag a prompt version for production via lex-prompt
|
|
49
60
|
- `legion eval compare --run1 NAME --run2 NAME` subcommand: side-by-side diff of two experiment runs
|
|
@@ -52,6 +63,12 @@
|
|
|
52
63
|
## [1.4.80] - 2026-03-20
|
|
53
64
|
|
|
54
65
|
### Added
|
|
66
|
+
- OpenInference OTel span helpers (LLM, EMBEDDING, TOOL, CHAIN, EVALUATOR, AGENT)
|
|
67
|
+
- SafetyMetrics sliding window module for behavioral monitoring
|
|
68
|
+
- 4 safety alert rules (action burst, scope escalation spike, probe detected, confidence collapse)
|
|
69
|
+
- OpenInference TOOL spans in Ingress.run
|
|
70
|
+
- OpenInference CHAIN spans in Subscription actor dispatch
|
|
71
|
+
- SafetyMetrics wired into service boot sequence
|
|
55
72
|
- `legion eval run` CLI subcommand for CI/CD threshold-based eval gating
|
|
56
73
|
- `--dataset`, `--threshold`, `--evaluator`, `--exit-code` options on `eval run`
|
|
57
74
|
- JSON report output to stdout with per-row scores, summary, and timestamp
|
data/lib/legion/alerts.rb
CHANGED
|
@@ -13,7 +13,18 @@ module Legion
|
|
|
13
13
|
condition: { count_threshold: 10, window_seconds: 60 }, severity: 'warning',
|
|
14
14
|
channels: %w[events log], cooldown_seconds: 300 },
|
|
15
15
|
{ name: 'budget_exceeded', event_pattern: 'finops.budget_exceeded', severity: 'warning',
|
|
16
|
-
channels: %w[events log], cooldown_seconds: 3600 }
|
|
16
|
+
channels: %w[events log], cooldown_seconds: 3600 },
|
|
17
|
+
{ name: 'safety_action_burst', event_pattern: 'ingress.received',
|
|
18
|
+
condition: { count_threshold: 100, window_seconds: 60 }, severity: 'warning',
|
|
19
|
+
channels: %w[events log], cooldown_seconds: 300 },
|
|
20
|
+
{ name: 'safety_scope_escalation_spike', event_pattern: 'rbac.deny',
|
|
21
|
+
condition: { count_threshold: 5, window_seconds: 300 }, severity: 'critical',
|
|
22
|
+
channels: %w[events log], cooldown_seconds: 300 },
|
|
23
|
+
{ name: 'safety_probe_detected', event_pattern: 'privatecore.probe_detected', severity: 'critical',
|
|
24
|
+
channels: %w[events log], cooldown_seconds: 0 },
|
|
25
|
+
{ name: 'safety_confidence_collapse', event_pattern: 'synapse.confidence_update',
|
|
26
|
+
condition: { count_threshold: 3, window_seconds: 300 }, severity: 'warning',
|
|
27
|
+
channels: %w[events log], cooldown_seconds: 300 }
|
|
17
28
|
].freeze
|
|
18
29
|
|
|
19
30
|
class Engine
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Legion
|
|
4
|
+
class API < Sinatra::Base
|
|
5
|
+
module Routes
|
|
6
|
+
module Acp
|
|
7
|
+
def self.registered(app)
|
|
8
|
+
app.get '/.well-known/agent.json' do
|
|
9
|
+
card = build_agent_card
|
|
10
|
+
content_type :json
|
|
11
|
+
Legion::JSON.dump(card)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
app.post '/api/acp/tasks' do
|
|
15
|
+
body = parse_request_body
|
|
16
|
+
payload = (body[:input] || {}).transform_keys(&:to_sym)
|
|
17
|
+
|
|
18
|
+
result = Legion::Ingress.run(
|
|
19
|
+
payload: payload,
|
|
20
|
+
runner_class: body[:runner_class],
|
|
21
|
+
function: body[:function],
|
|
22
|
+
source: 'acp'
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
json_response({ task_id: result[:task_id], status: 'queued' }, status_code: 202)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
app.get '/api/acp/tasks/:id' do
|
|
29
|
+
task = find_task(params[:id])
|
|
30
|
+
halt 404, json_error(404, 'Task not found') unless task
|
|
31
|
+
|
|
32
|
+
json_response({
|
|
33
|
+
task_id: task[:id],
|
|
34
|
+
status: translate_status(task[:status]),
|
|
35
|
+
output: { data: task[:result] },
|
|
36
|
+
created_at: task[:created_at]&.to_s,
|
|
37
|
+
completed_at: task[:completed_at]&.to_s
|
|
38
|
+
})
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
app.delete '/api/acp/tasks/:id' do
|
|
42
|
+
halt 501, json_error(501, 'Task cancellation not implemented')
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
helpers do
|
|
49
|
+
def build_agent_card
|
|
50
|
+
name = begin
|
|
51
|
+
Legion::Settings[:client][:name]
|
|
52
|
+
rescue StandardError
|
|
53
|
+
'legion'
|
|
54
|
+
end
|
|
55
|
+
port = begin
|
|
56
|
+
settings.port || 4567
|
|
57
|
+
rescue StandardError
|
|
58
|
+
4567
|
|
59
|
+
end
|
|
60
|
+
{
|
|
61
|
+
name: name,
|
|
62
|
+
description: 'LegionIO digital worker',
|
|
63
|
+
url: "http://#{request.host}:#{port}/api/acp",
|
|
64
|
+
version: '2.0',
|
|
65
|
+
protocol: 'acp/1.0',
|
|
66
|
+
capabilities: discover_capabilities,
|
|
67
|
+
authentication: { schemes: ['bearer'] },
|
|
68
|
+
defaultInputModes: ['text/plain', 'application/json'],
|
|
69
|
+
defaultOutputModes: ['text/plain', 'application/json']
|
|
70
|
+
}
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def discover_capabilities
|
|
74
|
+
if defined?(Legion::Extensions::Mesh::Helpers::Registry)
|
|
75
|
+
Legion::Extensions::Mesh::Helpers::Registry.new.capabilities.keys.map(&:to_s)
|
|
76
|
+
else
|
|
77
|
+
[]
|
|
78
|
+
end
|
|
79
|
+
rescue StandardError
|
|
80
|
+
[]
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def find_task(id)
|
|
84
|
+
return nil unless defined?(Legion::Data)
|
|
85
|
+
|
|
86
|
+
Legion::Data::Model::Task[id.to_i]&.values
|
|
87
|
+
rescue StandardError
|
|
88
|
+
nil
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def translate_status(status)
|
|
92
|
+
case status&.to_s
|
|
93
|
+
when /completed/ then 'completed'
|
|
94
|
+
when /exception|failed/ then 'failed'
|
|
95
|
+
when /queued|scheduled/ then 'queued'
|
|
96
|
+
else 'in_progress'
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
data/lib/legion/api.rb
CHANGED
|
@@ -37,6 +37,7 @@ require_relative 'api/catalog'
|
|
|
37
37
|
require_relative 'api/org_chart'
|
|
38
38
|
require_relative 'api/workflow'
|
|
39
39
|
require_relative 'api/governance'
|
|
40
|
+
require_relative 'api/acp'
|
|
40
41
|
|
|
41
42
|
module Legion
|
|
42
43
|
class API < Sinatra::Base
|
|
@@ -118,6 +119,7 @@ module Legion
|
|
|
118
119
|
register Routes::ExtensionCatalog
|
|
119
120
|
register Routes::OrgChart
|
|
120
121
|
register Routes::Governance
|
|
122
|
+
register Routes::Acp
|
|
121
123
|
|
|
122
124
|
use Legion::Rbac::Middleware if defined?(Legion::Rbac::Middleware)
|
|
123
125
|
|
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative 'base'
|
|
4
|
+
require_relative 'fingerprint'
|
|
4
5
|
|
|
5
6
|
module Legion
|
|
6
7
|
module Extensions
|
|
7
8
|
module Actors
|
|
8
9
|
class Every
|
|
9
10
|
include Legion::Extensions::Actors::Base
|
|
11
|
+
include Legion::Extensions::Actors::Fingerprint
|
|
10
12
|
|
|
11
13
|
def initialize(**_opts)
|
|
12
14
|
@timer = Concurrent::TimerTask.new(execution_interval: time, run_now: run_now?) do
|
|
13
|
-
use_runner? ? runner : manual
|
|
15
|
+
skip_or_run { use_runner? ? runner : manual }
|
|
14
16
|
end
|
|
15
17
|
|
|
16
18
|
@timer.execute
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'digest'
|
|
4
|
+
|
|
5
|
+
module Legion
|
|
6
|
+
module Extensions
|
|
7
|
+
module Actors
|
|
8
|
+
module Fingerprint
|
|
9
|
+
def skip_if_unchanged?
|
|
10
|
+
false
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def fingerprint_source
|
|
14
|
+
bucket = respond_to?(:time) ? time.to_i : 60
|
|
15
|
+
bucket = 1 if bucket < 1
|
|
16
|
+
(Time.now.utc.to_i / bucket).to_s
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def compute_fingerprint
|
|
20
|
+
Digest::SHA256.hexdigest(fingerprint_source.to_s)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def unchanged?
|
|
24
|
+
return false if @last_fingerprint.nil?
|
|
25
|
+
|
|
26
|
+
compute_fingerprint == @last_fingerprint
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def store_fingerprint!
|
|
30
|
+
@last_fingerprint = compute_fingerprint
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def skip_or_run
|
|
34
|
+
if skip_if_unchanged? && unchanged?
|
|
35
|
+
Legion::Logging.debug "#{self.class} skipped: fingerprint unchanged (#{@last_fingerprint[0, 8]}...)"
|
|
36
|
+
return
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
yield
|
|
40
|
+
store_fingerprint!
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative 'base'
|
|
4
|
+
require_relative 'fingerprint'
|
|
4
5
|
require 'time'
|
|
5
6
|
|
|
6
7
|
module Legion
|
|
@@ -8,38 +9,13 @@ module Legion
|
|
|
8
9
|
module Actors
|
|
9
10
|
class Poll
|
|
10
11
|
include Legion::Extensions::Actors::Base
|
|
12
|
+
include Legion::Extensions::Actors::Fingerprint
|
|
11
13
|
|
|
12
|
-
def initialize
|
|
14
|
+
def initialize
|
|
13
15
|
log.debug "Starting timer for #{self.class} with #{{ execution_interval: time, run_now: run_now?,
|
|
14
16
|
check_subtask: check_subtask? }}"
|
|
15
17
|
@timer = Concurrent::TimerTask.new(execution_interval: time, run_now: run_now?) do
|
|
16
|
-
|
|
17
|
-
log.debug "Running #{self.class}"
|
|
18
|
-
old_result = Legion::Cache.get(cache_name)
|
|
19
|
-
log.debug "Cached value for #{self.class}: #{old_result}"
|
|
20
|
-
results = Legion::JSON.load(Legion::JSON.dump(manual))
|
|
21
|
-
Legion::Cache.set(cache_name, results, time * 2)
|
|
22
|
-
|
|
23
|
-
unless old_result.nil?
|
|
24
|
-
results[:diff] = Hashdiff.diff(results, old_result, numeric_tolerance: 0.0, array_path: false) do |_path, obj1, obj2|
|
|
25
|
-
if int_percentage_normalize.positive? && obj1.is_a?(Integer) && obj2.is_a?(Integer)
|
|
26
|
-
obj1.between?(obj2 * (1 - int_percentage_normalize), obj2 * (1 + int_percentage_normalize))
|
|
27
|
-
end
|
|
28
|
-
end
|
|
29
|
-
results[:changed] = results[:diff].any?
|
|
30
|
-
|
|
31
|
-
Legion::Logging.info results[:diff] if results[:changed]
|
|
32
|
-
Legion::Transport::Messages::CheckSubtask.new(runner_class: runner_class.to_s,
|
|
33
|
-
function: runner_function,
|
|
34
|
-
result: results,
|
|
35
|
-
type: 'poll_result',
|
|
36
|
-
polling: true).publish
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
sleep_time = 1 - (Time.now - t1)
|
|
40
|
-
sleep(sleep_time) if sleep_time.positive?
|
|
41
|
-
log.debug("#{self.class} result: #{results}")
|
|
42
|
-
results
|
|
18
|
+
skip_or_run { poll_cycle }
|
|
43
19
|
rescue StandardError => e
|
|
44
20
|
Legion::Logging.fatal e.message
|
|
45
21
|
Legion::Logging.fatal e.backtrace
|
|
@@ -50,6 +26,39 @@ check_subtask: check_subtask? }}"
|
|
|
50
26
|
Legion::Logging.error e.backtrace
|
|
51
27
|
end
|
|
52
28
|
|
|
29
|
+
def poll_cycle
|
|
30
|
+
t1 = Time.now
|
|
31
|
+
log.debug "Running #{self.class}"
|
|
32
|
+
old_result = Legion::Cache.get(cache_name)
|
|
33
|
+
log.debug "Cached value for #{self.class}: #{old_result}"
|
|
34
|
+
results = Legion::JSON.load(Legion::JSON.dump(manual))
|
|
35
|
+
Legion::Cache.set(cache_name, results, time * 2)
|
|
36
|
+
|
|
37
|
+
unless old_result.nil?
|
|
38
|
+
results[:diff] = Hashdiff.diff(results, old_result, numeric_tolerance: 0.0, array_path: false) do |_path, obj1, obj2|
|
|
39
|
+
if int_percentage_normalize.positive? && obj1.is_a?(Integer) && obj2.is_a?(Integer)
|
|
40
|
+
obj1.between?(obj2 * (1 - int_percentage_normalize), obj2 * (1 + int_percentage_normalize))
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
results[:changed] = results[:diff].any?
|
|
44
|
+
|
|
45
|
+
Legion::Logging.info results[:diff] if results[:changed]
|
|
46
|
+
Legion::Transport::Messages::CheckSubtask.new(runner_class: runner_class.to_s,
|
|
47
|
+
function: runner_function,
|
|
48
|
+
result: results,
|
|
49
|
+
type: 'poll_result',
|
|
50
|
+
polling: true).publish
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
sleep_time = 1 - (Time.now - t1)
|
|
54
|
+
sleep(sleep_time) if sleep_time.positive?
|
|
55
|
+
log.debug("#{self.class} result: #{results}")
|
|
56
|
+
results
|
|
57
|
+
rescue StandardError => e
|
|
58
|
+
Legion::Logging.fatal e.message
|
|
59
|
+
Legion::Logging.fatal e.backtrace
|
|
60
|
+
end
|
|
61
|
+
|
|
53
62
|
def cache_name
|
|
54
63
|
"#{lex_name}_#{runner_name}"
|
|
55
64
|
end
|
|
@@ -111,14 +111,11 @@ module Legion
|
|
|
111
111
|
delivery_info = rmq_message.first
|
|
112
112
|
|
|
113
113
|
message = process_message(payload, metadata, delivery_info)
|
|
114
|
+
fn = find_function(message)
|
|
114
115
|
if use_runner?
|
|
115
|
-
|
|
116
|
-
runner_class: runner_class,
|
|
117
|
-
function: find_function(message),
|
|
118
|
-
check_subtask: check_subtask?,
|
|
119
|
-
generate_task: generate_task?)
|
|
116
|
+
dispatch_runner(message, runner_class, fn, check_subtask?, generate_task?)
|
|
120
117
|
else
|
|
121
|
-
runner_class.send(
|
|
118
|
+
runner_class.send(fn, **message)
|
|
122
119
|
end
|
|
123
120
|
@queue.acknowledge(delivery_info.delivery_tag) if manual_ack
|
|
124
121
|
|
|
@@ -131,6 +128,24 @@ module Legion
|
|
|
131
128
|
@queue.reject(delivery_info.delivery_tag) if manual_ack
|
|
132
129
|
end
|
|
133
130
|
end
|
|
131
|
+
|
|
132
|
+
private
|
|
133
|
+
|
|
134
|
+
def dispatch_runner(message, runner_cls, function, check_subtask, generate_task)
|
|
135
|
+
run_block = lambda {
|
|
136
|
+
Legion::Runner.run(**message,
|
|
137
|
+
runner_class: runner_cls,
|
|
138
|
+
function: function,
|
|
139
|
+
check_subtask: check_subtask,
|
|
140
|
+
generate_task: generate_task)
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
if defined?(Legion::Telemetry::OpenInference)
|
|
144
|
+
Legion::Telemetry::OpenInference.chain_span(type: 'task_chain') { |_span| run_block.call }
|
|
145
|
+
else
|
|
146
|
+
run_block.call
|
|
147
|
+
end
|
|
148
|
+
end
|
|
134
149
|
end
|
|
135
150
|
end
|
|
136
151
|
end
|
data/lib/legion/ingress.rb
CHANGED
|
@@ -73,13 +73,21 @@ module Legion
|
|
|
73
73
|
|
|
74
74
|
Legion::Events.emit('ingress.received', runner_class: rc.to_s, function: fn, source: source)
|
|
75
75
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
76
|
+
runner_block = lambda {
|
|
77
|
+
Legion::Runner.run(
|
|
78
|
+
runner_class: rc,
|
|
79
|
+
function: fn,
|
|
80
|
+
check_subtask: check_subtask,
|
|
81
|
+
generate_task: generate_task,
|
|
82
|
+
**message
|
|
83
|
+
)
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
if defined?(Legion::Telemetry::OpenInference)
|
|
87
|
+
Legion::Telemetry::OpenInference.tool_span(name: "#{rc}.#{fn}", parameters: message) { |_span| runner_block.call }
|
|
88
|
+
else
|
|
89
|
+
runner_block.call
|
|
90
|
+
end
|
|
83
91
|
rescue PayloadTooLarge => e
|
|
84
92
|
{ success: false, status: 'task.blocked', error: { code: 'payload_too_large', message: e.message } }
|
|
85
93
|
rescue InvalidRunnerClass => e
|
data/lib/legion/service.rb
CHANGED
|
@@ -11,7 +11,7 @@ module Legion
|
|
|
11
11
|
base.freeze
|
|
12
12
|
end
|
|
13
13
|
|
|
14
|
-
def initialize(transport: true, cache: true, data: true, supervision: true, extensions: true, # rubocop:disable Metrics/ParameterLists
|
|
14
|
+
def initialize(transport: true, cache: true, data: true, supervision: true, extensions: true, # rubocop:disable Metrics/ParameterLists,Metrics/MethodLength
|
|
15
15
|
crypt: true, api: true, llm: true, gaia: true, log_level: 'info', http_port: nil)
|
|
16
16
|
setup_logging(log_level: log_level)
|
|
17
17
|
Legion::Logging.debug('Starting Legion::Service')
|
|
@@ -58,6 +58,7 @@ module Legion
|
|
|
58
58
|
end
|
|
59
59
|
|
|
60
60
|
setup_telemetry
|
|
61
|
+
setup_safety_metrics
|
|
61
62
|
setup_supervision if supervision
|
|
62
63
|
|
|
63
64
|
if extensions
|
|
@@ -286,6 +287,15 @@ module Legion
|
|
|
286
287
|
Legion::Logging.warn "OpenTelemetry setup failed: #{e.message}"
|
|
287
288
|
end
|
|
288
289
|
|
|
290
|
+
def setup_safety_metrics
|
|
291
|
+
require_relative 'telemetry/safety_metrics'
|
|
292
|
+
Legion::Telemetry::SafetyMetrics.start
|
|
293
|
+
rescue LoadError
|
|
294
|
+
nil
|
|
295
|
+
rescue StandardError => e
|
|
296
|
+
Legion::Logging.debug "[safety_metrics] setup skipped: #{e.message}" if defined?(Legion::Logging)
|
|
297
|
+
end
|
|
298
|
+
|
|
289
299
|
def setup_supervision
|
|
290
300
|
require 'legion/supervision'
|
|
291
301
|
@supervision = Legion::Supervision.setup
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Legion
|
|
4
|
+
module Telemetry
|
|
5
|
+
module OpenInference
|
|
6
|
+
DEFAULT_TRUNCATE = 4096
|
|
7
|
+
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
def open_inference_enabled?
|
|
11
|
+
return false unless Legion::Telemetry.enabled?
|
|
12
|
+
|
|
13
|
+
settings = begin
|
|
14
|
+
Legion::Settings.dig(:telemetry, :open_inference)
|
|
15
|
+
rescue StandardError
|
|
16
|
+
{}
|
|
17
|
+
end
|
|
18
|
+
settings.is_a?(Hash) ? settings.fetch(:enabled, true) : true
|
|
19
|
+
rescue StandardError
|
|
20
|
+
false
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def include_io?
|
|
24
|
+
settings = begin
|
|
25
|
+
Legion::Settings.dig(:telemetry, :open_inference)
|
|
26
|
+
rescue StandardError
|
|
27
|
+
{}
|
|
28
|
+
end
|
|
29
|
+
settings.is_a?(Hash) ? settings.fetch(:include_input_output, true) : true
|
|
30
|
+
rescue StandardError
|
|
31
|
+
true
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def truncate_limit
|
|
35
|
+
settings = begin
|
|
36
|
+
Legion::Settings.dig(:telemetry, :open_inference)
|
|
37
|
+
rescue StandardError
|
|
38
|
+
{}
|
|
39
|
+
end
|
|
40
|
+
settings.is_a?(Hash) ? settings.fetch(:truncate_values_at, DEFAULT_TRUNCATE) : DEFAULT_TRUNCATE
|
|
41
|
+
rescue StandardError
|
|
42
|
+
DEFAULT_TRUNCATE
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def llm_span(model:, provider: nil, invocation_params: {}, input: nil)
|
|
46
|
+
unless open_inference_enabled?
|
|
47
|
+
return yield(nil) if block_given?
|
|
48
|
+
|
|
49
|
+
return
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
attrs = base_attrs('LLM').merge('llm.model_name' => model)
|
|
53
|
+
attrs['llm.provider'] = provider if provider
|
|
54
|
+
attrs['llm.invocation_parameters'] = invocation_params.to_json unless invocation_params.empty?
|
|
55
|
+
attrs['input.value'] = truncate_value(input.to_s) if input && include_io?
|
|
56
|
+
|
|
57
|
+
Legion::Telemetry.with_span("llm.#{model}", kind: :client, attributes: attrs) do |span|
|
|
58
|
+
result = yield(span)
|
|
59
|
+
annotate_llm_result(span, result) if span
|
|
60
|
+
result
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def embedding_span(model:, dimensions: nil, &)
|
|
65
|
+
unless open_inference_enabled?
|
|
66
|
+
return yield(nil) if block_given?
|
|
67
|
+
|
|
68
|
+
return
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
attrs = base_attrs('EMBEDDING').merge('embedding.model_name' => model)
|
|
72
|
+
attrs['embedding.dimensions'] = dimensions if dimensions
|
|
73
|
+
|
|
74
|
+
Legion::Telemetry.with_span("embedding.#{model}", kind: :client, attributes: attrs, &)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def tool_span(name:, parameters: {})
|
|
78
|
+
unless open_inference_enabled?
|
|
79
|
+
return yield(nil) if block_given?
|
|
80
|
+
|
|
81
|
+
return
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
attrs = base_attrs('TOOL').merge('tool.name' => name)
|
|
85
|
+
attrs['tool.parameters'] = parameters.to_json unless parameters.empty?
|
|
86
|
+
|
|
87
|
+
Legion::Telemetry.with_span("tool.#{name}", kind: :internal, attributes: attrs) do |span|
|
|
88
|
+
result = yield(span)
|
|
89
|
+
annotate_output(span, result) if span && include_io?
|
|
90
|
+
result
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def chain_span(type: 'task_chain', relationship_id: nil, &)
|
|
95
|
+
unless open_inference_enabled?
|
|
96
|
+
return yield(nil) if block_given?
|
|
97
|
+
|
|
98
|
+
return
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
attrs = base_attrs('CHAIN').merge('chain.type' => type)
|
|
102
|
+
attrs['chain.relationship_id'] = relationship_id if relationship_id
|
|
103
|
+
|
|
104
|
+
Legion::Telemetry.with_span("chain.#{type}", kind: :internal, attributes: attrs, &)
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def evaluator_span(template:)
|
|
108
|
+
unless open_inference_enabled?
|
|
109
|
+
return yield(nil) if block_given?
|
|
110
|
+
|
|
111
|
+
return
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
attrs = base_attrs('EVALUATOR').merge('eval.template' => template)
|
|
115
|
+
|
|
116
|
+
Legion::Telemetry.with_span("eval.#{template}", kind: :internal, attributes: attrs) do |span|
|
|
117
|
+
result = yield(span)
|
|
118
|
+
annotate_eval_result(span, result) if span && result.is_a?(Hash)
|
|
119
|
+
result
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def agent_span(name:, mode: nil, phase_count: nil, budget_ms: nil, &)
|
|
124
|
+
unless open_inference_enabled?
|
|
125
|
+
return yield(nil) if block_given?
|
|
126
|
+
|
|
127
|
+
return
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
attrs = base_attrs('AGENT').merge('agent.name' => name)
|
|
131
|
+
attrs['agent.mode'] = mode.to_s if mode
|
|
132
|
+
attrs['agent.phase_count'] = phase_count if phase_count
|
|
133
|
+
attrs['agent.budget_ms'] = budget_ms if budget_ms
|
|
134
|
+
|
|
135
|
+
Legion::Telemetry.with_span("agent.#{name}", kind: :internal, attributes: attrs, &)
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def truncate_value(str, max: nil)
|
|
139
|
+
limit = max || truncate_limit
|
|
140
|
+
str.length > limit ? str[0...limit] : str
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def base_attrs(kind)
|
|
144
|
+
{ 'openinference.span.kind' => kind }
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def annotate_llm_result(span, result)
|
|
148
|
+
return unless span.respond_to?(:set_attribute) && result.is_a?(Hash)
|
|
149
|
+
|
|
150
|
+
span.set_attribute('llm.token_count.prompt', result[:input_tokens]) if result[:input_tokens]
|
|
151
|
+
span.set_attribute('llm.token_count.completion', result[:output_tokens]) if result[:output_tokens]
|
|
152
|
+
span.set_attribute('output.value', truncate_value(result[:content].to_s)) if include_io? && result[:content]
|
|
153
|
+
rescue StandardError
|
|
154
|
+
nil
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def annotate_output(span, result)
|
|
158
|
+
return unless span.respond_to?(:set_attribute)
|
|
159
|
+
|
|
160
|
+
val = result.is_a?(Hash) ? result.to_json : result.to_s
|
|
161
|
+
span.set_attribute('output.value', truncate_value(val))
|
|
162
|
+
rescue StandardError
|
|
163
|
+
nil
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def annotate_eval_result(span, result)
|
|
167
|
+
return unless span.respond_to?(:set_attribute)
|
|
168
|
+
|
|
169
|
+
span.set_attribute('eval.score', result[:score]) if result[:score]
|
|
170
|
+
span.set_attribute('eval.passed', result[:passed]) unless result[:passed].nil?
|
|
171
|
+
span.set_attribute('eval.explanation', result[:explanation]) if result[:explanation]
|
|
172
|
+
rescue StandardError
|
|
173
|
+
nil
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
end
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Legion
|
|
4
|
+
module Telemetry
|
|
5
|
+
class SlidingWindow
|
|
6
|
+
def initialize(window_seconds)
|
|
7
|
+
@window = window_seconds
|
|
8
|
+
@entries = []
|
|
9
|
+
@mutex = Mutex.new
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def push(**entry)
|
|
13
|
+
@mutex.synchronize do
|
|
14
|
+
@entries << entry.merge(at: Time.now)
|
|
15
|
+
prune!
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def count
|
|
20
|
+
@mutex.synchronize do
|
|
21
|
+
prune!
|
|
22
|
+
@entries.size
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def count_for(**filters)
|
|
27
|
+
@mutex.synchronize do
|
|
28
|
+
prune!
|
|
29
|
+
@entries.count { |e| filters.all? { |k, v| e[k] == v } }
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def entries_matching(**filters)
|
|
34
|
+
@mutex.synchronize do
|
|
35
|
+
prune!
|
|
36
|
+
@entries.select { |e| filters.all? { |k, v| e[k] == v } }
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def prune!
|
|
43
|
+
cutoff = Time.now - @window
|
|
44
|
+
@entries.reject! { |e| e[:at] < cutoff }
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
module SafetyMetrics
|
|
49
|
+
WINDOWS = {
|
|
50
|
+
actions: 60,
|
|
51
|
+
failures: 300,
|
|
52
|
+
successes: 300,
|
|
53
|
+
confidence: 300
|
|
54
|
+
}.freeze
|
|
55
|
+
|
|
56
|
+
module_function
|
|
57
|
+
|
|
58
|
+
def start
|
|
59
|
+
return unless safety_enabled?
|
|
60
|
+
|
|
61
|
+
init_windows
|
|
62
|
+
register_prometheus_metrics
|
|
63
|
+
subscribe_events
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def init_windows
|
|
67
|
+
@windows = WINDOWS.transform_values { |secs| SlidingWindow.new(secs) }
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def subscribe_events
|
|
71
|
+
return unless defined?(Legion::Events)
|
|
72
|
+
|
|
73
|
+
Legion::Events.on('ingress.received') { |e| record_action(**e) }
|
|
74
|
+
Legion::Events.on('runner.failure') { |e| record_failure(**e) }
|
|
75
|
+
Legion::Events.on('runner.success') { |e| record_success(**e) }
|
|
76
|
+
Legion::Events.on('rbac.deny') { |e| record_escalation(**e) }
|
|
77
|
+
Legion::Events.on('governance.consent_violation') { |e| record_governance(**e) }
|
|
78
|
+
Legion::Events.on('privatecore.probe_detected') { |e| record_probe(**e) }
|
|
79
|
+
Legion::Events.on('synapse.confidence_update') { |e| record_confidence(**e) }
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def record_action(agent_id: 'unknown', **)
|
|
83
|
+
@windows[:actions]&.push(agent: agent_id)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def record_failure(agent_id: 'unknown', **)
|
|
87
|
+
@windows[:failures]&.push(agent: agent_id, type: :failure)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def record_success(agent_id: 'unknown', **)
|
|
91
|
+
@windows[:successes]&.push(agent: agent_id, type: :success)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def record_escalation(agent_id: 'unknown', **) # rubocop:disable Lint/UnusedMethodArgument
|
|
95
|
+
@escalation_count = (@escalation_count || 0) + 1
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def record_governance(**)
|
|
99
|
+
@governance_count = (@governance_count || 0) + 1
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def record_probe(**)
|
|
103
|
+
@probe_count = (@probe_count || 0) + 1
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def record_confidence(agent_id: 'unknown', delta: 0.0, **)
|
|
107
|
+
@windows[:confidence]&.push(agent: agent_id, delta: delta)
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def actions_per_minute(agent_id)
|
|
111
|
+
@windows[:actions]&.count_for(agent: agent_id) || 0
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def tool_failure_ratio(agent_id)
|
|
115
|
+
fails = @windows[:failures]&.count_for(agent: agent_id) || 0
|
|
116
|
+
successes = @windows[:successes]&.count_for(agent: agent_id) || 0
|
|
117
|
+
total = fails + successes
|
|
118
|
+
total.zero? ? 0.0 : fails.to_f / total
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def confidence_drift(agent_id)
|
|
122
|
+
entries = @windows[:confidence]&.entries_matching(agent: agent_id) || []
|
|
123
|
+
return 0.0 if entries.empty?
|
|
124
|
+
|
|
125
|
+
entries.sum { |e| e[:delta] || 0.0 } / entries.size
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def scope_escalation_total
|
|
129
|
+
@escalation_count || 0
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def governance_override_total
|
|
133
|
+
@governance_count || 0
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def probe_detection_total
|
|
137
|
+
@probe_count || 0
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def safety_enabled?
|
|
141
|
+
Legion::Settings.dig(:telemetry, :safety, :enabled)
|
|
142
|
+
rescue StandardError
|
|
143
|
+
false
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def register_prometheus_metrics
|
|
147
|
+
return unless defined?(Legion::Metrics) && Legion::Metrics.respond_to?(:register_gauge)
|
|
148
|
+
|
|
149
|
+
Legion::Metrics.register_gauge(:legion_safety_actions_per_minute,
|
|
150
|
+
'Runner invocations per agent per minute')
|
|
151
|
+
Legion::Metrics.register_gauge(:legion_safety_tool_failure_ratio,
|
|
152
|
+
'Tool failure percentage over 5m window')
|
|
153
|
+
Legion::Metrics.register_gauge(:legion_safety_confidence_drift,
|
|
154
|
+
'Rate of confidence decrease across synapses')
|
|
155
|
+
Legion::Metrics.register_counter(:legion_safety_scope_escalation_total,
|
|
156
|
+
'Denied access attempts')
|
|
157
|
+
Legion::Metrics.register_counter(:legion_safety_governance_override_total,
|
|
158
|
+
'Governance constraint violations')
|
|
159
|
+
Legion::Metrics.register_counter(:legion_safety_probe_detection_total,
|
|
160
|
+
'Detected prompt injection probes')
|
|
161
|
+
rescue StandardError
|
|
162
|
+
nil
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
end
|
data/lib/legion/telemetry.rb
CHANGED
data/lib/legion/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: legionio
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.4.
|
|
4
|
+
version: 1.4.87
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Esity
|
|
@@ -353,6 +353,7 @@ files:
|
|
|
353
353
|
- lib/legion.rb
|
|
354
354
|
- lib/legion/alerts.rb
|
|
355
355
|
- lib/legion/api.rb
|
|
356
|
+
- lib/legion/api/acp.rb
|
|
356
357
|
- lib/legion/api/audit.rb
|
|
357
358
|
- lib/legion/api/auth.rb
|
|
358
359
|
- lib/legion/api/auth_human.rb
|
|
@@ -549,6 +550,7 @@ files:
|
|
|
549
550
|
- lib/legion/extensions/actors/base.rb
|
|
550
551
|
- lib/legion/extensions/actors/defaults.rb
|
|
551
552
|
- lib/legion/extensions/actors/every.rb
|
|
553
|
+
- lib/legion/extensions/actors/fingerprint.rb
|
|
552
554
|
- lib/legion/extensions/actors/loop.rb
|
|
553
555
|
- lib/legion/extensions/actors/nothing.rb
|
|
554
556
|
- lib/legion/extensions/actors/once.rb
|
|
@@ -596,6 +598,8 @@ files:
|
|
|
596
598
|
- lib/legion/service.rb
|
|
597
599
|
- lib/legion/supervision.rb
|
|
598
600
|
- lib/legion/telemetry.rb
|
|
601
|
+
- lib/legion/telemetry/open_inference.rb
|
|
602
|
+
- lib/legion/telemetry/safety_metrics.rb
|
|
599
603
|
- lib/legion/tenant_context.rb
|
|
600
604
|
- lib/legion/tenants.rb
|
|
601
605
|
- lib/legion/trace_search.rb
|