lex-agentic-social 0.1.8 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/lib/legion/extensions/agentic/social/calibration/client.rb +23 -0
- data/lib/legion/extensions/agentic/social/calibration/helpers/calibration_store.rb +192 -0
- data/lib/legion/extensions/agentic/social/calibration/helpers/constants.rb +64 -0
- data/lib/legion/extensions/agentic/social/calibration/runners/calibration.rb +182 -0
- data/lib/legion/extensions/agentic/social/calibration.rb +17 -0
- data/lib/legion/extensions/agentic/social/version.rb +1 -1
- data/lib/legion/extensions/agentic/social.rb +1 -0
- data/spec/legion/extensions/agentic/social/calibration/helpers/calibration_store_spec.rb +177 -0
- data/spec/legion/extensions/agentic/social/calibration/helpers/constants_spec.rb +56 -0
- data/spec/legion/extensions/agentic/social/calibration/runners/calibration_spec.rb +90 -0
- metadata +9 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 56e3b653aee6a0da01599ef4ed7dfcb409c637f3b730ce7231b532db528e2e8b
|
|
4
|
+
data.tar.gz: 7669f59e7277f6418734d2f9ab55057151876896894f0aacc56a93fa923ffa3e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5832f51b26be00617b30a934b8e62759b8ee0f3837afdb6a42af04bdd505d65280e8ca2877513c0f5ebea9f39f14d3cc0e7d748a772c0bc21a166e88c62a0065
|
|
7
|
+
data.tar.gz: e117c35dda203a26a34aa3a879878c42e32c9cd4b9d53e64227a8bc8d02ca884696eba6262e7fab4ead62a066db752147ece0ca8f3423ca8a7bef01ad172be9c
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.1.9] - 2026-03-31
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
- Calibration sub-module: CalibrationStore with EMA tracking, explicit feedback detection, partner baseline tracking
|
|
7
|
+
- CalibrationRunner: update_calibration, record_advisory_meta, detect_explicit_feedback, calibration_weights, calibration_stats
|
|
8
|
+
- sync_partner_knowledge: LLM preference extraction (weekly) + partner knowledge promotion to Apollo Global
|
|
9
|
+
- Apollo Local persistence via dirty?/to_apollo_entries/from_apollo pattern
|
|
10
|
+
|
|
3
11
|
## [0.1.8] - 2026-03-31
|
|
4
12
|
|
|
5
13
|
### Added
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Legion
|
|
4
|
+
module Extensions
|
|
5
|
+
module Agentic
|
|
6
|
+
module Social
|
|
7
|
+
module Calibration
|
|
8
|
+
class Client
|
|
9
|
+
include Runners::Calibration
|
|
10
|
+
|
|
11
|
+
def initialize(**)
|
|
12
|
+
@calibration_store = Helpers::CalibrationStore.new
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
private
|
|
16
|
+
|
|
17
|
+
attr_reader :calibration_store
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Legion
|
|
4
|
+
module Extensions
|
|
5
|
+
module Agentic
|
|
6
|
+
module Social
|
|
7
|
+
module Calibration
|
|
8
|
+
module Helpers
|
|
9
|
+
class CalibrationStore
|
|
10
|
+
attr_reader :weights, :history
|
|
11
|
+
|
|
12
|
+
def initialize
|
|
13
|
+
@weights = Constants::ADVISORY_TYPES.to_h do |type|
|
|
14
|
+
[type.to_s, Constants::NEUTRAL_SCORE]
|
|
15
|
+
end
|
|
16
|
+
@history = {}
|
|
17
|
+
@last_advisory_meta = nil
|
|
18
|
+
@partner_baselines = { 'avg_latency' => nil, 'avg_length' => nil }
|
|
19
|
+
@dirty = false
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def dirty?
|
|
23
|
+
@dirty
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def mark_clean!
|
|
27
|
+
@dirty = false
|
|
28
|
+
self
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def record_advisory(advisory_id:, advisory_types:)
|
|
32
|
+
@last_advisory_meta = {
|
|
33
|
+
advisory_id: advisory_id,
|
|
34
|
+
advisory_types: Array(advisory_types).map(&:to_s),
|
|
35
|
+
timestamp: Time.now.utc
|
|
36
|
+
}
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def evaluate_reaction(observation:)
|
|
40
|
+
return nil unless @last_advisory_meta
|
|
41
|
+
return nil if @last_advisory_meta[:advisory_types].empty?
|
|
42
|
+
|
|
43
|
+
elapsed = Time.now.utc - @last_advisory_meta[:timestamp]
|
|
44
|
+
return nil if elapsed > 3600
|
|
45
|
+
|
|
46
|
+
reaction_score = compute_reaction_score(observation)
|
|
47
|
+
confidence = compute_confidence(observation)
|
|
48
|
+
|
|
49
|
+
deltas = @last_advisory_meta[:advisory_types].map do |type|
|
|
50
|
+
apply_delta(type: type, reaction_score: reaction_score, confidence: confidence)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
@last_advisory_meta = nil
|
|
54
|
+
@dirty = true
|
|
55
|
+
{ success: true, deltas: deltas }
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def calibration_weights
|
|
59
|
+
@weights.dup
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def detect_explicit_feedback(content)
|
|
63
|
+
text = content.to_s
|
|
64
|
+
return :positive if text.match?(Constants::POSITIVE_PATTERNS)
|
|
65
|
+
return :negative if text.match?(Constants::NEGATIVE_PATTERNS)
|
|
66
|
+
|
|
67
|
+
:neutral
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def update_baseline(latency:, length:)
|
|
71
|
+
@partner_baselines['avg_latency'] = if @partner_baselines['avg_latency']
|
|
72
|
+
(@partner_baselines['avg_latency'] * 0.9) + (latency.to_f * 0.1)
|
|
73
|
+
else
|
|
74
|
+
latency.to_f
|
|
75
|
+
end
|
|
76
|
+
@partner_baselines['avg_length'] = if @partner_baselines['avg_length']
|
|
77
|
+
(@partner_baselines['avg_length'] * 0.9) + (length.to_f * 0.1)
|
|
78
|
+
else
|
|
79
|
+
length.to_f
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def to_apollo_entries
|
|
84
|
+
entries = []
|
|
85
|
+
|
|
86
|
+
content = Legion::JSON.dump({
|
|
87
|
+
'weights' => @weights,
|
|
88
|
+
'baselines' => @partner_baselines,
|
|
89
|
+
'updated_at' => Time.now.utc.iso8601
|
|
90
|
+
})
|
|
91
|
+
entries << { content: content, tags: Constants::WEIGHTS_TAGS.dup + ['partner'] }
|
|
92
|
+
|
|
93
|
+
@history.each do |type, events|
|
|
94
|
+
hist_content = Legion::JSON.dump({
|
|
95
|
+
'advisory_type' => type,
|
|
96
|
+
'events' => events.last(Constants::MAX_HISTORY),
|
|
97
|
+
'current_weight' => @weights[type]
|
|
98
|
+
})
|
|
99
|
+
entries << { content: hist_content, tags: Constants::HISTORY_TAG_PREFIX + [type] }
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
entries
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def from_apollo(store:)
|
|
106
|
+
result = store.query(text: 'bond calibration weights', tags: Constants::WEIGHTS_TAGS)
|
|
107
|
+
return false unless result[:success] && result[:results]&.any?
|
|
108
|
+
|
|
109
|
+
entry = result[:results].first
|
|
110
|
+
parsed = ::JSON.parse(entry[:content])
|
|
111
|
+
@weights.merge!(parsed['weights']) if parsed['weights']
|
|
112
|
+
@partner_baselines = parsed['baselines'] if parsed['baselines']
|
|
113
|
+
true
|
|
114
|
+
rescue StandardError => e
|
|
115
|
+
Legion::Logging.warn("[calibration_store] from_apollo error: #{e.message}")
|
|
116
|
+
false
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
private
|
|
120
|
+
|
|
121
|
+
def compute_reaction_score(observation)
|
|
122
|
+
score = 0.0
|
|
123
|
+
|
|
124
|
+
feedback = detect_explicit_feedback(observation[:content] || '')
|
|
125
|
+
case feedback
|
|
126
|
+
when :positive then score += Constants::SIGNAL_WEIGHTS[:explicit_feedback]
|
|
127
|
+
when :negative then score -= Constants::SIGNAL_WEIGHTS[:explicit_feedback]
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
if @partner_baselines['avg_latency'] && observation[:latency]
|
|
131
|
+
ratio = observation[:latency].to_f / @partner_baselines['avg_latency']
|
|
132
|
+
latency_signal = if ratio < 0.8
|
|
133
|
+
1.0
|
|
134
|
+
else
|
|
135
|
+
(ratio > 1.5 ? -1.0 : 0.0)
|
|
136
|
+
end
|
|
137
|
+
score += Constants::SIGNAL_WEIGHTS[:response_latency] * latency_signal
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
if @partner_baselines['avg_length'] && observation[:content_length]
|
|
141
|
+
length_signal = observation[:content_length].to_f / [@partner_baselines['avg_length'], 1].max > 0.5 ? 1.0 : -1.0
|
|
142
|
+
score += Constants::SIGNAL_WEIGHTS[:message_length] * length_signal
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
score += Constants::SIGNAL_WEIGHTS[:direct_address] if observation[:direct_address]
|
|
146
|
+
|
|
147
|
+
((score + 1.0) / 2.0).clamp(0.0, 1.0)
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def compute_confidence(observation)
|
|
151
|
+
signals = 0
|
|
152
|
+
signals += 1 if observation[:content].to_s.length.positive?
|
|
153
|
+
signals += 1 if observation[:latency]
|
|
154
|
+
signals += 1 if observation[:content_length]
|
|
155
|
+
signals += 1 if observation.key?(:direct_address)
|
|
156
|
+
(signals / 4.0).clamp(0.2, 1.0)
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def apply_delta(type:, reaction_score:, confidence:)
|
|
160
|
+
delta = (reaction_score - Constants::NEUTRAL_SCORE) * confidence * Constants::EMA_ALPHA
|
|
161
|
+
old_weight = @weights[type] || Constants::NEUTRAL_SCORE
|
|
162
|
+
new_weight = (old_weight + delta).clamp(0.0, 1.0)
|
|
163
|
+
@weights[type] = new_weight
|
|
164
|
+
|
|
165
|
+
event = {
|
|
166
|
+
'reaction_score' => reaction_score.round(3),
|
|
167
|
+
'confidence' => confidence.round(3),
|
|
168
|
+
'delta' => delta.round(4),
|
|
169
|
+
'old_weight' => old_weight.round(3),
|
|
170
|
+
'new_weight' => new_weight.round(3),
|
|
171
|
+
'timestamp' => Time.now.utc.iso8601
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
@history[type] ||= []
|
|
175
|
+
@history[type] << event
|
|
176
|
+
@history[type] = @history[type].last(Constants::MAX_HISTORY)
|
|
177
|
+
|
|
178
|
+
{
|
|
179
|
+
advisory_type: type,
|
|
180
|
+
reaction_score: reaction_score.round(3),
|
|
181
|
+
confidence: confidence.round(3),
|
|
182
|
+
delta: delta.round(4),
|
|
183
|
+
new_weight: new_weight.round(3)
|
|
184
|
+
}
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
end
|
|
192
|
+
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Legion
|
|
4
|
+
module Extensions
|
|
5
|
+
module Agentic
|
|
6
|
+
module Social
|
|
7
|
+
module Calibration
|
|
8
|
+
module Helpers
|
|
9
|
+
module Constants
|
|
10
|
+
ADVISORY_TYPES = %i[
|
|
11
|
+
tone_adjustment
|
|
12
|
+
verbosity_adjustment
|
|
13
|
+
format_adjustment
|
|
14
|
+
context_injection
|
|
15
|
+
partner_hint
|
|
16
|
+
].freeze
|
|
17
|
+
|
|
18
|
+
EMA_ALPHA = 0.1
|
|
19
|
+
NEUTRAL_SCORE = 0.5
|
|
20
|
+
SUPPRESSION_THRESHOLD = 0.4
|
|
21
|
+
HIGH_CALIBRATION = 0.7
|
|
22
|
+
MAX_HISTORY = 50
|
|
23
|
+
|
|
24
|
+
TAG_PREFIX = %w[bond calibration].freeze
|
|
25
|
+
WEIGHTS_TAGS = (TAG_PREFIX + ['weights']).freeze
|
|
26
|
+
HISTORY_TAG_PREFIX = (TAG_PREFIX + ['history']).freeze
|
|
27
|
+
|
|
28
|
+
POSITIVE_PATTERNS = /\b(thanks|perfect|exactly|great|good|helpful|nice|yes)\b/i
|
|
29
|
+
NEGATIVE_PATTERNS = /\b(no|wrong|not what|stop|don't|didn't ask|incorrect)\b/i
|
|
30
|
+
|
|
31
|
+
SIGNAL_WEIGHTS = {
|
|
32
|
+
explicit_feedback: 0.35,
|
|
33
|
+
response_latency: 0.20,
|
|
34
|
+
message_length: 0.15,
|
|
35
|
+
direct_address: 0.15,
|
|
36
|
+
continuation: 0.15
|
|
37
|
+
}.freeze
|
|
38
|
+
|
|
39
|
+
PROMOTABLE_TAGS = [
|
|
40
|
+
%w[bond attachment],
|
|
41
|
+
%w[bond communication_pattern],
|
|
42
|
+
%w[bond calibration weights],
|
|
43
|
+
%w[partner preference]
|
|
44
|
+
].freeze
|
|
45
|
+
|
|
46
|
+
PROMOTION_MIN_CONFIDENCE = 0.6
|
|
47
|
+
PREFERENCE_EXTRACTION_INTERVAL = 604_800 # 7 days
|
|
48
|
+
|
|
49
|
+
module_function
|
|
50
|
+
|
|
51
|
+
def advisory_type?(type)
|
|
52
|
+
ADVISORY_TYPES.include?(type.to_sym)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def suppressed?(score)
|
|
56
|
+
score < SUPPRESSION_THRESHOLD
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Legion
|
|
4
|
+
module Extensions
|
|
5
|
+
module Agentic
|
|
6
|
+
module Social
|
|
7
|
+
module Calibration
|
|
8
|
+
module Runners
|
|
9
|
+
module Calibration
|
|
10
|
+
include Legion::Extensions::Helpers::Lex if defined?(Legion::Extensions::Helpers) &&
|
|
11
|
+
Legion::Extensions::Helpers.const_defined?(:Lex, false)
|
|
12
|
+
|
|
13
|
+
def update_calibration(observation: nil, **)
|
|
14
|
+
return { success: true, skipped: :no_observation } unless observation
|
|
15
|
+
|
|
16
|
+
calibration_store.update_baseline(
|
|
17
|
+
latency: observation[:latency] || 0,
|
|
18
|
+
length: observation[:content_length] || 0
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
result = calibration_store.evaluate_reaction(observation: observation)
|
|
22
|
+
return { success: true, skipped: :no_advisory } unless result
|
|
23
|
+
|
|
24
|
+
{ success: true, deltas: result[:deltas], weights: calibration_store.calibration_weights }
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def record_advisory_meta(advisory_id:, advisory_types:, **)
|
|
28
|
+
calibration_store.record_advisory(advisory_id: advisory_id, advisory_types: advisory_types)
|
|
29
|
+
{ success: true }
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def detect_explicit_feedback(content:, **)
|
|
33
|
+
feedback = calibration_store.detect_explicit_feedback(content)
|
|
34
|
+
{ success: true, feedback: feedback }
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def calibration_weights(**)
|
|
38
|
+
{ success: true, weights: calibration_store.calibration_weights }
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def calibration_stats(**)
|
|
42
|
+
{
|
|
43
|
+
success: true,
|
|
44
|
+
weights: calibration_store.calibration_weights,
|
|
45
|
+
history_counts: calibration_store.history.transform_values(&:size),
|
|
46
|
+
dirty: calibration_store.dirty?
|
|
47
|
+
}
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def sync_partner_knowledge(**)
|
|
51
|
+
results = {}
|
|
52
|
+
results[:preferences] = extract_preferences_via_llm
|
|
53
|
+
results[:promotion] = promote_partner_knowledge
|
|
54
|
+
{ success: true, results: results }
|
|
55
|
+
rescue StandardError => e
|
|
56
|
+
{ success: false, error: e.message }
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def extract_preferences_via_llm(**)
|
|
60
|
+
return { success: true, skipped: :too_soon } unless should_extract_preferences?
|
|
61
|
+
return { success: true, skipped: :llm_unavailable } unless llm_available?
|
|
62
|
+
|
|
63
|
+
traces = retrieve_interaction_traces
|
|
64
|
+
return { success: true, skipped: :insufficient_data } if traces.empty?
|
|
65
|
+
|
|
66
|
+
context = summarize_traces(traces)
|
|
67
|
+
prompt = build_preference_prompt(context)
|
|
68
|
+
result = Legion::LLM.ask(message: prompt)
|
|
69
|
+
return { success: false, error: :llm_failed } unless result&.content
|
|
70
|
+
|
|
71
|
+
parsed = parse_preference_response(result.content)
|
|
72
|
+
return { success: false, error: :parse_failed } unless parsed
|
|
73
|
+
|
|
74
|
+
store_llm_preferences(parsed)
|
|
75
|
+
@last_preference_extraction_at = Time.now.utc
|
|
76
|
+
{ success: true, preferences_extracted: parsed.size }
|
|
77
|
+
rescue StandardError => e
|
|
78
|
+
{ success: false, error: e.message }
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def promote_partner_knowledge(**)
|
|
82
|
+
return { success: true, skipped: :local_unavailable } unless apollo_local_available?
|
|
83
|
+
|
|
84
|
+
total = 0
|
|
85
|
+
Helpers::Constants::PROMOTABLE_TAGS.each do |tags|
|
|
86
|
+
result = Legion::Apollo::Local.promote_to_global(tags: tags, min_confidence: Helpers::Constants::PROMOTION_MIN_CONFIDENCE)
|
|
87
|
+
total += result[:promoted] if result[:success]
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
{ success: true, promoted: total }
|
|
91
|
+
rescue StandardError => e
|
|
92
|
+
{ success: false, error: e.message }
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
private
|
|
96
|
+
|
|
97
|
+
def calibration_store
|
|
98
|
+
@calibration_store ||= Helpers::CalibrationStore.new
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def should_extract_preferences?
|
|
102
|
+
return true if @last_preference_extraction_at.nil?
|
|
103
|
+
|
|
104
|
+
(Time.now.utc - @last_preference_extraction_at) > Helpers::Constants::PREFERENCE_EXTRACTION_INTERVAL
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def llm_available?
|
|
108
|
+
defined?(Legion::LLM) && Legion::LLM.started?
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def apollo_local_available?
|
|
112
|
+
defined?(Legion::Apollo::Local) && Legion::Apollo::Local.started?
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def retrieve_interaction_traces
|
|
116
|
+
return [] unless defined?(Legion::Extensions::Agentic::Memory::Trace::Runners::Traces)
|
|
117
|
+
|
|
118
|
+
runner = Object.new
|
|
119
|
+
runner.extend(Legion::Extensions::Agentic::Memory::Trace::Runners::Traces)
|
|
120
|
+
result = runner.retrieve_by_domain(domain_tag: 'partner_interaction', limit: 50)
|
|
121
|
+
return [] unless result[:success]
|
|
122
|
+
|
|
123
|
+
result[:traces] || []
|
|
124
|
+
rescue StandardError => e
|
|
125
|
+
Legion::Logging.warn("[calibration] retrieve_interaction_traces error: #{e.message}")
|
|
126
|
+
[]
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def summarize_traces(traces)
|
|
130
|
+
traces.first(50).map do |t|
|
|
131
|
+
payload = t[:content_payload] || t[:content] || ''
|
|
132
|
+
"[#{t[:recorded_at] || t[:created_at]}] #{payload}"
|
|
133
|
+
end.join("\n")
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def build_preference_prompt(context)
|
|
137
|
+
<<~PROMPT
|
|
138
|
+
Based on these interaction patterns with my partner, what communication preferences can I infer? Consider:
|
|
139
|
+
- Preferred verbosity (concise/normal/detailed)
|
|
140
|
+
- Preferred tone (casual/professional/technical)
|
|
141
|
+
- Preferred format (prose/structured/bullet_points)
|
|
142
|
+
- Technical depth (high_level/moderate/deep)
|
|
143
|
+
|
|
144
|
+
Interaction summary:
|
|
145
|
+
#{context}
|
|
146
|
+
|
|
147
|
+
Return ONLY a JSON array of objects, each with: domain (string), value (string), confidence (float 0-1).
|
|
148
|
+
PROMPT
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def parse_preference_response(content)
|
|
152
|
+
json_match = content.match(/\[.*\]/m)
|
|
153
|
+
return nil unless json_match
|
|
154
|
+
|
|
155
|
+
Legion::JSON.parse(json_match[0])
|
|
156
|
+
rescue StandardError => e
|
|
157
|
+
Legion::Logging.warn("[calibration] parse_preference_response error: #{e.message}")
|
|
158
|
+
nil
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def store_llm_preferences(preferences)
|
|
162
|
+
return unless apollo_local_available?
|
|
163
|
+
|
|
164
|
+
base_tags = %w[partner preference llm_inference]
|
|
165
|
+
preferences.each do |pref|
|
|
166
|
+
content = Legion::JSON.dump({
|
|
167
|
+
'domain' => pref['domain'],
|
|
168
|
+
'value' => pref['value'],
|
|
169
|
+
'source' => 'llm_inference',
|
|
170
|
+
'confidence' => pref['confidence'] || 0.65
|
|
171
|
+
})
|
|
172
|
+
tags = base_tags + ["preference:#{pref['domain']}"]
|
|
173
|
+
Legion::Apollo::Local.upsert(content: content, tags: tags, confidence: pref['confidence'] || 0.65)
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'calibration/helpers/constants'
|
|
4
|
+
require_relative 'calibration/helpers/calibration_store'
|
|
5
|
+
require_relative 'calibration/runners/calibration'
|
|
6
|
+
require_relative 'calibration/client'
|
|
7
|
+
|
|
8
|
+
module Legion
|
|
9
|
+
module Extensions
|
|
10
|
+
module Agentic
|
|
11
|
+
module Social
|
|
12
|
+
module Calibration
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'spec_helper'
|
|
4
|
+
|
|
5
|
+
RSpec.describe Legion::Extensions::Agentic::Social::Calibration::Helpers::CalibrationStore do
|
|
6
|
+
subject(:store) { described_class.new }
|
|
7
|
+
|
|
8
|
+
let(:constants) { Legion::Extensions::Agentic::Social::Calibration::Helpers::Constants }
|
|
9
|
+
|
|
10
|
+
describe '#initialize' do
|
|
11
|
+
it 'starts with neutral weights for all advisory types' do
|
|
12
|
+
expect(store.weights.size).to eq(5)
|
|
13
|
+
store.weights.each_value { |v| expect(v).to eq(0.5) }
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
it 'starts clean' do
|
|
17
|
+
expect(store).not_to be_dirty
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
it 'starts with empty history' do
|
|
21
|
+
expect(store.history).to be_empty
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
describe '#dirty? and #mark_clean!' do
|
|
26
|
+
it 'becomes dirty after evaluation' do
|
|
27
|
+
store.record_advisory(advisory_id: 'test', advisory_types: [:tone_adjustment])
|
|
28
|
+
store.evaluate_reaction(observation: { content: 'thanks', direct_address: true, content_length: 50 })
|
|
29
|
+
expect(store).to be_dirty
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
it 'returns self from mark_clean!' do
|
|
33
|
+
expect(store.mark_clean!).to eq(store)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
it 'is clean after mark_clean!' do
|
|
37
|
+
store.record_advisory(advisory_id: 'test', advisory_types: [:tone_adjustment])
|
|
38
|
+
store.evaluate_reaction(observation: { content: 'thanks' })
|
|
39
|
+
store.mark_clean!
|
|
40
|
+
expect(store).not_to be_dirty
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
describe '#record_advisory' do
|
|
45
|
+
it 'stores advisory metadata for next evaluation' do
|
|
46
|
+
store.record_advisory(advisory_id: 'abc', advisory_types: %i[tone_adjustment verbosity_adjustment])
|
|
47
|
+
result = store.evaluate_reaction(observation: { content: 'thanks' })
|
|
48
|
+
expect(result).not_to be_nil
|
|
49
|
+
expect(result[:deltas].size).to eq(2)
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
describe '#evaluate_reaction' do
|
|
54
|
+
it 'returns nil when no advisory recorded' do
|
|
55
|
+
expect(store.evaluate_reaction(observation: { content: 'hello' })).to be_nil
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
it 'returns nil when advisory types are empty' do
|
|
59
|
+
store.record_advisory(advisory_id: 'test', advisory_types: [])
|
|
60
|
+
expect(store.evaluate_reaction(observation: { content: 'hello' })).to be_nil
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
context 'with recorded advisory' do
|
|
64
|
+
before { store.record_advisory(advisory_id: 'test', advisory_types: [:tone_adjustment]) }
|
|
65
|
+
|
|
66
|
+
it 'produces deltas for positive feedback' do
|
|
67
|
+
result = store.evaluate_reaction(observation: { content: 'thanks, perfect!', direct_address: true, content_length: 20 })
|
|
68
|
+
expect(result[:success]).to be true
|
|
69
|
+
expect(result[:deltas].first[:advisory_type]).to eq('tone_adjustment')
|
|
70
|
+
expect(result[:deltas].first[:new_weight]).to be > 0.5
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
it 'produces deltas for negative feedback' do
|
|
74
|
+
result = store.evaluate_reaction(observation: { content: 'no, wrong', content_length: 5 })
|
|
75
|
+
expect(result[:success]).to be true
|
|
76
|
+
expect(result[:deltas].first[:new_weight]).to be < 0.5
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
it 'clears advisory meta after evaluation' do
|
|
80
|
+
store.evaluate_reaction(observation: { content: 'ok' })
|
|
81
|
+
expect(store.evaluate_reaction(observation: { content: 'hello' })).to be_nil
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
it 'marks store as dirty' do
|
|
85
|
+
store.evaluate_reaction(observation: { content: 'thanks' })
|
|
86
|
+
expect(store).to be_dirty
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
it 'records history entry' do
|
|
90
|
+
store.evaluate_reaction(observation: { content: 'thanks' })
|
|
91
|
+
expect(store.history['tone_adjustment']).not_to be_empty
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
describe '#detect_explicit_feedback' do
|
|
97
|
+
it 'detects positive' do
|
|
98
|
+
expect(store.detect_explicit_feedback('thanks for that')).to eq(:positive)
|
|
99
|
+
expect(store.detect_explicit_feedback('exactly what I needed')).to eq(:positive)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
it 'detects negative' do
|
|
103
|
+
expect(store.detect_explicit_feedback('no that is wrong')).to eq(:negative)
|
|
104
|
+
expect(store.detect_explicit_feedback("didn't ask for that")).to eq(:negative)
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
it 'returns neutral for ambiguous' do
|
|
108
|
+
expect(store.detect_explicit_feedback('I see')).to eq(:neutral)
|
|
109
|
+
expect(store.detect_explicit_feedback('interesting')).to eq(:neutral)
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
describe '#calibration_weights' do
|
|
114
|
+
it 'returns a copy' do
|
|
115
|
+
weights = store.calibration_weights
|
|
116
|
+
weights['tone_adjustment'] = 999
|
|
117
|
+
expect(store.calibration_weights['tone_adjustment']).to eq(0.5)
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
describe '#update_baseline' do
|
|
122
|
+
it 'initializes baselines on first call' do
|
|
123
|
+
store.update_baseline(latency: 2.0, length: 100)
|
|
124
|
+
entries = store.to_apollo_entries
|
|
125
|
+
weights_entry = entries.find { |e| e[:tags].include?('weights') }
|
|
126
|
+
parsed = Legion::JSON.parse(weights_entry[:content])
|
|
127
|
+
expect(parsed[:baselines][:avg_latency]).to eq(2.0)
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
it 'applies EMA on subsequent calls' do
|
|
131
|
+
store.update_baseline(latency: 2.0, length: 100)
|
|
132
|
+
store.update_baseline(latency: 4.0, length: 200)
|
|
133
|
+
entries = store.to_apollo_entries
|
|
134
|
+
weights_entry = entries.find { |e| e[:tags].include?('weights') }
|
|
135
|
+
parsed = Legion::JSON.parse(weights_entry[:content])
|
|
136
|
+
expect(parsed[:baselines][:avg_latency]).to be_between(2.0, 4.0)
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
describe '#to_apollo_entries' do
|
|
141
|
+
it 'produces a weights entry with partner tag' do
|
|
142
|
+
entries = store.to_apollo_entries
|
|
143
|
+
weights_entry = entries.find { |e| e[:tags].include?('weights') }
|
|
144
|
+
expect(weights_entry).not_to be_nil
|
|
145
|
+
expect(weights_entry[:tags]).to include('bond', 'calibration', 'weights', 'partner')
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
it 'produces history entries after evaluation' do
|
|
149
|
+
store.record_advisory(advisory_id: 'test', advisory_types: [:tone_adjustment])
|
|
150
|
+
store.evaluate_reaction(observation: { content: 'thanks' })
|
|
151
|
+
entries = store.to_apollo_entries
|
|
152
|
+
history_entries = entries.select { |e| e[:tags].include?('history') }
|
|
153
|
+
expect(history_entries).not_to be_empty
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
describe '#from_apollo' do
|
|
158
|
+
let(:mock_store) { double('store') }
|
|
159
|
+
|
|
160
|
+
it 'restores weights from Apollo Local' do
|
|
161
|
+
content = Legion::JSON.dump({ 'weights' => { 'tone_adjustment' => 0.8 }, 'baselines' => {}, 'updated_at' => Time.now.utc.iso8601 })
|
|
162
|
+
allow(mock_store).to receive(:query).and_return({ success: true, results: [{ content: content, confidence: 0.9 }] })
|
|
163
|
+
expect(store.from_apollo(store: mock_store)).to be true
|
|
164
|
+
expect(store.weights['tone_adjustment']).to eq(0.8)
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
it 'returns false when no data' do
|
|
168
|
+
allow(mock_store).to receive(:query).and_return({ success: true, results: [] })
|
|
169
|
+
expect(store.from_apollo(store: mock_store)).to be false
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
it 'returns false on error' do
|
|
173
|
+
allow(mock_store).to receive(:query).and_raise(StandardError, 'boom')
|
|
174
|
+
expect(store.from_apollo(store: mock_store)).to be false
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'spec_helper'
|
|
4
|
+
|
|
5
|
+
RSpec.describe Legion::Extensions::Agentic::Social::Calibration::Helpers::Constants do
|
|
6
|
+
describe 'ADVISORY_TYPES' do
|
|
7
|
+
it 'defines five advisory types' do
|
|
8
|
+
expect(described_class::ADVISORY_TYPES.size).to eq(5)
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
it 'contains expected types' do
|
|
12
|
+
expect(described_class::ADVISORY_TYPES).to include(:tone_adjustment, :verbosity_adjustment)
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
describe '.advisory_type?' do
|
|
17
|
+
it 'returns true for valid types' do
|
|
18
|
+
expect(described_class.advisory_type?(:tone_adjustment)).to be true
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
it 'returns false for invalid types' do
|
|
22
|
+
expect(described_class.advisory_type?(:bogus)).to be false
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
it 'accepts string conversion' do
|
|
26
|
+
expect(described_class.advisory_type?('partner_hint')).to be true
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
describe '.suppressed?' do
|
|
31
|
+
it 'returns true below threshold' do
|
|
32
|
+
expect(described_class.suppressed?(0.3)).to be true
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
it 'returns false at or above threshold' do
|
|
36
|
+
expect(described_class.suppressed?(0.4)).to be false
|
|
37
|
+
expect(described_class.suppressed?(0.7)).to be false
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
describe 'SIGNAL_WEIGHTS' do
|
|
42
|
+
it 'sums to 1.0' do
|
|
43
|
+
expect(described_class::SIGNAL_WEIGHTS.values.sum).to be_within(0.001).of(1.0)
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
describe 'TAG constants' do
|
|
48
|
+
it 'has correct weights tags' do
|
|
49
|
+
expect(described_class::WEIGHTS_TAGS).to eq(%w[bond calibration weights])
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
it 'has correct history tag prefix' do
|
|
53
|
+
expect(described_class::HISTORY_TAG_PREFIX).to eq(%w[bond calibration history])
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'spec_helper'
|
|
4
|
+
|
|
5
|
+
RSpec.describe Legion::Extensions::Agentic::Social::Calibration::Runners::Calibration do
|
|
6
|
+
let(:client) { Legion::Extensions::Agentic::Social::Calibration::Client.new }
|
|
7
|
+
|
|
8
|
+
describe '#update_calibration' do
|
|
9
|
+
it 'skips when no observation' do
|
|
10
|
+
result = client.update_calibration
|
|
11
|
+
expect(result[:skipped]).to eq(:no_observation)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
it 'skips when no advisory recorded' do
|
|
15
|
+
result = client.update_calibration(observation: { content: 'hi', content_length: 2 })
|
|
16
|
+
expect(result[:skipped]).to eq(:no_advisory)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
it 'produces deltas when advisory recorded' do
|
|
20
|
+
client.record_advisory_meta(advisory_id: 'test', advisory_types: [:tone_adjustment])
|
|
21
|
+
result = client.update_calibration(observation: { content: 'thanks', direct_address: true, content_length: 20 })
|
|
22
|
+
expect(result[:success]).to be true
|
|
23
|
+
expect(result[:deltas]).not_to be_empty
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
describe '#record_advisory_meta' do
|
|
28
|
+
it 'succeeds' do
|
|
29
|
+
result = client.record_advisory_meta(advisory_id: 'abc', advisory_types: [:partner_hint])
|
|
30
|
+
expect(result[:success]).to be true
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
describe '#detect_explicit_feedback' do
|
|
35
|
+
it 'returns positive for thanks' do
|
|
36
|
+
result = client.detect_explicit_feedback(content: 'thanks!')
|
|
37
|
+
expect(result[:feedback]).to eq(:positive)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
it 'returns negative for wrong' do
|
|
41
|
+
result = client.detect_explicit_feedback(content: 'wrong answer')
|
|
42
|
+
expect(result[:feedback]).to eq(:negative)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
it 'returns neutral for ambiguous' do
|
|
46
|
+
result = client.detect_explicit_feedback(content: 'okay')
|
|
47
|
+
expect(result[:feedback]).to eq(:neutral)
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
describe '#calibration_weights' do
|
|
52
|
+
it 'returns all advisory types' do
|
|
53
|
+
result = client.calibration_weights
|
|
54
|
+
expect(result[:weights].keys.size).to eq(5)
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
describe '#calibration_stats' do
|
|
59
|
+
it 'returns stats hash' do
|
|
60
|
+
result = client.calibration_stats
|
|
61
|
+
expect(result[:success]).to be true
|
|
62
|
+
expect(result).to have_key(:weights)
|
|
63
|
+
expect(result).to have_key(:history_counts)
|
|
64
|
+
expect(result).to have_key(:dirty)
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
describe '#extract_preferences_via_llm' do
|
|
69
|
+
it 'skips when LLM unavailable' do
|
|
70
|
+
result = client.extract_preferences_via_llm
|
|
71
|
+
expect(result[:skipped]).to eq(:llm_unavailable)
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
describe '#promote_partner_knowledge' do
|
|
76
|
+
it 'skips when Apollo Local unavailable' do
|
|
77
|
+
result = client.promote_partner_knowledge
|
|
78
|
+
expect(result[:skipped]).to eq(:local_unavailable)
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
describe '#sync_partner_knowledge' do
|
|
83
|
+
it 'returns combined results' do
|
|
84
|
+
result = client.sync_partner_knowledge
|
|
85
|
+
expect(result[:success]).to be true
|
|
86
|
+
expect(result[:results]).to have_key(:preferences)
|
|
87
|
+
expect(result[:results]).to have_key(:promotion)
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: lex-agentic-social
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.9
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Esity
|
|
@@ -190,6 +190,11 @@ files:
|
|
|
190
190
|
- lib/legion/extensions/agentic/social/attachment/helpers/constants.rb
|
|
191
191
|
- lib/legion/extensions/agentic/social/attachment/runners/attachment.rb
|
|
192
192
|
- lib/legion/extensions/agentic/social/attachment/version.rb
|
|
193
|
+
- lib/legion/extensions/agentic/social/calibration.rb
|
|
194
|
+
- lib/legion/extensions/agentic/social/calibration/client.rb
|
|
195
|
+
- lib/legion/extensions/agentic/social/calibration/helpers/calibration_store.rb
|
|
196
|
+
- lib/legion/extensions/agentic/social/calibration/helpers/constants.rb
|
|
197
|
+
- lib/legion/extensions/agentic/social/calibration/runners/calibration.rb
|
|
193
198
|
- lib/legion/extensions/agentic/social/conflict.rb
|
|
194
199
|
- lib/legion/extensions/agentic/social/conflict/actors/stale_check.rb
|
|
195
200
|
- lib/legion/extensions/agentic/social/conflict/client.rb
|
|
@@ -328,6 +333,9 @@ files:
|
|
|
328
333
|
- spec/legion/extensions/agentic/social/attachment/helpers/attachment_store_spec.rb
|
|
329
334
|
- spec/legion/extensions/agentic/social/attachment/helpers/constants_spec.rb
|
|
330
335
|
- spec/legion/extensions/agentic/social/attachment/runners/attachment_spec.rb
|
|
336
|
+
- spec/legion/extensions/agentic/social/calibration/helpers/calibration_store_spec.rb
|
|
337
|
+
- spec/legion/extensions/agentic/social/calibration/helpers/constants_spec.rb
|
|
338
|
+
- spec/legion/extensions/agentic/social/calibration/runners/calibration_spec.rb
|
|
331
339
|
- spec/legion/extensions/agentic/social/conflict/actors/stale_check_spec.rb
|
|
332
340
|
- spec/legion/extensions/agentic/social/conflict/client_spec.rb
|
|
333
341
|
- spec/legion/extensions/agentic/social/conflict/helpers/conflict_log_spec.rb
|