language-operator 0.1.57 → 0.1.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/lib/language_operator/agent/base.rb +19 -0
- data/lib/language_operator/agent/executor.rb +11 -0
- data/lib/language_operator/agent/task_executor.rb +77 -22
- data/lib/language_operator/agent/telemetry.rb +22 -11
- data/lib/language_operator/agent.rb +3 -0
- data/lib/language_operator/cli/base_command.rb +7 -1
- data/lib/language_operator/cli/commands/agent.rb +578 -1
- data/lib/language_operator/cli/formatters/optimization_formatter.rb +226 -0
- data/lib/language_operator/cli/formatters/progress_formatter.rb +1 -1
- data/lib/language_operator/client/base.rb +72 -2
- data/lib/language_operator/client/mcp_connector.rb +28 -6
- data/lib/language_operator/instrumentation/task_tracer.rb +64 -2
- data/lib/language_operator/kubernetes/resource_builder.rb +3 -1
- data/lib/language_operator/learning/adapters/base_adapter.rb +147 -0
- data/lib/language_operator/learning/adapters/jaeger_adapter.rb +218 -0
- data/lib/language_operator/learning/adapters/signoz_adapter.rb +432 -0
- data/lib/language_operator/learning/adapters/tempo_adapter.rb +236 -0
- data/lib/language_operator/learning/optimizer.rb +318 -0
- data/lib/language_operator/learning/pattern_detector.rb +260 -0
- data/lib/language_operator/learning/task_synthesizer.rb +261 -0
- data/lib/language_operator/learning/trace_analyzer.rb +280 -0
- data/lib/language_operator/templates/schema/agent_dsl_openapi.yaml +1 -1
- data/lib/language_operator/templates/schema/agent_dsl_schema.json +1 -1
- data/lib/language_operator/templates/task_synthesis.tmpl +97 -0
- data/lib/language_operator/tool_loader.rb +5 -3
- data/lib/language_operator/ux/concerns/provider_helpers.rb +2 -2
- data/lib/language_operator/version.rb +1 -1
- data/synth/003/Makefile +10 -0
- data/synth/003/output.log +68 -0
- data/synth/README.md +1 -3
- metadata +12 -1
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'pastel'
|
|
4
|
+
|
|
5
|
+
module LanguageOperator
|
|
6
|
+
module CLI
|
|
7
|
+
module Formatters
|
|
8
|
+
# Formats optimization analysis and proposals for CLI display
|
|
9
|
+
class OptimizationFormatter
|
|
10
|
+
def initialize
|
|
11
|
+
@pastel = Pastel.new
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# Format analysis results showing optimization opportunities
|
|
15
|
+
#
|
|
16
|
+
# @param agent_name [String] Name of the agent
|
|
17
|
+
# @param opportunities [Array<Hash>] Optimization opportunities
|
|
18
|
+
# @return [String] Formatted output
|
|
19
|
+
def format_analysis(agent_name:, opportunities:)
|
|
20
|
+
output = []
|
|
21
|
+
output << ''
|
|
22
|
+
output << @pastel.bold("Analyzing agent '#{agent_name}'...")
|
|
23
|
+
output << @pastel.dim('─' * 70)
|
|
24
|
+
output << ''
|
|
25
|
+
|
|
26
|
+
if opportunities.empty?
|
|
27
|
+
output << @pastel.yellow('No optimization opportunities found.')
|
|
28
|
+
output << ''
|
|
29
|
+
output << 'Possible reasons:'
|
|
30
|
+
output << ' • All tasks are already symbolic'
|
|
31
|
+
output << " • Neural tasks haven't executed enough times (need 10+)"
|
|
32
|
+
output << ' • Execution patterns are too inconsistent (<85%)'
|
|
33
|
+
output << ''
|
|
34
|
+
return output.join("\n")
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Group by status
|
|
38
|
+
ready = opportunities.select { |opp| opp[:ready_for_learning] }
|
|
39
|
+
not_ready = opportunities.reject { |opp| opp[:ready_for_learning] }
|
|
40
|
+
|
|
41
|
+
output << @pastel.bold("Found #{opportunities.size} neural task(s)\n")
|
|
42
|
+
|
|
43
|
+
# Show ready tasks
|
|
44
|
+
if ready.any?
|
|
45
|
+
output << @pastel.green.bold("✓ Ready for Optimization (#{ready.size})")
|
|
46
|
+
output << ''
|
|
47
|
+
ready.each do |opp|
|
|
48
|
+
output << format_opportunity(opp, ready: true)
|
|
49
|
+
output << ''
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Show not ready tasks
|
|
54
|
+
if not_ready.any?
|
|
55
|
+
output << @pastel.yellow.bold("⚠ Not Ready (#{not_ready.size})")
|
|
56
|
+
output << ''
|
|
57
|
+
not_ready.each do |opp|
|
|
58
|
+
output << format_opportunity(opp, ready: false)
|
|
59
|
+
output << ''
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Summary
|
|
64
|
+
output << @pastel.dim('─' * 70)
|
|
65
|
+
output << if ready.any?
|
|
66
|
+
@pastel.green.bold("#{ready.size}/#{opportunities.size} tasks eligible for optimization")
|
|
67
|
+
else
|
|
68
|
+
@pastel.yellow("0/#{opportunities.size} tasks ready - check requirements above")
|
|
69
|
+
end
|
|
70
|
+
output << ''
|
|
71
|
+
|
|
72
|
+
output.join("\n")
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Format a single optimization opportunity
|
|
76
|
+
#
|
|
77
|
+
# @param opp [Hash] Opportunity data
|
|
78
|
+
# @param ready [Boolean] Whether task is ready for optimization
|
|
79
|
+
# @return [String] Formatted output
|
|
80
|
+
def format_opportunity(opp, ready:)
|
|
81
|
+
output = []
|
|
82
|
+
|
|
83
|
+
# Task name
|
|
84
|
+
status_icon = ready ? @pastel.green('✓') : @pastel.yellow('⚠')
|
|
85
|
+
output << " #{status_icon} #{@pastel.bold(opp[:task_name])}"
|
|
86
|
+
|
|
87
|
+
# Metrics
|
|
88
|
+
exec_label = @pastel.dim(' Executions:')
|
|
89
|
+
exec_value = format_count_status(opp[:execution_count], 10, ready)
|
|
90
|
+
output << "#{exec_label} #{exec_value}"
|
|
91
|
+
|
|
92
|
+
cons_label = @pastel.dim(' Consistency:')
|
|
93
|
+
cons_value = format_percentage_status(opp[:consistency_score], 0.85, ready)
|
|
94
|
+
output << "#{cons_label} #{cons_value}"
|
|
95
|
+
|
|
96
|
+
# Pattern or reason
|
|
97
|
+
if ready && opp[:common_pattern]
|
|
98
|
+
output << "#{@pastel.dim(' Pattern:')} #{opp[:common_pattern]}"
|
|
99
|
+
elsif opp[:reason]
|
|
100
|
+
output << "#{@pastel.dim(' Reason:')} #{@pastel.yellow(opp[:reason])}"
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
output.join("\n")
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Format optimization proposal with diff and metrics
|
|
107
|
+
#
|
|
108
|
+
# @param proposal [Hash] Proposal data
|
|
109
|
+
# @return [String] Formatted output
|
|
110
|
+
def format_proposal(proposal:)
|
|
111
|
+
output = []
|
|
112
|
+
output << ''
|
|
113
|
+
output << @pastel.bold("Optimization Proposal: #{proposal[:task_name]}")
|
|
114
|
+
output << @pastel.dim('=' * 70)
|
|
115
|
+
output << ''
|
|
116
|
+
|
|
117
|
+
# Current code
|
|
118
|
+
output << @pastel.yellow.bold('Current (Neural):')
|
|
119
|
+
output << @pastel.dim('─' * 70)
|
|
120
|
+
proposal[:current_code].each_line do |line|
|
|
121
|
+
output << @pastel.yellow(" #{line.rstrip}")
|
|
122
|
+
end
|
|
123
|
+
output << ''
|
|
124
|
+
|
|
125
|
+
# Proposed code
|
|
126
|
+
output << @pastel.green.bold('Proposed (Symbolic):')
|
|
127
|
+
output << @pastel.dim('─' * 70)
|
|
128
|
+
proposal[:proposed_code].each_line do |line|
|
|
129
|
+
output << @pastel.green(" #{line.rstrip}")
|
|
130
|
+
end
|
|
131
|
+
output << ''
|
|
132
|
+
|
|
133
|
+
# Performance impact
|
|
134
|
+
output << @pastel.bold('Performance Impact:')
|
|
135
|
+
output << @pastel.dim('─' * 70)
|
|
136
|
+
impact = proposal[:performance_impact]
|
|
137
|
+
output << format_impact_line('Execution Time:', impact[:current_avg_time], impact[:optimized_avg_time], 's', impact[:time_reduction_pct])
|
|
138
|
+
output << format_impact_line('Cost Per Call:', impact[:current_avg_cost], impact[:optimized_avg_cost], '$', impact[:cost_reduction_pct])
|
|
139
|
+
output << ''
|
|
140
|
+
output << " #{@pastel.dim('Projected Monthly Savings:')} #{@pastel.green.bold("$#{impact[:projected_monthly_savings]}")}"
|
|
141
|
+
output << ''
|
|
142
|
+
|
|
143
|
+
# Metadata
|
|
144
|
+
output << @pastel.bold('Analysis:')
|
|
145
|
+
output << @pastel.dim('─' * 70)
|
|
146
|
+
output << " #{@pastel.dim('Executions Observed:')} #{proposal[:execution_count]}"
|
|
147
|
+
output << " #{@pastel.dim('Pattern Consistency:')} #{format_percentage(proposal[:consistency_score])}"
|
|
148
|
+
output << " #{@pastel.dim('Tool Sequence:')} #{proposal[:pattern]}"
|
|
149
|
+
output << " #{@pastel.dim('Validation:')} #{proposal[:validation_violations].empty? ? @pastel.green('✓ Passed') : @pastel.red('✗ Failed')}"
|
|
150
|
+
output << ''
|
|
151
|
+
|
|
152
|
+
output.join("\n")
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# Format success message after applying optimization
|
|
156
|
+
#
|
|
157
|
+
# @param result [Hash] Application result
|
|
158
|
+
# @return [String] Formatted output
|
|
159
|
+
def format_success(result:)
|
|
160
|
+
output = []
|
|
161
|
+
output << ''
|
|
162
|
+
|
|
163
|
+
if result[:success]
|
|
164
|
+
output << @pastel.green.bold('✓ Optimization applied successfully!')
|
|
165
|
+
output << ''
|
|
166
|
+
output << " Task '#{result[:task_name]}' has been optimized to symbolic execution."
|
|
167
|
+
output << ''
|
|
168
|
+
output << @pastel.dim('Next steps:')
|
|
169
|
+
output << " • Monitor performance: aictl agent logs #{result[:task_name]}"
|
|
170
|
+
output << " • View changes: aictl agent code #{result[:task_name]}"
|
|
171
|
+
else
|
|
172
|
+
output << @pastel.red.bold('✗ Optimization failed!')
|
|
173
|
+
output << ''
|
|
174
|
+
output << " Task: #{result[:task_name]}"
|
|
175
|
+
output << " Error: #{result[:error]}"
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
output << ''
|
|
179
|
+
output.join("\n")
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
private
|
|
183
|
+
|
|
184
|
+
# Format a count with status indicator
|
|
185
|
+
def format_count_status(count, threshold, ready)
|
|
186
|
+
if count >= threshold
|
|
187
|
+
@pastel.green("#{count} (≥#{threshold})")
|
|
188
|
+
else
|
|
189
|
+
ready ? @pastel.yellow("#{count}/#{threshold}") : @pastel.red("#{count}/#{threshold}")
|
|
190
|
+
end
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# Format a percentage with status indicator
|
|
194
|
+
def format_percentage_status(score, threshold, ready)
|
|
195
|
+
return @pastel.red('N/A') if score.nil?
|
|
196
|
+
|
|
197
|
+
pct = (score * 100).round(1)
|
|
198
|
+
threshold_pct = (threshold * 100).round(1)
|
|
199
|
+
|
|
200
|
+
if score >= threshold
|
|
201
|
+
@pastel.green("#{pct}% (≥#{threshold_pct}%)")
|
|
202
|
+
else
|
|
203
|
+
ready ? @pastel.yellow("#{pct}%/#{threshold_pct}%") : @pastel.red("#{pct}%/#{threshold_pct}%")
|
|
204
|
+
end
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# Format percentage value
|
|
208
|
+
def format_percentage(score)
|
|
209
|
+
return @pastel.red('N/A') if score.nil?
|
|
210
|
+
|
|
211
|
+
pct = (score * 100).round(1)
|
|
212
|
+
@pastel.green("#{pct}%")
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# Format performance impact line
|
|
216
|
+
def format_impact_line(label, current, optimized, unit, reduction_pct)
|
|
217
|
+
current_str = unit == '$' ? format('$%.4f', current) : "#{current}#{unit}"
|
|
218
|
+
optimized_str = unit == '$' ? format('$%.4f', optimized) : "#{optimized}#{unit}"
|
|
219
|
+
|
|
220
|
+
" #{@pastel.dim(label)} #{current_str} → #{@pastel.green(optimized_str)} " \
|
|
221
|
+
"#{@pastel.green("(#{reduction_pct}% faster)")}"
|
|
222
|
+
end
|
|
223
|
+
end
|
|
224
|
+
end
|
|
225
|
+
end
|
|
226
|
+
end
|
|
@@ -115,9 +115,18 @@ module LanguageOperator
|
|
|
115
115
|
|
|
116
116
|
# Get all available tools from connected servers
|
|
117
117
|
#
|
|
118
|
-
#
|
|
118
|
+
# Wraps MCP tools with OpenTelemetry instrumentation to trace tool executions.
|
|
119
|
+
#
|
|
120
|
+
# @return [Array] Array of instrumented tool objects
|
|
119
121
|
def tools
|
|
120
|
-
@clients.flat_map(&:tools)
|
|
122
|
+
raw_tools = @clients.flat_map(&:tools)
|
|
123
|
+
|
|
124
|
+
# Wrap each tool with instrumentation if telemetry is enabled
|
|
125
|
+
if ENV.fetch('OTEL_EXPORTER_OTLP_ENDPOINT', nil)
|
|
126
|
+
raw_tools.map { |tool| wrap_tool_with_instrumentation(tool) }
|
|
127
|
+
else
|
|
128
|
+
raw_tools
|
|
129
|
+
end
|
|
121
130
|
end
|
|
122
131
|
|
|
123
132
|
# Get information about connected servers
|
|
@@ -165,6 +174,67 @@ module LanguageOperator
|
|
|
165
174
|
'Client'
|
|
166
175
|
end
|
|
167
176
|
|
|
177
|
+
# Wrap an MCP tool with OpenTelemetry instrumentation
|
|
178
|
+
#
|
|
179
|
+
# Creates a wrapper that traces tool executions with proper semantic conventions.
|
|
180
|
+
# The wrapper preserves the original tool's interface while adding telemetry.
|
|
181
|
+
#
|
|
182
|
+
# @param tool [Object] Original MCP tool object
|
|
183
|
+
# @return [Object] Instrumented tool wrapper
|
|
184
|
+
def wrap_tool_with_instrumentation(tool)
|
|
185
|
+
# Create a new tool object that wraps the original
|
|
186
|
+
tool_wrapper = Object.new
|
|
187
|
+
tool_wrapper.define_singleton_method(:name) { tool.name }
|
|
188
|
+
tool_wrapper.define_singleton_method(:description) { tool.description }
|
|
189
|
+
tool_wrapper.define_singleton_method(:parameters) { tool.parameters }
|
|
190
|
+
tool_wrapper.define_singleton_method(:params_schema) { tool.params_schema }
|
|
191
|
+
tool_wrapper.define_singleton_method(:provider_params) { tool.provider_params }
|
|
192
|
+
|
|
193
|
+
# Wrap the call method with instrumentation
|
|
194
|
+
original_tool = tool
|
|
195
|
+
tool_wrapper.define_singleton_method(:call) do |arguments|
|
|
196
|
+
# Get current OpenTelemetry context at call time (not definition time)
|
|
197
|
+
# This ensures we pick up the active span from the task execution
|
|
198
|
+
tracer = OpenTelemetry.tracer_provider.tracer('language-operator')
|
|
199
|
+
tool_name = original_tool.name
|
|
200
|
+
|
|
201
|
+
# Get current span to check if we have a parent
|
|
202
|
+
current_span = OpenTelemetry::Trace.current_span
|
|
203
|
+
current_span_context = current_span.context if current_span
|
|
204
|
+
|
|
205
|
+
# Log trace ID for debugging
|
|
206
|
+
if current_span_context&.valid?
|
|
207
|
+
trace_id = current_span_context.hex_trace_id
|
|
208
|
+
LanguageOperator.logger.debug('Tool call with parent trace', tool: tool_name, trace_id: trace_id) if defined?(LanguageOperator.logger)
|
|
209
|
+
elsif defined?(LanguageOperator.logger)
|
|
210
|
+
LanguageOperator.logger.debug('Tool call without parent trace', tool: tool_name)
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# Create span in the current context - this will automatically attach to
|
|
214
|
+
# the active span if one exists
|
|
215
|
+
tracer.in_span("execute_tool.#{tool_name}", attributes: {
|
|
216
|
+
'gen_ai.operation.name' => 'execute_tool',
|
|
217
|
+
'gen_ai.tool.name' => tool_name,
|
|
218
|
+
'gen_ai.tool.call.arguments.size' => arguments.to_json.bytesize
|
|
219
|
+
}) do |span|
|
|
220
|
+
# Execute the original tool
|
|
221
|
+
result = original_tool.call(arguments)
|
|
222
|
+
|
|
223
|
+
# Record the result size
|
|
224
|
+
result_str = result.is_a?(String) ? result : result.to_json
|
|
225
|
+
span.set_attribute('gen_ai.tool.call.result.size', result_str.bytesize)
|
|
226
|
+
|
|
227
|
+
result
|
|
228
|
+
rescue StandardError => e
|
|
229
|
+
span.record_exception(e)
|
|
230
|
+
span.status = OpenTelemetry::Trace::Status.error("Tool execution failed: #{e.message}")
|
|
231
|
+
raise
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
tool_wrapper
|
|
236
|
+
end
|
|
237
|
+
|
|
168
238
|
# Configure RubyLLM with provider settings
|
|
169
239
|
#
|
|
170
240
|
# @raise [RuntimeError] If provider is unknown
|
|
@@ -12,8 +12,6 @@ module LanguageOperator
|
|
|
12
12
|
def connect_mcp_servers
|
|
13
13
|
enabled_servers = @config['mcp_servers'].select { |s| s['enabled'] }
|
|
14
14
|
|
|
15
|
-
all_tools = []
|
|
16
|
-
|
|
17
15
|
if enabled_servers.empty?
|
|
18
16
|
logger.info('No MCP servers configured, agent will run without tools')
|
|
19
17
|
else
|
|
@@ -25,7 +23,6 @@ module LanguageOperator
|
|
|
25
23
|
|
|
26
24
|
@clients << client
|
|
27
25
|
tool_count = client.tools.length
|
|
28
|
-
all_tools.concat(client.tools)
|
|
29
26
|
|
|
30
27
|
# Debug: inspect tool objects
|
|
31
28
|
if @debug
|
|
@@ -50,18 +47,22 @@ module LanguageOperator
|
|
|
50
47
|
end
|
|
51
48
|
|
|
52
49
|
logger.info('MCP connection summary',
|
|
53
|
-
connected_servers: @clients.length
|
|
54
|
-
total_tools: all_tools.length)
|
|
50
|
+
connected_servers: @clients.length)
|
|
55
51
|
end
|
|
56
52
|
|
|
57
53
|
# Create chat with all collected tools (even if empty)
|
|
54
|
+
# Use the tools() method which applies OpenTelemetry instrumentation wrapping
|
|
58
55
|
llm_config = @config['llm']
|
|
59
56
|
chat_params = build_chat_params(llm_config)
|
|
60
57
|
@chat = RubyLLM.chat(**chat_params)
|
|
61
58
|
|
|
59
|
+
all_tools = tools
|
|
62
60
|
@chat.with_tools(*all_tools) unless all_tools.empty?
|
|
63
61
|
|
|
64
|
-
|
|
62
|
+
# Set up callbacks to log tool invocations
|
|
63
|
+
setup_tool_callbacks
|
|
64
|
+
|
|
65
|
+
logger.info('Chat session initialized', with_tools: !all_tools.empty?, total_tools: all_tools.length)
|
|
65
66
|
end
|
|
66
67
|
|
|
67
68
|
# Connect to MCP server with exponential backoff retry logic
|
|
@@ -125,6 +126,27 @@ module LanguageOperator
|
|
|
125
126
|
|
|
126
127
|
chat_params
|
|
127
128
|
end
|
|
129
|
+
|
|
130
|
+
# Set up callbacks to log tool calls and results
|
|
131
|
+
#
|
|
132
|
+
# @return [void]
|
|
133
|
+
def setup_tool_callbacks
|
|
134
|
+
@chat.on_tool_call do |tool_call|
|
|
135
|
+
logger.info('Tool call initiated by LLM',
|
|
136
|
+
event: 'tool_call_initiated',
|
|
137
|
+
tool_name: tool_call.name,
|
|
138
|
+
tool_id: tool_call.id,
|
|
139
|
+
arguments: tool_call.arguments,
|
|
140
|
+
arguments_json: tool_call.arguments.to_json)
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
@chat.on_tool_result do |result|
|
|
144
|
+
logger.info('Tool call result received',
|
|
145
|
+
event: 'tool_result_received',
|
|
146
|
+
result: result,
|
|
147
|
+
result_preview: result.to_s[0..500])
|
|
148
|
+
end
|
|
149
|
+
end
|
|
128
150
|
end
|
|
129
151
|
end
|
|
130
152
|
end
|
|
@@ -26,6 +26,7 @@ module LanguageOperator
|
|
|
26
26
|
# Instrumentation adds <5% overhead with default settings
|
|
27
27
|
# Overhead may increase to ~10% with full data capture enabled
|
|
28
28
|
#
|
|
29
|
+
# rubocop:disable Metrics/ModuleLength
|
|
29
30
|
module TaskTracer
|
|
30
31
|
# Maximum length for captured data before truncation
|
|
31
32
|
MAX_CAPTURED_LENGTH = 1000
|
|
@@ -169,6 +170,11 @@ module LanguageOperator
|
|
|
169
170
|
return unless response.respond_to?(:tool_calls)
|
|
170
171
|
return unless response.tool_calls&.any?
|
|
171
172
|
|
|
173
|
+
logger&.info('Tool calls detected in LLM response',
|
|
174
|
+
event: 'tool_calls_detected',
|
|
175
|
+
tool_call_count: response.tool_calls.length,
|
|
176
|
+
tool_names: response.tool_calls.map { |tc| extract_tool_name(tc) })
|
|
177
|
+
|
|
172
178
|
response.tool_calls.each do |tool_call|
|
|
173
179
|
record_single_tool_call(tool_call, parent_span)
|
|
174
180
|
end
|
|
@@ -182,11 +188,26 @@ module LanguageOperator
|
|
|
182
188
|
# @param parent_span [OpenTelemetry::Trace::Span] Parent span
|
|
183
189
|
def record_single_tool_call(tool_call, _parent_span)
|
|
184
190
|
tool_name = extract_tool_name(tool_call)
|
|
191
|
+
tool_id = tool_call.respond_to?(:id) ? tool_call.id : nil
|
|
192
|
+
|
|
193
|
+
# Extract and log tool arguments
|
|
194
|
+
arguments = extract_tool_arguments(tool_call)
|
|
185
195
|
|
|
196
|
+
logger&.info('Tool invoked by LLM',
|
|
197
|
+
event: 'tool_call_invoked',
|
|
198
|
+
tool_name: tool_name,
|
|
199
|
+
tool_id: tool_id,
|
|
200
|
+
arguments: arguments,
|
|
201
|
+
arguments_json: (arguments.is_a?(Hash) ? JSON.generate(arguments) : arguments.to_s))
|
|
202
|
+
|
|
203
|
+
start_time = Time.now
|
|
186
204
|
tracer.in_span("execute_tool #{tool_name}", attributes: build_tool_call_attributes(tool_call)) do |tool_span|
|
|
187
205
|
# Tool execution already completed by ruby_llm
|
|
188
206
|
# Just record the metadata
|
|
189
|
-
|
|
207
|
+
if tool_call.respond_to?(:result) && tool_call.result
|
|
208
|
+
duration_ms = ((Time.now - start_time) * 1000).round(2)
|
|
209
|
+
record_tool_result(tool_call.result, tool_span, tool_name, tool_id, duration_ms)
|
|
210
|
+
end
|
|
190
211
|
end
|
|
191
212
|
rescue StandardError => e
|
|
192
213
|
logger&.warn('Failed to record tool call span', error: e.message, tool: tool_name)
|
|
@@ -206,6 +227,34 @@ module LanguageOperator
|
|
|
206
227
|
end
|
|
207
228
|
end
|
|
208
229
|
|
|
230
|
+
# Extract tool arguments from tool call object
|
|
231
|
+
#
|
|
232
|
+
# @param tool_call [Object] Tool call object
|
|
233
|
+
# @return [Hash, String] Tool arguments
|
|
234
|
+
def extract_tool_arguments(tool_call)
|
|
235
|
+
if tool_call.respond_to?(:arguments)
|
|
236
|
+
args = tool_call.arguments
|
|
237
|
+
parse_json_args(args)
|
|
238
|
+
elsif tool_call.respond_to?(:function) && tool_call.function.respond_to?(:arguments)
|
|
239
|
+
args = tool_call.function.arguments
|
|
240
|
+
parse_json_args(args)
|
|
241
|
+
else
|
|
242
|
+
{}
|
|
243
|
+
end
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
# Parse JSON arguments safely
|
|
247
|
+
#
|
|
248
|
+
# @param args [String, Object] Arguments to parse
|
|
249
|
+
# @return [Hash, String] Parsed arguments or original
|
|
250
|
+
def parse_json_args(args)
|
|
251
|
+
return args unless args.is_a?(String)
|
|
252
|
+
|
|
253
|
+
JSON.parse(args)
|
|
254
|
+
rescue JSON::ParserError
|
|
255
|
+
args
|
|
256
|
+
end
|
|
257
|
+
|
|
209
258
|
# Build attributes for tool call span
|
|
210
259
|
#
|
|
211
260
|
# @param tool_call [Object] Tool call object
|
|
@@ -244,13 +293,25 @@ module LanguageOperator
|
|
|
244
293
|
#
|
|
245
294
|
# @param result [Object] Tool call result
|
|
246
295
|
# @param span [OpenTelemetry::Trace::Span] The span to update
|
|
247
|
-
|
|
296
|
+
# @param tool_name [String] Tool name (for logging)
|
|
297
|
+
# @param tool_id [String] Tool call ID (for logging)
|
|
298
|
+
# @param duration_ms [Float] Execution duration in milliseconds (for logging)
|
|
299
|
+
def record_tool_result(result, span, tool_name = nil, tool_id = nil, duration_ms = nil)
|
|
248
300
|
result_str = result.is_a?(String) ? result : JSON.generate(result)
|
|
249
301
|
span.set_attribute('gen_ai.tool.call.result.size', result_str.bytesize)
|
|
250
302
|
|
|
251
303
|
if (sanitized_result = sanitize_data(result, :tool_results))
|
|
252
304
|
span.set_attribute('gen_ai.tool.call.result', sanitized_result)
|
|
253
305
|
end
|
|
306
|
+
|
|
307
|
+
# Log tool execution completion
|
|
308
|
+
logger&.info('Tool execution completed',
|
|
309
|
+
event: 'tool_call_completed',
|
|
310
|
+
tool_name: tool_name,
|
|
311
|
+
tool_id: tool_id,
|
|
312
|
+
result_size: result_str.bytesize,
|
|
313
|
+
result: sanitize_data(result, :tool_results),
|
|
314
|
+
duration_ms: duration_ms)
|
|
254
315
|
rescue StandardError => e
|
|
255
316
|
logger&.warn('Failed to record tool result', error: e.message)
|
|
256
317
|
end
|
|
@@ -281,5 +342,6 @@ module LanguageOperator
|
|
|
281
342
|
logger&.warn('Failed to record output metadata', error: e.message)
|
|
282
343
|
end
|
|
283
344
|
end
|
|
345
|
+
# rubocop:enable Metrics/ModuleLength
|
|
284
346
|
end
|
|
285
347
|
end
|
|
@@ -25,7 +25,7 @@ module LanguageOperator
|
|
|
25
25
|
|
|
26
26
|
# Build a LanguageAgent resource
|
|
27
27
|
def language_agent(name, instructions:, cluster: nil, schedule: nil, persona: nil, tools: [], models: [],
|
|
28
|
-
mode: nil, labels: {})
|
|
28
|
+
mode: nil, workspace: true, labels: {})
|
|
29
29
|
# Determine mode: reactive, scheduled, or autonomous
|
|
30
30
|
spec_mode = mode || (schedule ? 'scheduled' : 'autonomous')
|
|
31
31
|
|
|
@@ -41,6 +41,8 @@ module LanguageOperator
|
|
|
41
41
|
spec['toolRefs'] = tools.map { |t| { 'name' => t } } unless tools.empty?
|
|
42
42
|
# Convert model names to modelRef objects
|
|
43
43
|
spec['modelRefs'] = models.map { |m| { 'name' => m } } unless models.empty?
|
|
44
|
+
# Enable workspace by default for state persistence
|
|
45
|
+
spec['workspace'] = { 'enabled' => workspace } if workspace
|
|
44
46
|
|
|
45
47
|
{
|
|
46
48
|
'apiVersion' => 'langop.io/v1alpha1',
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module LanguageOperator
|
|
4
|
+
module Learning
|
|
5
|
+
module Adapters
|
|
6
|
+
# Abstract base class for OTLP backend query adapters
|
|
7
|
+
#
|
|
8
|
+
# Defines the interface that all backend adapters must implement.
|
|
9
|
+
# Adapters translate generic query requests into backend-specific
|
|
10
|
+
# API calls and normalize responses.
|
|
11
|
+
#
|
|
12
|
+
# @example Implementing a custom adapter
|
|
13
|
+
# class CustomAdapter < BaseAdapter
|
|
14
|
+
# def self.available?(endpoint, api_key = nil)
|
|
15
|
+
# # Check if backend is reachable
|
|
16
|
+
# true
|
|
17
|
+
# end
|
|
18
|
+
#
|
|
19
|
+
# def query_spans(filter:, time_range:, limit:)
|
|
20
|
+
# # Query backend and return normalized spans
|
|
21
|
+
# []
|
|
22
|
+
# end
|
|
23
|
+
# end
|
|
24
|
+
class BaseAdapter
|
|
25
|
+
# Initialize adapter with connection details
|
|
26
|
+
#
|
|
27
|
+
# @param endpoint [String] Backend endpoint URL
|
|
28
|
+
# @param api_key [String, nil] API key for authentication (if required)
|
|
29
|
+
# @param options [Hash] Additional adapter-specific options
|
|
30
|
+
def initialize(endpoint, api_key = nil, **options)
|
|
31
|
+
@endpoint = endpoint
|
|
32
|
+
@api_key = api_key
|
|
33
|
+
@logger = options[:logger] || ::Logger.new($stdout, level: ::Logger::WARN)
|
|
34
|
+
@options = options
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Check if this backend is available at the given endpoint
|
|
38
|
+
#
|
|
39
|
+
# @param endpoint [String] Backend endpoint URL
|
|
40
|
+
# @param api_key [String, nil] API key for authentication (optional)
|
|
41
|
+
# @return [Boolean] True if backend is reachable and compatible
|
|
42
|
+
def self.available?(endpoint, api_key = nil)
|
|
43
|
+
raise NotImplementedError, "#{self}.available? must be implemented"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Query spans from the backend
|
|
47
|
+
#
|
|
48
|
+
# @param filter [Hash] Filter criteria
|
|
49
|
+
# @option filter [String] :task_name Task name to filter by
|
|
50
|
+
# @option filter [Hash] :attributes Additional span attributes to match
|
|
51
|
+
# @param time_range [Range<Time>] Time range for query
|
|
52
|
+
# @param limit [Integer] Maximum number of spans to return
|
|
53
|
+
# @return [Array<Hash>] Array of normalized span hashes
|
|
54
|
+
def query_spans(filter:, time_range:, limit:)
|
|
55
|
+
raise NotImplementedError, "#{self.class}#query_spans must be implemented"
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Extract task execution data from spans
|
|
59
|
+
#
|
|
60
|
+
# Groups spans by trace and extracts task-specific metadata:
|
|
61
|
+
# - inputs: Task input parameters
|
|
62
|
+
# - outputs: Task output values
|
|
63
|
+
# - tool_calls: Sequence of tools invoked
|
|
64
|
+
# - duration: Execution duration
|
|
65
|
+
#
|
|
66
|
+
# @param spans [Array<Hash>] Raw spans from backend
|
|
67
|
+
# @return [Array<Hash>] Task execution data grouped by trace
|
|
68
|
+
def extract_task_data(spans)
|
|
69
|
+
spans.group_by { |span| span[:trace_id] }.map do |trace_id, trace_spans|
|
|
70
|
+
task_span = trace_spans.find { |s| s[:name]&.include?('task_executor') }
|
|
71
|
+
next unless task_span
|
|
72
|
+
|
|
73
|
+
{
|
|
74
|
+
trace_id: trace_id,
|
|
75
|
+
task_name: task_span.dig(:attributes, 'task.name'),
|
|
76
|
+
inputs: extract_inputs(task_span),
|
|
77
|
+
outputs: extract_outputs(task_span),
|
|
78
|
+
tool_calls: extract_tool_calls(trace_spans),
|
|
79
|
+
duration_ms: task_span[:duration_ms],
|
|
80
|
+
timestamp: task_span[:timestamp]
|
|
81
|
+
}
|
|
82
|
+
end.compact
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
protected
|
|
86
|
+
|
|
87
|
+
attr_reader :endpoint, :api_key, :options
|
|
88
|
+
|
|
89
|
+
# Extract inputs from task span attributes
|
|
90
|
+
#
|
|
91
|
+
# @param span [Hash] Task execution span
|
|
92
|
+
# @return [Hash] Input parameters
|
|
93
|
+
def extract_inputs(span)
|
|
94
|
+
attrs = span[:attributes] || {}
|
|
95
|
+
input_keys = attrs['task.input.keys']&.split(',') || []
|
|
96
|
+
|
|
97
|
+
input_keys.each_with_object({}) do |key, inputs|
|
|
98
|
+
value_attr = "task.input.#{key}"
|
|
99
|
+
inputs[key.to_sym] = attrs[value_attr] if attrs[value_attr]
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Extract outputs from task span attributes
|
|
104
|
+
#
|
|
105
|
+
# @param span [Hash] Task execution span
|
|
106
|
+
# @return [Hash] Output values
|
|
107
|
+
def extract_outputs(span)
|
|
108
|
+
attrs = span[:attributes] || {}
|
|
109
|
+
output_keys = attrs['task.output.keys']&.split(',') || []
|
|
110
|
+
|
|
111
|
+
output_keys.each_with_object({}) do |key, outputs|
|
|
112
|
+
value_attr = "task.output.#{key}"
|
|
113
|
+
outputs[key.to_sym] = attrs[value_attr] if attrs[value_attr]
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Extract tool call sequence from trace spans
|
|
118
|
+
#
|
|
119
|
+
# @param trace_spans [Array<Hash>] All spans in trace
|
|
120
|
+
# @return [Array<Hash>] Ordered tool call sequence
|
|
121
|
+
def extract_tool_calls(trace_spans)
|
|
122
|
+
trace_spans
|
|
123
|
+
.select { |s| s.dig(:attributes, 'gen_ai.operation.name') == 'execute_tool' }
|
|
124
|
+
.sort_by { |s| s[:timestamp] }
|
|
125
|
+
.map do |tool_span|
|
|
126
|
+
{
|
|
127
|
+
tool_name: tool_span.dig(:attributes, 'gen_ai.tool.name'),
|
|
128
|
+
arguments_size: tool_span.dig(:attributes, 'gen_ai.tool.call.arguments.size'),
|
|
129
|
+
result_size: tool_span.dig(:attributes, 'gen_ai.tool.call.result.size')
|
|
130
|
+
}
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Parse time range into backend-specific format
|
|
135
|
+
#
|
|
136
|
+
# @param time_range [Range<Time>] Time range
|
|
137
|
+
# @return [Hash] Start and end times
|
|
138
|
+
def parse_time_range(time_range)
|
|
139
|
+
{
|
|
140
|
+
start: time_range.begin || (Time.now - (24 * 60 * 60)),
|
|
141
|
+
end: time_range.end || Time.now
|
|
142
|
+
}
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
end
|