opentelemetry-instrumentation-ruby_llm 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +9 -1
- data/.gitignore +1 -0
- data/Appraisals +18 -0
- data/Gemfile +4 -1
- data/README.md +103 -3
- data/gemfiles/ruby_llm_1.12.1.gemfile +18 -0
- data/gemfiles/ruby_llm_1.8.0.gemfile +18 -0
- data/gemfiles/ruby_llm_1_latest.gemfile +18 -0
- data/lib/opentelemetry/instrumentation/ruby_llm/instrumentation.rb +44 -0
- data/lib/opentelemetry/instrumentation/ruby_llm/message_formatter.rb +97 -0
- data/lib/opentelemetry/instrumentation/ruby_llm/patches/agent.rb +84 -0
- data/lib/opentelemetry/instrumentation/ruby_llm/patches/chat.rb +44 -32
- data/lib/opentelemetry/instrumentation/ruby_llm/patches/chat_methods.rb +17 -0
- data/lib/opentelemetry/instrumentation/ruby_llm/version.rb +1 -1
- data/test/active_record_chat_methods_test.rb +195 -0
- data/test/agent_instrumentation_test.rb +245 -0
- data/test/instrumentation_test.rb +264 -190
- data/test/test_helper.rb +30 -0
- metadata +10 -1
|
@@ -1,37 +1,44 @@
|
|
|
1
1
|
require "test_helper"
|
|
2
2
|
|
|
3
3
|
class InstrumentationTest < Minitest::Test
|
|
4
|
+
include ChatCompletionStubs
|
|
5
|
+
|
|
4
6
|
def setup
|
|
5
7
|
EXPORTER.reset
|
|
6
8
|
|
|
7
9
|
RubyLLM.configure do |c|
|
|
8
10
|
c.openai_api_key = "fake-key-for-testing"
|
|
11
|
+
c.anthropic_api_key = "fake-key-for-testing"
|
|
9
12
|
end
|
|
10
13
|
end
|
|
11
14
|
|
|
15
|
+
def test_compatible_is_true_for_current_ruby_llm_version
|
|
16
|
+
instrumentation = OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance
|
|
17
|
+
assert_equal true, instrumentation.compatible?
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def test_compatible_is_false_when_ruby_llm_below_minimum
|
|
21
|
+
original_version = ::RubyLLM::VERSION
|
|
22
|
+
::RubyLLM.send(:remove_const, :VERSION)
|
|
23
|
+
::RubyLLM.const_set(:VERSION, "1.7.99")
|
|
24
|
+
|
|
25
|
+
instrumentation = OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance
|
|
26
|
+
assert_equal false, instrumentation.compatible?
|
|
27
|
+
ensure
|
|
28
|
+
::RubyLLM.send(:remove_const, :VERSION)
|
|
29
|
+
::RubyLLM.const_set(:VERSION, original_version)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def test_minimum_ruby_llm_version_is_pinned_at_1_8_0
|
|
33
|
+
assert_equal "1.8.0", OpenTelemetry::Instrumentation::RubyLLM::Instrumentation::MINIMUM_RUBY_LLM_VERSION
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def test_agent_minimum_ruby_llm_version_is_pinned_at_1_12_1
|
|
37
|
+
assert_equal "1.12.1", OpenTelemetry::Instrumentation::RubyLLM::Instrumentation::AGENT_MINIMUM_RUBY_LLM_VERSION
|
|
38
|
+
end
|
|
39
|
+
|
|
12
40
|
def test_creates_span_with_attributes
|
|
13
|
-
|
|
14
|
-
.to_return(
|
|
15
|
-
status: 200,
|
|
16
|
-
headers: { "Content-Type" => "application/json" },
|
|
17
|
-
body: {
|
|
18
|
-
id: "chatcmpl-123",
|
|
19
|
-
object: "chat.completion",
|
|
20
|
-
model: "gpt-4o-mini",
|
|
21
|
-
choices: [
|
|
22
|
-
{
|
|
23
|
-
index: 0,
|
|
24
|
-
message: { role: "assistant", content: "Hello, world!" },
|
|
25
|
-
finish_reason: "stop"
|
|
26
|
-
}
|
|
27
|
-
],
|
|
28
|
-
usage: {
|
|
29
|
-
prompt_tokens: 10,
|
|
30
|
-
completion_tokens: 5,
|
|
31
|
-
total_tokens: 15
|
|
32
|
-
}
|
|
33
|
-
}.to_json
|
|
34
|
-
)
|
|
41
|
+
stub_chat_completion
|
|
35
42
|
|
|
36
43
|
chat = RubyLLM.chat(model: "gpt-4o-mini")
|
|
37
44
|
chat.ask("Hi")
|
|
@@ -45,10 +52,83 @@ class InstrumentationTest < Minitest::Test
|
|
|
45
52
|
assert_equal "openai", span.attributes["gen_ai.provider.name"]
|
|
46
53
|
assert_equal "gpt-4o-mini", span.attributes["gen_ai.request.model"]
|
|
47
54
|
assert_equal "chat", span.attributes["gen_ai.operation.name"]
|
|
55
|
+
# Per GenAI semconv, `gen_ai.request.stream` is set only when streaming.
|
|
56
|
+
assert_nil span.attributes["gen_ai.request.stream"]
|
|
48
57
|
assert_equal 10, span.attributes["gen_ai.usage.input_tokens"]
|
|
49
58
|
assert_equal 5, span.attributes["gen_ai.usage.output_tokens"]
|
|
50
59
|
end
|
|
51
60
|
|
|
61
|
+
def test_marks_streaming_chat_requests
|
|
62
|
+
stub_chat_completion(
|
|
63
|
+
chat_completion_body(content: "Hi", usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 })
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
chat = RubyLLM.chat(model: "gpt-4o-mini")
|
|
67
|
+
chat.ask("Hi") { |_chunk| }
|
|
68
|
+
|
|
69
|
+
span = EXPORTER.finished_spans.first
|
|
70
|
+
assert_equal true, span.attributes["gen_ai.request.stream"]
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def test_records_openai_prompt_cache_read_tokens
|
|
74
|
+
# OpenAI exposes only `cached_tokens` via `prompt_tokens_details.cached_tokens`.
|
|
75
|
+
# Its provider in ruby_llm does not surface a `cache_creation_tokens` value
|
|
76
|
+
# until 1.15.0 (we only assert the cache-read attribute here).
|
|
77
|
+
# The accessor itself was added in ruby_llm 1.9.0.
|
|
78
|
+
skip "cached_tokens accessor not available before ruby_llm 1.9.0" unless RubyLLM::Message.instance_methods.include?(:cached_tokens)
|
|
79
|
+
stub_chat_completion(
|
|
80
|
+
chat_completion_body(
|
|
81
|
+
content: "Hello!",
|
|
82
|
+
usage: {
|
|
83
|
+
prompt_tokens: 100,
|
|
84
|
+
completion_tokens: 5,
|
|
85
|
+
total_tokens: 105,
|
|
86
|
+
prompt_tokens_details: { cached_tokens: 75 }
|
|
87
|
+
}
|
|
88
|
+
)
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
chat = RubyLLM.chat(model: "gpt-4o-mini")
|
|
92
|
+
chat.ask("Hi")
|
|
93
|
+
|
|
94
|
+
span = EXPORTER.finished_spans.first
|
|
95
|
+
assert_equal 75, span.attributes["gen_ai.usage.cache_read.input_tokens"]
|
|
96
|
+
assert_equal 0, span.attributes["gen_ai.usage.cache_creation.input_tokens"]
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def test_records_anthropic_prompt_cache_tokens
|
|
100
|
+
# Anthropic's provider surfaces both `cached_tokens` (via
|
|
101
|
+
# `cache_read_input_tokens`) and `cache_creation_tokens` (via
|
|
102
|
+
# `cache_creation_input_tokens`). Accessors were added in ruby_llm 1.9.0.
|
|
103
|
+
skip "cache token accessors not available before ruby_llm 1.9.0" unless RubyLLM::Message.instance_methods.include?(:cache_creation_tokens)
|
|
104
|
+
stub_request(:post, "https://api.anthropic.com/v1/messages")
|
|
105
|
+
.to_return(
|
|
106
|
+
status: 200,
|
|
107
|
+
headers: { "Content-Type" => "application/json" },
|
|
108
|
+
body: {
|
|
109
|
+
id: "msg_cache",
|
|
110
|
+
type: "message",
|
|
111
|
+
role: "assistant",
|
|
112
|
+
model: "claude-3-5-sonnet-20241022",
|
|
113
|
+
content: [{ type: "text", text: "Hello!" }],
|
|
114
|
+
stop_reason: "end_turn",
|
|
115
|
+
usage: {
|
|
116
|
+
input_tokens: 100,
|
|
117
|
+
output_tokens: 5,
|
|
118
|
+
cache_read_input_tokens: 75,
|
|
119
|
+
cache_creation_input_tokens: 20
|
|
120
|
+
}
|
|
121
|
+
}.to_json
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
chat = RubyLLM.chat(model: "claude-3-5-sonnet-20241022")
|
|
125
|
+
chat.ask("Hi")
|
|
126
|
+
|
|
127
|
+
span = EXPORTER.finished_spans.first
|
|
128
|
+
assert_equal 75, span.attributes["gen_ai.usage.cache_read.input_tokens"]
|
|
129
|
+
assert_equal 20, span.attributes["gen_ai.usage.cache_creation.input_tokens"]
|
|
130
|
+
end
|
|
131
|
+
|
|
52
132
|
def test_records_error_on_api_failure
|
|
53
133
|
stub_request(:post, "https://api.openai.com/v1/chat/completions")
|
|
54
134
|
.to_return(status: 500, body: "Internal Server Error")
|
|
@@ -68,22 +148,7 @@ class InstrumentationTest < Minitest::Test
|
|
|
68
148
|
end
|
|
69
149
|
|
|
70
150
|
def test_instruments_complete_called_directly
|
|
71
|
-
|
|
72
|
-
.to_return(
|
|
73
|
-
status: 200,
|
|
74
|
-
headers: { "Content-Type" => "application/json" },
|
|
75
|
-
body: {
|
|
76
|
-
id: "chatcmpl-123",
|
|
77
|
-
object: "chat.completion",
|
|
78
|
-
model: "gpt-4o-mini",
|
|
79
|
-
choices: [{
|
|
80
|
-
index: 0,
|
|
81
|
-
message: { role: "assistant", content: "Hello, world!" },
|
|
82
|
-
finish_reason: "stop"
|
|
83
|
-
}],
|
|
84
|
-
usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
|
|
85
|
-
}.to_json
|
|
86
|
-
)
|
|
151
|
+
stub_chat_completion
|
|
87
152
|
|
|
88
153
|
chat = RubyLLM.chat(model: "gpt-4o-mini")
|
|
89
154
|
chat.add_message(role: :user, content: "Hi")
|
|
@@ -111,47 +176,20 @@ class InstrumentationTest < Minitest::Test
|
|
|
111
176
|
end
|
|
112
177
|
end
|
|
113
178
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
role: "assistant",
|
|
127
|
-
content: nil,
|
|
128
|
-
tool_calls: [{
|
|
129
|
-
id: "call_abc123",
|
|
130
|
-
type: "function",
|
|
131
|
-
function: { name: "calculator", arguments: '{"expression":"2+2"}' }
|
|
132
|
-
}]
|
|
133
|
-
},
|
|
134
|
-
finish_reason: "tool_calls"
|
|
135
|
-
}],
|
|
136
|
-
usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
|
|
137
|
-
}.to_json
|
|
138
|
-
},
|
|
139
|
-
{
|
|
140
|
-
status: 200,
|
|
141
|
-
headers: { "Content-Type" => "application/json" },
|
|
142
|
-
body: {
|
|
143
|
-
id: "chatcmpl-456",
|
|
144
|
-
object: "chat.completion",
|
|
145
|
-
model: "gpt-4o-mini",
|
|
146
|
-
choices: [{
|
|
147
|
-
index: 0,
|
|
148
|
-
message: { role: "assistant", content: "The answer is 4" },
|
|
149
|
-
finish_reason: "stop"
|
|
150
|
-
}],
|
|
151
|
-
usage: { prompt_tokens: 20, completion_tokens: 5, total_tokens: 25 }
|
|
152
|
-
}.to_json
|
|
153
|
-
}
|
|
179
|
+
stub_chat_completion(
|
|
180
|
+
chat_completion_body(
|
|
181
|
+
content: nil,
|
|
182
|
+
tool_calls: [{
|
|
183
|
+
id: "call_abc123",
|
|
184
|
+
type: "function",
|
|
185
|
+
function: { name: "calculator", arguments: '{"expression":"2+2"}' }
|
|
186
|
+
}]
|
|
187
|
+
),
|
|
188
|
+
chat_completion_body(
|
|
189
|
+
content: "The answer is 4",
|
|
190
|
+
usage: { prompt_tokens: 20, completion_tokens: 5, total_tokens: 25 }
|
|
154
191
|
)
|
|
192
|
+
)
|
|
155
193
|
|
|
156
194
|
chat = RubyLLM.chat(model: "gpt-4o-mini")
|
|
157
195
|
chat.with_tool(calculator)
|
|
@@ -168,30 +206,82 @@ class InstrumentationTest < Minitest::Test
|
|
|
168
206
|
tool_span = tool_spans.first
|
|
169
207
|
assert_equal OpenTelemetry::Trace::SpanKind::INTERNAL, tool_span.kind
|
|
170
208
|
assert_equal "execute_tool calculator", tool_span.name
|
|
209
|
+
assert_equal "execute_tool", tool_span.attributes["gen_ai.operation.name"]
|
|
171
210
|
assert_equal "calculator", tool_span.attributes["gen_ai.tool.name"]
|
|
211
|
+
assert_equal "Performs math", tool_span.attributes["gen_ai.tool.description"]
|
|
172
212
|
assert_equal '{"expression":"2+2"}', tool_span.attributes["gen_ai.tool.call.arguments"]
|
|
173
213
|
assert_equal "4", tool_span.attributes["gen_ai.tool.call.result"]
|
|
174
214
|
assert_equal "call_abc123", tool_span.attributes["gen_ai.tool.call.id"]
|
|
175
215
|
assert_equal "function", tool_span.attributes["gen_ai.tool.type"]
|
|
176
216
|
end
|
|
177
217
|
|
|
178
|
-
def
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
218
|
+
def test_truncates_tool_result_to_configured_max_length
|
|
219
|
+
long_value = "x" * 1000
|
|
220
|
+
echo = Class.new(RubyLLM::Tool) do
|
|
221
|
+
def self.name = "echo"
|
|
222
|
+
description "Echoes a long string"
|
|
223
|
+
|
|
224
|
+
define_method(:execute) { long_value }
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
stub_chat_completion(
|
|
228
|
+
chat_completion_body(
|
|
229
|
+
content: nil,
|
|
230
|
+
tool_calls: [{
|
|
231
|
+
id: "call_echo",
|
|
232
|
+
type: "function",
|
|
233
|
+
function: { name: "echo", arguments: "{}" }
|
|
234
|
+
}]
|
|
235
|
+
),
|
|
236
|
+
chat_completion_body(
|
|
237
|
+
content: "done",
|
|
238
|
+
usage: { prompt_tokens: 20, completion_tokens: 5, total_tokens: 25 }
|
|
239
|
+
)
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:tool_result_max_length] = 700
|
|
243
|
+
|
|
244
|
+
chat = RubyLLM.chat(model: "gpt-4o-mini").with_tool(echo)
|
|
245
|
+
chat.ask("echo please")
|
|
246
|
+
|
|
247
|
+
tool_span = EXPORTER.finished_spans.find { |s| s.name.start_with?("execute_tool ") }
|
|
248
|
+
assert_equal "x" * 700, tool_span.attributes["gen_ai.tool.call.result"]
|
|
249
|
+
ensure
|
|
250
|
+
OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:tool_result_max_length] = 500
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
def test_records_error_when_tool_raises
|
|
254
|
+
boom = Class.new(RubyLLM::Tool) do
|
|
255
|
+
def self.name = "boom"
|
|
256
|
+
description "Always raises"
|
|
257
|
+
|
|
258
|
+
def execute
|
|
259
|
+
raise ArgumentError, "tool failure"
|
|
260
|
+
end
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
stub_chat_completion(
|
|
264
|
+
chat_completion_body(
|
|
265
|
+
content: nil,
|
|
266
|
+
tool_calls: [{
|
|
267
|
+
id: "call_x",
|
|
268
|
+
type: "function",
|
|
269
|
+
function: { name: "boom", arguments: "{}" }
|
|
270
|
+
}],
|
|
271
|
+
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }
|
|
194
272
|
)
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
chat = RubyLLM.chat(model: "gpt-4o-mini").with_tool(boom)
|
|
276
|
+
assert_raises(ArgumentError) { chat.ask("trigger") }
|
|
277
|
+
|
|
278
|
+
tool_span = EXPORTER.finished_spans.find { |s| s.name.start_with?("execute_tool ") }
|
|
279
|
+
assert_equal "ArgumentError", tool_span.attributes["error.type"]
|
|
280
|
+
assert_equal OpenTelemetry::Trace::Status::ERROR, tool_span.status.code
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
def test_does_not_capture_content_by_default
|
|
284
|
+
stub_chat_completion
|
|
195
285
|
|
|
196
286
|
chat = RubyLLM.chat(model: "gpt-4o-mini")
|
|
197
287
|
chat.with_instructions("You are helpful")
|
|
@@ -206,22 +296,7 @@ class InstrumentationTest < Minitest::Test
|
|
|
206
296
|
def test_captures_content_when_enabled
|
|
207
297
|
OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:capture_content] = true
|
|
208
298
|
|
|
209
|
-
|
|
210
|
-
.to_return(
|
|
211
|
-
status: 200,
|
|
212
|
-
headers: { "Content-Type" => "application/json" },
|
|
213
|
-
body: {
|
|
214
|
-
id: "chatcmpl-123",
|
|
215
|
-
object: "chat.completion",
|
|
216
|
-
model: "gpt-4o-mini",
|
|
217
|
-
choices: [{
|
|
218
|
-
index: 0,
|
|
219
|
-
message: { role: "assistant", content: "Hello, world!" },
|
|
220
|
-
finish_reason: "stop"
|
|
221
|
-
}],
|
|
222
|
-
usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
|
|
223
|
-
}.to_json
|
|
224
|
-
)
|
|
299
|
+
stub_chat_completion
|
|
225
300
|
|
|
226
301
|
chat = RubyLLM.chat(model: "gpt-4o-mini")
|
|
227
302
|
chat.with_instructions("You are helpful")
|
|
@@ -293,22 +368,7 @@ class InstrumentationTest < Minitest::Test
|
|
|
293
368
|
end
|
|
294
369
|
|
|
295
370
|
def test_with_otel_attributes_sets_span_attributes
|
|
296
|
-
|
|
297
|
-
.to_return(
|
|
298
|
-
status: 200,
|
|
299
|
-
headers: { "Content-Type" => "application/json" },
|
|
300
|
-
body: {
|
|
301
|
-
id: "chatcmpl-123",
|
|
302
|
-
object: "chat.completion",
|
|
303
|
-
model: "gpt-4o-mini",
|
|
304
|
-
choices: [{
|
|
305
|
-
index: 0,
|
|
306
|
-
message: { role: "assistant", content: "Hello!" },
|
|
307
|
-
finish_reason: "stop"
|
|
308
|
-
}],
|
|
309
|
-
usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
|
|
310
|
-
}.to_json
|
|
311
|
-
)
|
|
371
|
+
stub_chat_completion(chat_completion_body(content: "Hello!"))
|
|
312
372
|
|
|
313
373
|
chat = RubyLLM.chat(model: "gpt-4o-mini")
|
|
314
374
|
chat.with_otel_attributes(
|
|
@@ -323,22 +383,7 @@ class InstrumentationTest < Minitest::Test
|
|
|
323
383
|
end
|
|
324
384
|
|
|
325
385
|
def test_with_otel_attributes_returns_self_for_chaining
|
|
326
|
-
|
|
327
|
-
.to_return(
|
|
328
|
-
status: 200,
|
|
329
|
-
headers: { "Content-Type" => "application/json" },
|
|
330
|
-
body: {
|
|
331
|
-
id: "chatcmpl-123",
|
|
332
|
-
object: "chat.completion",
|
|
333
|
-
model: "gpt-4o-mini",
|
|
334
|
-
choices: [{
|
|
335
|
-
index: 0,
|
|
336
|
-
message: { role: "assistant", content: "Hello!" },
|
|
337
|
-
finish_reason: "stop"
|
|
338
|
-
}],
|
|
339
|
-
usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
|
|
340
|
-
}.to_json
|
|
341
|
-
)
|
|
386
|
+
stub_chat_completion(chat_completion_body(content: "Hello!"))
|
|
342
387
|
|
|
343
388
|
chat = RubyLLM.chat(model: "gpt-4o-mini")
|
|
344
389
|
result = chat.with_otel_attributes("custom.category" => "test")
|
|
@@ -347,22 +392,7 @@ class InstrumentationTest < Minitest::Test
|
|
|
347
392
|
end
|
|
348
393
|
|
|
349
394
|
def test_with_otel_attributes_evaluates_callables
|
|
350
|
-
|
|
351
|
-
.to_return(
|
|
352
|
-
status: 200,
|
|
353
|
-
headers: { "Content-Type" => "application/json" },
|
|
354
|
-
body: {
|
|
355
|
-
id: "chatcmpl-123",
|
|
356
|
-
object: "chat.completion",
|
|
357
|
-
model: "gpt-4o-mini",
|
|
358
|
-
choices: [{
|
|
359
|
-
index: 0,
|
|
360
|
-
message: { role: "assistant", content: "Hello!" },
|
|
361
|
-
finish_reason: "stop"
|
|
362
|
-
}],
|
|
363
|
-
usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
|
|
364
|
-
}.to_json
|
|
365
|
-
)
|
|
395
|
+
stub_chat_completion(chat_completion_body(content: "Hello!"))
|
|
366
396
|
|
|
367
397
|
chat = RubyLLM.chat(model: "gpt-4o-mini")
|
|
368
398
|
chat.with_otel_attributes(
|
|
@@ -377,22 +407,7 @@ class InstrumentationTest < Minitest::Test
|
|
|
377
407
|
end
|
|
378
408
|
|
|
379
409
|
def test_works_without_otel_attributes
|
|
380
|
-
|
|
381
|
-
.to_return(
|
|
382
|
-
status: 200,
|
|
383
|
-
headers: { "Content-Type" => "application/json" },
|
|
384
|
-
body: {
|
|
385
|
-
id: "chatcmpl-123",
|
|
386
|
-
object: "chat.completion",
|
|
387
|
-
model: "gpt-4o-mini",
|
|
388
|
-
choices: [{
|
|
389
|
-
index: 0,
|
|
390
|
-
message: { role: "assistant", content: "Hello!" },
|
|
391
|
-
finish_reason: "stop"
|
|
392
|
-
}],
|
|
393
|
-
usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
|
|
394
|
-
}.to_json
|
|
395
|
-
)
|
|
410
|
+
stub_chat_completion(chat_completion_body(content: "Hello!"))
|
|
396
411
|
|
|
397
412
|
chat = RubyLLM.chat(model: "gpt-4o-mini")
|
|
398
413
|
response = chat.ask("Hi")
|
|
@@ -400,25 +415,84 @@ class InstrumentationTest < Minitest::Test
|
|
|
400
415
|
assert_equal "Hello!", response.content
|
|
401
416
|
end
|
|
402
417
|
|
|
418
|
+
def test_captures_ruby_llm_content_with_attachments
|
|
419
|
+
OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:capture_content] = true
|
|
420
|
+
|
|
421
|
+
stub_chat_completion(chat_completion_body(content: "A cat."))
|
|
422
|
+
|
|
423
|
+
content = RubyLLM::Content.new("What is this?", "https://example.com/cat.png")
|
|
424
|
+
|
|
425
|
+
chat = RubyLLM.chat(model: "gpt-4o-mini")
|
|
426
|
+
chat.add_message(role: :user, content: content)
|
|
427
|
+
chat.complete
|
|
428
|
+
|
|
429
|
+
span = EXPORTER.finished_spans.first
|
|
430
|
+
input_messages = JSON.parse(span.attributes["gen_ai.input.messages"])
|
|
431
|
+
assert_equal(
|
|
432
|
+
[
|
|
433
|
+
{ "type" => "text", "content" => "What is this?" },
|
|
434
|
+
{
|
|
435
|
+
"type" => "uri",
|
|
436
|
+
"modality" => "image",
|
|
437
|
+
"mime_type" => "image/png",
|
|
438
|
+
"uri" => "https://example.com/cat.png"
|
|
439
|
+
}
|
|
440
|
+
],
|
|
441
|
+
input_messages[0]["parts"]
|
|
442
|
+
)
|
|
443
|
+
ensure
|
|
444
|
+
OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:capture_content] = false
|
|
445
|
+
end
|
|
446
|
+
|
|
447
|
+
def test_captures_ruby_llm_content_raw
|
|
448
|
+
skip "RubyLLM::Content::Raw not available before ruby_llm 1.9.0" unless defined?(RubyLLM::Content::Raw)
|
|
449
|
+
OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:capture_content] = true
|
|
450
|
+
|
|
451
|
+
stub_chat_completion(chat_completion_body(content: "Acknowledged."))
|
|
452
|
+
|
|
453
|
+
raw = RubyLLM::Content::Raw.new([{ type: "text", text: "raw payload" }])
|
|
454
|
+
|
|
455
|
+
chat = RubyLLM.chat(model: "gpt-4o-mini")
|
|
456
|
+
chat.add_message(role: :user, content: raw)
|
|
457
|
+
chat.complete
|
|
458
|
+
|
|
459
|
+
span = EXPORTER.finished_spans.first
|
|
460
|
+
input_messages = JSON.parse(span.attributes["gen_ai.input.messages"])
|
|
461
|
+
assert_equal(
|
|
462
|
+
[{ "type" => "raw", "content" => [{ "type" => "text", "text" => "raw payload" }].to_json }],
|
|
463
|
+
input_messages[0]["parts"]
|
|
464
|
+
)
|
|
465
|
+
ensure
|
|
466
|
+
OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:capture_content] = false
|
|
467
|
+
end
|
|
468
|
+
|
|
469
|
+
def test_captures_ruby_llm_content_raw_system_instructions
|
|
470
|
+
skip "RubyLLM::Content::Raw not available before ruby_llm 1.9.0" unless defined?(RubyLLM::Content::Raw)
|
|
471
|
+
OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:capture_content] = true
|
|
472
|
+
|
|
473
|
+
stub_chat_completion(chat_completion_body(content: "Acknowledged."))
|
|
474
|
+
|
|
475
|
+
raw_block = RubyLLM::Content::Raw.new([{ type: "text", text: "You are helpful" }])
|
|
476
|
+
|
|
477
|
+
chat = RubyLLM.chat(model: "gpt-4o-mini")
|
|
478
|
+
chat.add_message(role: :system, content: raw_block)
|
|
479
|
+
chat.add_message(role: :user, content: "Hi")
|
|
480
|
+
chat.complete
|
|
481
|
+
|
|
482
|
+
span = EXPORTER.finished_spans.first
|
|
483
|
+
system_instructions = JSON.parse(span.attributes["gen_ai.system_instructions"])
|
|
484
|
+
assert_equal(
|
|
485
|
+
[{ "type" => "raw", "content" => [{ "type" => "text", "text" => "You are helpful" }].to_json }],
|
|
486
|
+
system_instructions
|
|
487
|
+
)
|
|
488
|
+
ensure
|
|
489
|
+
OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:capture_content] = false
|
|
490
|
+
end
|
|
491
|
+
|
|
403
492
|
def test_captures_content_when_enabled_via_env_var
|
|
404
493
|
ENV["OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"] = "true"
|
|
405
494
|
|
|
406
|
-
|
|
407
|
-
.to_return(
|
|
408
|
-
status: 200,
|
|
409
|
-
headers: { "Content-Type" => "application/json" },
|
|
410
|
-
body: {
|
|
411
|
-
id: "chatcmpl-123",
|
|
412
|
-
object: "chat.completion",
|
|
413
|
-
model: "gpt-4o-mini",
|
|
414
|
-
choices: [{
|
|
415
|
-
index: 0,
|
|
416
|
-
message: { role: "assistant", content: "Hello, world!" },
|
|
417
|
-
finish_reason: "stop"
|
|
418
|
-
}],
|
|
419
|
-
usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
|
|
420
|
-
}.to_json
|
|
421
|
-
)
|
|
495
|
+
stub_chat_completion
|
|
422
496
|
|
|
423
497
|
chat = RubyLLM.chat(model: "gpt-4o-mini")
|
|
424
498
|
chat.ask("Hi")
|
data/test/test_helper.rb
CHANGED
|
@@ -6,6 +6,36 @@ require "ruby_llm"
|
|
|
6
6
|
require "opentelemetry/sdk"
|
|
7
7
|
require "opentelemetry-instrumentation-ruby_llm"
|
|
8
8
|
|
|
9
|
+
module ChatCompletionStubs
|
|
10
|
+
DEFAULT_USAGE = { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }.freeze
|
|
11
|
+
|
|
12
|
+
def chat_completion_body(content: "Hello, world!", model: "gpt-4o-mini", tool_calls: nil, usage: DEFAULT_USAGE)
|
|
13
|
+
message = { role: "assistant", content: content }
|
|
14
|
+
message[:tool_calls] = tool_calls if tool_calls
|
|
15
|
+
|
|
16
|
+
{
|
|
17
|
+
id: "chatcmpl-123",
|
|
18
|
+
object: "chat.completion",
|
|
19
|
+
model: model,
|
|
20
|
+
choices: [{
|
|
21
|
+
index: 0,
|
|
22
|
+
message: message,
|
|
23
|
+
finish_reason: tool_calls ? "tool_calls" : "stop"
|
|
24
|
+
}],
|
|
25
|
+
usage: usage
|
|
26
|
+
}.to_json
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def stub_chat_completion(*bodies)
|
|
30
|
+
bodies = [chat_completion_body] if bodies.empty?
|
|
31
|
+
responses = bodies.map do |body|
|
|
32
|
+
{ status: 200, headers: { "Content-Type" => "application/json" }, body: body }
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
stub_request(:post, "https://api.openai.com/v1/chat/completions").to_return(*responses)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
9
39
|
EXPORTER = OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter.new
|
|
10
40
|
span_processor = OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor.new(EXPORTER)
|
|
11
41
|
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: opentelemetry-instrumentation-ruby_llm
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.7.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Clarissa Borges
|
|
@@ -48,6 +48,7 @@ files:
|
|
|
48
48
|
- ".github/workflows/dynamic-security.yml"
|
|
49
49
|
- ".github/workflows/main.yml"
|
|
50
50
|
- ".gitignore"
|
|
51
|
+
- Appraisals
|
|
51
52
|
- CODEOWNERS
|
|
52
53
|
- CODE_OF_CONDUCT.md
|
|
53
54
|
- Gemfile
|
|
@@ -60,12 +61,20 @@ files:
|
|
|
60
61
|
- example/trace_demonstration_with_langfuse_and_tools.rb
|
|
61
62
|
- example/trace_demonstration_with_langfuse_ingredient_search.rb
|
|
62
63
|
- example/trace_demonstration_with_tools.rb
|
|
64
|
+
- gemfiles/ruby_llm_1.12.1.gemfile
|
|
65
|
+
- gemfiles/ruby_llm_1.8.0.gemfile
|
|
66
|
+
- gemfiles/ruby_llm_1_latest.gemfile
|
|
63
67
|
- lib/opentelemetry-instrumentation-ruby_llm.rb
|
|
64
68
|
- lib/opentelemetry/instrumentation/ruby_llm/instrumentation.rb
|
|
69
|
+
- lib/opentelemetry/instrumentation/ruby_llm/message_formatter.rb
|
|
70
|
+
- lib/opentelemetry/instrumentation/ruby_llm/patches/agent.rb
|
|
65
71
|
- lib/opentelemetry/instrumentation/ruby_llm/patches/chat.rb
|
|
72
|
+
- lib/opentelemetry/instrumentation/ruby_llm/patches/chat_methods.rb
|
|
66
73
|
- lib/opentelemetry/instrumentation/ruby_llm/patches/embedding.rb
|
|
67
74
|
- lib/opentelemetry/instrumentation/ruby_llm/version.rb
|
|
68
75
|
- opentelemetry-instrumentation-ruby_llm.gemspec
|
|
76
|
+
- test/active_record_chat_methods_test.rb
|
|
77
|
+
- test/agent_instrumentation_test.rb
|
|
69
78
|
- test/instrumentation_test.rb
|
|
70
79
|
- test/test_helper.rb
|
|
71
80
|
homepage: https://github.com/thoughtbot/opentelemetry-instrumentation-ruby_llm
|