ask-instrumentation 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/README.md +95 -211
- data/lib/ask/instrumentation/runtime_adapter.rb +168 -0
- data/lib/ask/instrumentation/tool.rb +1 -0
- data/lib/ask/instrumentation/version.rb +1 -1
- data/lib/ask/instrumentation.rb +28 -0
- metadata +17 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: fb15ee2f3cd35d3b74a3fbcaf2fa8bfa57e50dae2d23276909874a9a94cd2c2b
|
|
4
|
+
data.tar.gz: c0fcdb8b98b86ff4aef64a90afdd87e35e3ae064fcfcdb02aac34018c299eaf4
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b84962ccb2b800356b53c0babba22866f5347f31cb24d0d3417c2e46401d08c5f957a8c4ecb4cc0d20277c527f4fa682d05afc07f84bd4bc451399588da5707d
|
|
7
|
+
data.tar.gz: 136863ebb3cd6109ad2deeeadd6e2ac311e6b08e65d02d1f26cf48d80a34430bbbdf814bde99459e1b31e3ffc050dbac8f54c5efdfb3b568ce5a865122c95b3f
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
## [0.2.1] - 2026-06-25
|
|
2
2
|
|
|
3
|
+
### Added
|
|
4
|
+
- `RuntimeAdapter` — bridges ask-runtime EventSink events to Ask::Instrumentation events with stable names (`tool.started.ask`, `tool.completed.ask`, `tool.failed.ask`, `tool.cancelled.ask`, `tool.timed_out.ask`) and uniform payload schema (tool_call_id, tool_name, session_id, turn, duration, outcome, error, event).
|
|
5
|
+
- `Ask::Instrumentation.install_runtime_sink` — one-liner helper to create and return an adapter sink.
|
|
6
|
+
- Opt-in, no-op safe, thread-safe. No hard dependency from ask-runtime to ask-instrumentation.
|
|
7
|
+
|
|
3
8
|
### Changed
|
|
4
9
|
- Submodule tests: Chat(3t), Embedding(2t), Tool(7t). Infrastructure: rubocop, overcommit, CI matrix, gemspec, SimpleCov.
|
|
5
10
|
# Changelog
|
data/README.md
CHANGED
|
@@ -3,11 +3,10 @@
|
|
|
3
3
|
[](https://badge.fury.io/rb/ask-instrumentation)
|
|
4
4
|
[](https://github.com/ask-rb/ask-instrumentation/actions/workflows/ci.yml)
|
|
5
5
|
|
|
6
|
-
LLM observability for the ask-rb ecosystem.
|
|
7
|
-
events for chat completions, embeddings, tool calls, and image
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
for cost tracking, logging, analytics, or alerting.
|
|
6
|
+
LLM observability for the ask-rb ecosystem. Wraps `ActiveSupport::Notifications`
|
|
7
|
+
and emits events for chat completions, embeddings, tool calls, and image
|
|
8
|
+
generation. Works with any LLM provider: subscribe to events for cost
|
|
9
|
+
tracking, logging, analytics, or alerting.
|
|
11
10
|
|
|
12
11
|
## Installation
|
|
13
12
|
|
|
@@ -15,12 +14,6 @@ for cost tracking, logging, analytics, or alerting.
|
|
|
15
14
|
gem "ask-instrumentation"
|
|
16
15
|
```
|
|
17
16
|
|
|
18
|
-
Or install it yourself:
|
|
19
|
-
|
|
20
|
-
```bash
|
|
21
|
-
gem install ask-instrumentation
|
|
22
|
-
```
|
|
23
|
-
|
|
24
17
|
## Quick Start
|
|
25
18
|
|
|
26
19
|
```ruby
|
|
@@ -31,14 +24,14 @@ Ask::Instrumentation.subscribe do |event|
|
|
|
31
24
|
puts "#{event.name}: #{event.duration}ms"
|
|
32
25
|
end
|
|
33
26
|
|
|
34
|
-
# Instrument a
|
|
27
|
+
# Instrument a block of work
|
|
35
28
|
Ask::Instrumentation.instrument("chat.ask", provider: "openai", model: "gpt-4") do
|
|
36
29
|
# your LLM call here
|
|
37
30
|
end
|
|
38
31
|
|
|
39
|
-
#
|
|
32
|
+
# Attach context to every event emitted inside the block
|
|
40
33
|
Ask::Instrumentation.with_metadata(user_id: 42, session_id: "abc") do
|
|
41
|
-
Ask::Instrumentation.instrument("chat.ask",
|
|
34
|
+
Ask::Instrumentation.instrument("chat.ask", provider: "openai", model: "gpt-4") do
|
|
42
35
|
# ...
|
|
43
36
|
end
|
|
44
37
|
end
|
|
@@ -46,231 +39,122 @@ end
|
|
|
46
39
|
|
|
47
40
|
## Events
|
|
48
41
|
|
|
49
|
-
| Event |
|
|
50
|
-
|
|
51
|
-
| `chat.ask` |
|
|
52
|
-
| `chat.stream.ask` |
|
|
53
|
-
| `tool.ask` |
|
|
54
|
-
| `embedding.ask` |
|
|
55
|
-
| `image.ask` |
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
42
|
+
| Event | Description |
|
|
43
|
+
|---|---|
|
|
44
|
+
| `chat.ask` | Chat completion |
|
|
45
|
+
| `chat.stream.ask` | Streaming chat |
|
|
46
|
+
| `tool.ask` | Tool execution |
|
|
47
|
+
| `embedding.ask` | Embedding generation |
|
|
48
|
+
| `image.ask` | Image generation |
|
|
49
|
+
| `tool_call.ask` | LLM requested a tool call |
|
|
50
|
+
| `tool_result.ask` | Tool call result (or error) |
|
|
51
|
+
| `tool.started.ask` | Runtime tool execution started |
|
|
52
|
+
| `tool.completed.ask` | Runtime tool execution completed |
|
|
53
|
+
| `tool.failed.ask` | Runtime tool execution failed |
|
|
54
|
+
| `tool.cancelled.ask` | Runtime tool execution cancelled |
|
|
55
|
+
| `tool.timed_out.ask` | Runtime tool execution timed out |
|
|
56
|
+
|
|
57
|
+
Event payloads carry provider, model, and token counts when available, plus
|
|
58
|
+
any metadata set with `with_metadata`.
|
|
59
|
+
|
|
60
|
+
## Essential API
|
|
61
|
+
|
|
62
|
+
| Entry point | Purpose |
|
|
63
|
+
|---|---|
|
|
64
|
+
| `Ask::Instrumentation.subscribe(pattern = /\.ask$/, &block)` | Subscribe to matching events; returns a subscriber for `unsubscribe` |
|
|
65
|
+
| `Ask::Instrumentation.unsubscribe(subscriber_or_pattern)` | Remove a subscriber |
|
|
66
|
+
| `Ask::Instrumentation.instrument(name, payload = {}) { }` | Emit an event, timing the block and passing through its return value |
|
|
67
|
+
| `Ask::Instrumentation.with_metadata(hash) { }` | Thread-local metadata merged into all events emitted inside the block |
|
|
68
|
+
| `Ask::Instrumentation.current_metadata` | Current thread's metadata hash |
|
|
69
|
+
|
|
70
|
+
Metadata lives in `Thread.current`, so concurrent threads each carry their own
|
|
71
|
+
context. Nested `with_metadata` calls merge, with inner values taking
|
|
72
|
+
precedence.
|
|
73
|
+
|
|
74
|
+
## Runtime Adapter
|
|
75
|
+
|
|
76
|
+
The `RuntimeAdapter` bridges [ask-runtime](https://github.com/ask-rb/ask-runtime)
|
|
77
|
+
tool lifecycle events to Ask::Instrumentation events. This lets you observe
|
|
78
|
+
tool execution through the same instrumentation pipeline as LLM calls.
|
|
62
79
|
|
|
63
80
|
```ruby
|
|
64
81
|
require "ask/instrumentation"
|
|
82
|
+
require "ask/instrumentation/runtime_adapter"
|
|
65
83
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
"gpt-3.5-turbo" => { input: 0.001 / 1000, output: 0.002 / 1000 },
|
|
69
|
-
"claude-3-opus" => { input: 0.015 / 1000, output: 0.075 / 1000 }
|
|
70
|
-
}.freeze
|
|
71
|
-
|
|
72
|
-
total_cost = 0.0
|
|
73
|
-
|
|
74
|
-
Ask::Instrumentation.subscribe(/chat\.ask/) do |event|
|
|
75
|
-
payload = event.payload
|
|
76
|
-
model = payload[:model]
|
|
77
|
-
pricing = COST_PER_TOKEN[model]
|
|
78
|
-
next unless pricing
|
|
79
|
-
|
|
80
|
-
cost = (payload[:input_tokens].to_i * pricing[:input]) +
|
|
81
|
-
(payload[:output_tokens].to_i * pricing[:output])
|
|
82
|
-
total_cost += cost
|
|
83
|
-
|
|
84
|
-
puts "[COST] #{model}: $%.6f (total: $%.4f)" % [cost, total_cost]
|
|
85
|
-
end
|
|
86
|
-
|
|
87
|
-
# Later, in your application code:
|
|
88
|
-
Ask::Instrumentation.with_metadata(session_id: "sess_123") do
|
|
89
|
-
Ask::Instrumentation.instrument("chat.ask",
|
|
90
|
-
provider: "openai",
|
|
91
|
-
model: "gpt-4",
|
|
92
|
-
input_tokens: 150,
|
|
93
|
-
output_tokens: 50
|
|
94
|
-
) do
|
|
95
|
-
# actual LLM API call
|
|
96
|
-
end
|
|
97
|
-
end
|
|
98
|
-
```
|
|
99
|
-
|
|
100
|
-
### Request Logging
|
|
84
|
+
# One-liner: creates a sink that forwards events to Ask::Instrumentation
|
|
85
|
+
sink = Ask::Instrumentation.install_runtime_sink
|
|
101
86
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
require "logger"
|
|
107
|
-
|
|
108
|
-
logger = Logger.new("log/llm.log")
|
|
87
|
+
# Wire into an execution context
|
|
88
|
+
ctx = Ask::Runtime::ExecutionContext.new(
|
|
89
|
+
session_id: "s_001", turn: 1, event_sink: sink
|
|
90
|
+
)
|
|
109
91
|
|
|
92
|
+
# Subscribe to the forwarded events
|
|
110
93
|
Ask::Instrumentation.subscribe do |event|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
### Usage Analytics with Metadata
|
|
120
|
-
|
|
121
|
-
Track per-user and per-session usage:
|
|
122
|
-
|
|
123
|
-
```ruby
|
|
124
|
-
require "ask/instrumentation"
|
|
125
|
-
|
|
126
|
-
# In your application (e.g., a Rails controller):
|
|
127
|
-
class ChatController < ApplicationController
|
|
128
|
-
def create
|
|
129
|
-
Ask::Instrumentation.with_metadata(
|
|
130
|
-
user_id: current_user.id,
|
|
131
|
-
session_id: request.session.id,
|
|
132
|
-
request_id: request.request_id
|
|
133
|
-
) do
|
|
134
|
-
# All events emitted here automatically include user/session context
|
|
135
|
-
response = llm_client.chat(params[:message])
|
|
136
|
-
render json: response
|
|
137
|
-
end
|
|
94
|
+
case event.name
|
|
95
|
+
when "tool.started.ask"
|
|
96
|
+
puts "Tool started: #{event.payload[:tool_name]}"
|
|
97
|
+
when "tool.completed.ask"
|
|
98
|
+
puts "Tool #{event.payload[:tool_name]} completed in #{event.payload[:duration]}s"
|
|
99
|
+
when "tool.failed.ask"
|
|
100
|
+
puts "Tool #{event.payload[:tool_name]} failed: #{event.payload[:error]}"
|
|
138
101
|
end
|
|
139
102
|
end
|
|
140
|
-
|
|
141
|
-
# In a monitoring background worker:
|
|
142
|
-
Ask::Instrumentation.subscribe(/chat\.ask/) do |event|
|
|
143
|
-
payload = event.payload
|
|
144
|
-
UsageReport.increment(
|
|
145
|
-
user_id: payload[:user_id],
|
|
146
|
-
model: payload[:model],
|
|
147
|
-
tokens_in: payload[:input_tokens],
|
|
148
|
-
tokens_out: payload[:output_tokens]
|
|
149
|
-
)
|
|
150
|
-
end
|
|
151
103
|
```
|
|
152
104
|
|
|
153
|
-
###
|
|
105
|
+
### Event Mapping
|
|
154
106
|
|
|
155
|
-
|
|
107
|
+
| Runtime EventSink type | Ask::Instrumentation event |
|
|
108
|
+
|---|---|
|
|
109
|
+
| `:tool_started` | `tool.started.ask` |
|
|
110
|
+
| `:tool_completed` | `tool.completed.ask` |
|
|
111
|
+
| `:tool_failed` | `tool.failed.ask` |
|
|
112
|
+
| `:tool_cancelled` | `tool.cancelled.ask` |
|
|
113
|
+
| `:tool_timed_out` | `tool.timed_out.ask` |
|
|
156
114
|
|
|
157
|
-
|
|
158
|
-
require "ask/provider"
|
|
115
|
+
### Payload Schema
|
|
159
116
|
|
|
160
|
-
|
|
161
|
-
provider = Ask::Provider.for(:openai)
|
|
162
|
-
provider.complete(prompt: "Hello!") # emits chat.ask
|
|
163
|
-
|
|
164
|
-
# Subscribe to track everything:
|
|
165
|
-
Ask::Instrumentation.subscribe do |event|
|
|
166
|
-
puts "[#{event.name}] #{event.payload[:model]} (#{event.duration}ms)"
|
|
167
|
-
end
|
|
168
|
-
```
|
|
117
|
+
Every forwarded event includes:
|
|
169
118
|
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
input_tokens: prompt.length / 4
|
|
181
|
-
) do
|
|
182
|
-
response = call_api(prompt)
|
|
183
|
-
# enrich payload after the call by returning a hash from the block?
|
|
184
|
-
# No — use with_metadata or include everything upfront.
|
|
185
|
-
response
|
|
186
|
-
end
|
|
187
|
-
end
|
|
188
|
-
end
|
|
189
|
-
```
|
|
190
|
-
|
|
191
|
-
## API
|
|
192
|
-
|
|
193
|
-
### `.subscribe(pattern = /\.ask$/, &block)`
|
|
194
|
-
|
|
195
|
-
Subscribe to ask events. Accepts an optional pattern (defaults to all `.ask` events).
|
|
196
|
-
|
|
197
|
-
```ruby
|
|
198
|
-
# All ask events
|
|
199
|
-
Ask::Instrumentation.subscribe { |event| ... }
|
|
200
|
-
|
|
201
|
-
# Only chat events
|
|
202
|
-
Ask::Instrumentation.subscribe(/chat\.ask/) { |event| ... }
|
|
203
|
-
```
|
|
204
|
-
|
|
205
|
-
### `.unsubscribe(subscriber_or_name)`
|
|
206
|
-
|
|
207
|
-
Remove a subscriber by passing the object returned from `subscribe` or a string/regexp.
|
|
208
|
-
|
|
209
|
-
```ruby
|
|
210
|
-
subscriber = Ask::Instrumentation.subscribe { |e| ... }
|
|
211
|
-
Ask::Instrumentation.unsubscribe(subscriber)
|
|
212
|
-
```
|
|
213
|
-
|
|
214
|
-
### `.instrument(name, payload = {}, &block)`
|
|
215
|
-
|
|
216
|
-
Emit an event. The block is instrumented and its return value is passed through.
|
|
217
|
-
Metadata from `with_metadata` is automatically merged into the payload.
|
|
218
|
-
|
|
219
|
-
```ruby
|
|
220
|
-
result = Ask::Instrumentation.instrument("chat.ask",
|
|
221
|
-
provider: "openai",
|
|
222
|
-
model: "gpt-4"
|
|
223
|
-
) do
|
|
224
|
-
llm_call
|
|
225
|
-
end
|
|
226
|
-
```
|
|
227
|
-
|
|
228
|
-
### `.with_metadata(hash, &block)`
|
|
119
|
+
| Key | Type | Description |
|
|
120
|
+
|---|---|---|
|
|
121
|
+
| `tool_call_id` | String | Unique tool-call identifier |
|
|
122
|
+
| `tool_name` | String | The tool being executed |
|
|
123
|
+
| `session_id` | String, nil | Session correlation |
|
|
124
|
+
| `turn` | Integer, nil | Turn number |
|
|
125
|
+
| `duration` | Float, nil | Seconds (nil for `tool.started.ask`) |
|
|
126
|
+
| `outcome` | String, nil | `"success"`, `"failure"`, `"cancelled"`, `"timed_out"`, or nil |
|
|
127
|
+
| `error` | String, nil | Error message (failures and cancellations) |
|
|
128
|
+
| `event` | Object | The original runtime event object |
|
|
229
129
|
|
|
230
|
-
|
|
231
|
-
Nested blocks merge inner metadata into outer, with inner values taking precedence.
|
|
130
|
+
### Lifecycle
|
|
232
131
|
|
|
233
132
|
```ruby
|
|
234
|
-
Ask::Instrumentation.
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
133
|
+
adapter = Ask::Instrumentation::RuntimeAdapter.new
|
|
134
|
+
sink = adapter.sink # use this as your EventSink
|
|
135
|
+
adapter.subscribed? # => true
|
|
136
|
+
adapter.unsubscribe # stop forwarding events
|
|
137
|
+
adapter.subscribed? # => false
|
|
239
138
|
```
|
|
240
139
|
|
|
241
|
-
###
|
|
140
|
+
### Dependencies
|
|
242
141
|
|
|
243
|
-
|
|
142
|
+
The adapter requires `ask-runtime` at runtime. If `ask-runtime` is not
|
|
143
|
+
available, `install_runtime_sink` returns `Ask::Runtime::EventSink.null`
|
|
144
|
+
(a no-op sink). The adapter is fully opt-in and creates no hard dependency
|
|
145
|
+
from ask-runtime to ask-instrumentation.
|
|
244
146
|
|
|
245
|
-
|
|
246
|
-
Ask::Instrumentation.current_metadata # => {}
|
|
247
|
-
```
|
|
147
|
+
## Full documentation
|
|
248
148
|
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
```ruby
|
|
255
|
-
Ask::Instrumentation.with_metadata(thread: "main") do
|
|
256
|
-
Thread.new do
|
|
257
|
-
# This thread has its own metadata context
|
|
258
|
-
Ask::Instrumentation.current_metadata # => {}
|
|
259
|
-
Ask::Instrumentation.with_metadata(thread: "worker") do
|
|
260
|
-
# ...
|
|
261
|
-
end
|
|
262
|
-
end.join
|
|
263
|
-
end
|
|
264
|
-
```
|
|
149
|
+
The full ask-rb documentation lives at https://ask-rb.github.io/ask-docs.
|
|
150
|
+
https://ask-rb.github.io/ask-docs/production/observability covers
|
|
151
|
+
ask-instrumentation in depth. API reference:
|
|
152
|
+
https://ask-rb.github.io/ask-docs/reference/api.
|
|
265
153
|
|
|
266
154
|
## Development
|
|
267
155
|
|
|
268
|
-
```bash
|
|
269
|
-
git clone https://github.com/ask-rb/ask-instrumentation.git
|
|
270
|
-
cd ask-instrumentation
|
|
271
156
|
bundle install
|
|
272
157
|
bundle exec rake test
|
|
273
|
-
```
|
|
274
158
|
|
|
275
159
|
## License
|
|
276
160
|
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ask/instrumentation"
|
|
4
|
+
|
|
5
|
+
module Ask
|
|
6
|
+
module Instrumentation
|
|
7
|
+
# Subscribes to an Ask::Runtime::EventSink and re-emits events through
|
|
8
|
+
# Ask::Instrumentation (ActiveSupport::Notifications) with stable names
|
|
9
|
+
# and a uniform payload schema.
|
|
10
|
+
#
|
|
11
|
+
# This is an opt-in adapter: it creates an EventSink that proxies
|
|
12
|
+
# runtime lifecycle events to the instrumentation layer so that
|
|
13
|
+
# existing subscribers (logging, analytics, cost tracking) receive
|
|
14
|
+
# tool execution events without coupling to ask-runtime internals.
|
|
15
|
+
#
|
|
16
|
+
# == Event Mapping
|
|
17
|
+
#
|
|
18
|
+
# Runtime EventSink type → Ask::Instrumentation event name
|
|
19
|
+
# ───────────────────────── ──────────────────────────────
|
|
20
|
+
# :tool_started → "tool.started.ask"
|
|
21
|
+
# :tool_completed → "tool.completed.ask"
|
|
22
|
+
# :tool_failed → "tool.failed.ask"
|
|
23
|
+
# :tool_cancelled → "tool.cancelled.ask"
|
|
24
|
+
# :tool_timed_out → "tool.timed_out.ask"
|
|
25
|
+
#
|
|
26
|
+
# == Payload Schema
|
|
27
|
+
#
|
|
28
|
+
# Every emitted event includes:
|
|
29
|
+
#
|
|
30
|
+
# tool_call_id [String] — unique tool-call identifier
|
|
31
|
+
# tool_name [String] — the tool being executed
|
|
32
|
+
# session_id [String, nil] — session correlation
|
|
33
|
+
# turn [Integer, nil] — turn number
|
|
34
|
+
# duration [Float, nil] — seconds (nil for ToolStarted)
|
|
35
|
+
# outcome [String] — "success" | "failure" | "cancelled" | "timed_out" | nil (started)
|
|
36
|
+
# error [String, nil] — error message (terminal failures only)
|
|
37
|
+
# event [Object] — the original runtime event object
|
|
38
|
+
#
|
|
39
|
+
# == Usage
|
|
40
|
+
#
|
|
41
|
+
# require "ask/instrumentation"
|
|
42
|
+
# require "ask/instrumentation/runtime_adapter"
|
|
43
|
+
#
|
|
44
|
+
# # One-liner: creates and returns the adapter sink
|
|
45
|
+
# sink = Ask::Instrumentation.install_runtime_sink
|
|
46
|
+
#
|
|
47
|
+
# # Or build explicitly
|
|
48
|
+
# adapter = Ask::Instrumentation::RuntimeAdapter.new
|
|
49
|
+
# sink = adapter.sink
|
|
50
|
+
#
|
|
51
|
+
# # Wire into an execution context
|
|
52
|
+
# ctx = Ask::Runtime::ExecutionContext.new(
|
|
53
|
+
# session_id: "s_001", turn: 1, event_sink: sink
|
|
54
|
+
# )
|
|
55
|
+
#
|
|
56
|
+
# # Later, unsubscribe to stop forwarding
|
|
57
|
+
# adapter.unsubscribe
|
|
58
|
+
#
|
|
59
|
+
# == Thread Safety
|
|
60
|
+
#
|
|
61
|
+
# The adapter is thread-safe. The underlying EventSink uses a mutex
|
|
62
|
+
# for listener management, and Ask::Instrumentation delegates to
|
|
63
|
+
# ActiveSupport::Notifications which is also thread-safe.
|
|
64
|
+
#
|
|
65
|
+
# == No-Op Safe
|
|
66
|
+
#
|
|
67
|
+
# If ask-runtime is not loaded, install_runtime_sink returns an
|
|
68
|
+
# Ask::Runtime::EventSink.null that discards all events.
|
|
69
|
+
#
|
|
70
|
+
class RuntimeAdapter
|
|
71
|
+
RUNTIME_EVENT_MAP = {
|
|
72
|
+
tool_started: "tool.started.ask",
|
|
73
|
+
tool_completed: "tool.completed.ask",
|
|
74
|
+
tool_failed: "tool.failed.ask",
|
|
75
|
+
tool_cancelled: "tool.cancelled.ask",
|
|
76
|
+
tool_timed_out: "tool.timed_out.ask"
|
|
77
|
+
}.freeze
|
|
78
|
+
|
|
79
|
+
attr_reader :sink
|
|
80
|
+
|
|
81
|
+
def initialize
|
|
82
|
+
require "ask/runtime"
|
|
83
|
+
@sink = Ask::Runtime::EventSink.new
|
|
84
|
+
@forwarding = true
|
|
85
|
+
@mutex = Mutex.new
|
|
86
|
+
subscribe_all
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Stop forwarding events to Ask::Instrumentation. The sink continues
|
|
90
|
+
# to exist but events are silently dropped.
|
|
91
|
+
def unsubscribe
|
|
92
|
+
@mutex.synchronize { @forwarding = false }
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Whether the adapter is currently forwarding events.
|
|
96
|
+
def subscribed?
|
|
97
|
+
@mutex.synchronize { @forwarding }
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
private
|
|
101
|
+
|
|
102
|
+
def subscribe_all
|
|
103
|
+
RUNTIME_EVENT_MAP.each do |runtime_type, instrumentation_name|
|
|
104
|
+
@sink.on(runtime_type) do |payload|
|
|
105
|
+
next unless @mutex.synchronize { @forwarding }
|
|
106
|
+
|
|
107
|
+
forward_event(instrumentation_name, payload)
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def forward_event(name, payload)
|
|
113
|
+
runtime_event = payload[:event]
|
|
114
|
+
return unless runtime_event
|
|
115
|
+
|
|
116
|
+
instrument_payload = build_payload(runtime_event)
|
|
117
|
+
Ask::Instrumentation.instrument(name, instrument_payload)
|
|
118
|
+
rescue => e
|
|
119
|
+
# Never let adapter errors break the runtime event pipeline.
|
|
120
|
+
$stderr.puts "[ask-instrumentation] RuntimeAdapter error: #{e.message}" if $DEBUG
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def build_payload(runtime_event)
|
|
124
|
+
base = {
|
|
125
|
+
tool_call_id: runtime_event.tool_call_id,
|
|
126
|
+
tool_name: runtime_event.tool_name,
|
|
127
|
+
session_id: runtime_event.execution_context&.session_id,
|
|
128
|
+
turn: runtime_event.execution_context&.turn,
|
|
129
|
+
event: runtime_event
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
# Add duration for terminal events (all except ToolStarted).
|
|
133
|
+
if runtime_event.respond_to?(:duration)
|
|
134
|
+
base[:duration] = runtime_event.duration
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Classify outcome.
|
|
138
|
+
base[:outcome] = classify_outcome(runtime_event)
|
|
139
|
+
|
|
140
|
+
# Attach error for failures.
|
|
141
|
+
if runtime_event.respond_to?(:error)
|
|
142
|
+
base[:error] = runtime_event.error
|
|
143
|
+
elsif runtime_event.respond_to?(:reason)
|
|
144
|
+
base[:error] = runtime_event.reason
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
base
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def classify_outcome(runtime_event)
|
|
151
|
+
case runtime_event
|
|
152
|
+
when Ask::Runtime::Events::ToolStarted
|
|
153
|
+
nil
|
|
154
|
+
when Ask::Runtime::Events::ToolCompleted
|
|
155
|
+
"success"
|
|
156
|
+
when Ask::Runtime::Events::ToolFailed
|
|
157
|
+
"failure"
|
|
158
|
+
when Ask::Runtime::Events::ToolCancelled
|
|
159
|
+
"cancelled"
|
|
160
|
+
when Ask::Runtime::Events::ToolTimedOut
|
|
161
|
+
"timed_out"
|
|
162
|
+
else
|
|
163
|
+
nil
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
end
|
data/lib/ask/instrumentation.rb
CHANGED
|
@@ -48,6 +48,7 @@ module Ask
|
|
|
48
48
|
autoload :Chat, "ask/instrumentation/chat"
|
|
49
49
|
autoload :Embedding, "ask/instrumentation/embedding"
|
|
50
50
|
autoload :Tool, "ask/instrumentation/tool"
|
|
51
|
+
autoload :RuntimeAdapter, "ask/instrumentation/runtime_adapter"
|
|
51
52
|
|
|
52
53
|
class << self
|
|
53
54
|
# Subscribe to ask instrumentation events.
|
|
@@ -158,6 +159,33 @@ module Ask
|
|
|
158
159
|
def current_metadata
|
|
159
160
|
Thread.current[:ask_instrumentation_metadata] || {}
|
|
160
161
|
end
|
|
162
|
+
|
|
163
|
+
# Create and return a RuntimeAdapter that bridges ask-runtime
|
|
164
|
+
# EventSink events to Ask::Instrumentation events.
|
|
165
|
+
#
|
|
166
|
+
# The returned sink is an Ask::Runtime::EventSink that should be
|
|
167
|
+
# passed as the +event_sink:+ parameter to an ExecutionContext.
|
|
168
|
+
# Runtime tool lifecycle events emitted through the sink will be
|
|
169
|
+
# re-emitted as Ask::Instrumentation events with stable names:
|
|
170
|
+
#
|
|
171
|
+
# tool.started.ask, tool.completed.ask, tool.failed.ask,
|
|
172
|
+
# tool.cancelled.ask, tool.timed_out.ask
|
|
173
|
+
#
|
|
174
|
+
# If ask-runtime is not available, returns Ask::Runtime::EventSink.null.
|
|
175
|
+
#
|
|
176
|
+
# @return [Ask::Runtime::EventSink, Ask::Runtime::EventSink::NullSink]
|
|
177
|
+
#
|
|
178
|
+
# @example
|
|
179
|
+
# sink = Ask::Instrumentation.install_runtime_sink
|
|
180
|
+
# ctx = Ask::Runtime::ExecutionContext.new(event_sink: sink)
|
|
181
|
+
#
|
|
182
|
+
def install_runtime_sink
|
|
183
|
+
require "ask/instrumentation/runtime_adapter"
|
|
184
|
+
RuntimeAdapter.new.sink
|
|
185
|
+
rescue LoadError
|
|
186
|
+
require "ask/runtime"
|
|
187
|
+
Ask::Runtime::EventSink.null
|
|
188
|
+
end
|
|
161
189
|
end
|
|
162
190
|
end
|
|
163
191
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: ask-instrumentation
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.
|
|
4
|
+
version: 0.2.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Kaka Ruto
|
|
@@ -23,6 +23,20 @@ dependencies:
|
|
|
23
23
|
- - ">="
|
|
24
24
|
- !ruby/object:Gem::Version
|
|
25
25
|
version: '7.0'
|
|
26
|
+
- !ruby/object:Gem::Dependency
|
|
27
|
+
name: ask-runtime
|
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
|
29
|
+
requirements:
|
|
30
|
+
- - ">="
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: 0.1.0
|
|
33
|
+
type: :runtime
|
|
34
|
+
prerelease: false
|
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - ">="
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: 0.1.0
|
|
26
40
|
- !ruby/object:Gem::Dependency
|
|
27
41
|
name: minitest
|
|
28
42
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -81,6 +95,7 @@ files:
|
|
|
81
95
|
- lib/ask/instrumentation.rb
|
|
82
96
|
- lib/ask/instrumentation/chat.rb
|
|
83
97
|
- lib/ask/instrumentation/embedding.rb
|
|
98
|
+
- lib/ask/instrumentation/runtime_adapter.rb
|
|
84
99
|
- lib/ask/instrumentation/tool.rb
|
|
85
100
|
- lib/ask/instrumentation/version.rb
|
|
86
101
|
homepage: https://github.com/ask-rb/ask-instrumentation
|
|
@@ -104,7 +119,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
104
119
|
- !ruby/object:Gem::Version
|
|
105
120
|
version: '0'
|
|
106
121
|
requirements: []
|
|
107
|
-
rubygems_version: 4.0.
|
|
122
|
+
rubygems_version: 4.0.18
|
|
108
123
|
specification_version: 4
|
|
109
124
|
summary: LLM observability for the ask-rb ecosystem
|
|
110
125
|
test_files: []
|