riffer 0.35.0 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.release-please-manifest.json +1 -1
- data/CHANGELOG.md +18 -0
- data/README.md +0 -1
- data/docs/01_OVERVIEW.md +0 -1
- data/docs/03_AGENTS.md +3 -7
- data/docs/10_CONFIGURATION.md +0 -17
- data/docs/providers/07_CUSTOM_PROVIDERS.md +4 -19
- data/lib/riffer/agent/run.rb +11 -35
- data/lib/riffer/agent.rb +1 -1
- data/lib/riffer/config.rb +0 -45
- data/lib/riffer/guardrails/runner.rb +7 -32
- data/lib/riffer/providers/anthropic.rb +1 -1
- data/lib/riffer/providers/base.rb +11 -75
- data/lib/riffer/providers/repository.rb +44 -4
- data/lib/riffer/tools/runtime.rb +8 -29
- data/lib/riffer/version.rb +1 -1
- data/sig/_private/opentelemetry.rbs +3 -6
- data/sig/generated/riffer/agent/run.rbs +4 -8
- data/sig/generated/riffer/agent.rbs +1 -1
- data/sig/generated/riffer/config.rbs +0 -33
- data/sig/generated/riffer/guardrails/runner.rbs +0 -8
- data/sig/generated/riffer/providers/base.rbs +2 -20
- data/sig/generated/riffer/providers/repository.rbs +30 -2
- data/sig/generated/riffer/tools/runtime.rbs +2 -6
- metadata +7 -18
- data/docs/17_METRICS.md +0 -155
- data/lib/riffer/metrics/instruments.rb +0 -31
- data/lib/riffer/metrics/no_op.rb +0 -14
- data/lib/riffer/metrics/otel.rb +0 -80
- data/lib/riffer/metrics.rb +0 -94
- data/sig/generated/riffer/metrics/instruments.rbs +0 -15
- data/sig/generated/riffer/metrics/no_op.rbs +0 -10
- data/sig/generated/riffer/metrics/otel.rbs +0 -48
- data/sig/generated/riffer/metrics.rbs +0 -72
- data/sig/manual/riffer/metrics/no_op.rbs +0 -5
- data/sig/manual/riffer/metrics.rbs +0 -5
|
@@ -1,13 +1,10 @@
|
|
|
1
|
-
# Minimal signatures for the `opentelemetry-api`
|
|
2
|
-
#
|
|
1
|
+
# Minimal signatures for the `opentelemetry-api` gem (undeclared, runtime
|
|
2
|
+
# detected), which ships no RBS.
|
|
3
3
|
#
|
|
4
|
-
# Only the surface used by Riffer::Tracing::Otel
|
|
5
|
-
# declared.
|
|
4
|
+
# Only the surface used by Riffer::Tracing::Otel is declared.
|
|
6
5
|
module OpenTelemetry
|
|
7
6
|
def self.tracer_provider: () -> untyped
|
|
8
7
|
|
|
9
|
-
def self.meter_provider: () -> untyped
|
|
10
|
-
|
|
11
8
|
class Context
|
|
12
9
|
def self.current: () -> OpenTelemetry::Context
|
|
13
10
|
|
|
@@ -21,8 +21,8 @@ module Riffer::Agent::Run
|
|
|
21
21
|
|
|
22
22
|
# Both +generate+ and +stream+ funnel here, so this is the single place raw
|
|
23
23
|
# +tags+ are normalized. The clean <tt>String => String</tt> map is then
|
|
24
|
-
# threaded to every span
|
|
25
|
-
#
|
|
24
|
+
# threaded to every span builder in the run as +riffer.tag.*+ and to each
|
|
25
|
+
# provider call (via +merged_model_options+) for native request-metadata
|
|
26
26
|
# mapping.
|
|
27
27
|
# --
|
|
28
28
|
# : (Riffer::Agent, ?tags: Hash[(String | Symbol), untyped]?, ?stream_yielder: Enumerator::Yielder?) -> Riffer::Agent::Response
|
|
@@ -86,8 +86,8 @@ module Riffer::Agent::Run
|
|
|
86
86
|
|
|
87
87
|
# +tags+ rides in the options hash as a curated key the providers extract for
|
|
88
88
|
# native request-metadata mapping (alongside +:structured_output+); it never
|
|
89
|
-
# reaches an SDK call verbatim. Span
|
|
90
|
-
#
|
|
89
|
+
# reaches an SDK call verbatim. Span tagging is threaded separately to each
|
|
90
|
+
# builder.
|
|
91
91
|
# --
|
|
92
92
|
# : (Riffer::Agent, ?Hash[String, String]) -> Hash[Symbol, untyped]
|
|
93
93
|
def merged_model_options: (Riffer::Agent, ?Hash[String, String]) -> Hash[Symbol, untyped]
|
|
@@ -114,10 +114,6 @@ module Riffer::Agent::Run
|
|
|
114
114
|
# : (Riffer::Agent, ?Hash[String, String]) -> Hash[String, untyped]
|
|
115
115
|
def run_span_attributes: (Riffer::Agent, ?Hash[String, String]) -> Hash[String, untyped]
|
|
116
116
|
|
|
117
|
-
# --
|
|
118
|
-
# : (Riffer::Agent, String?, ?Hash[String, String]) -> Hash[String, untyped]
|
|
119
|
-
def run_metric_attributes: (Riffer::Agent, String?, ?Hash[String, String]) -> Hash[String, untyped]
|
|
120
|
-
|
|
121
117
|
# --
|
|
122
118
|
# : (Hash[(String | Symbol), untyped]?) -> Hash[String, String]
|
|
123
119
|
def normalize_tags: (Hash[String | Symbol, untyped]?) -> Hash[String, String]
|
|
@@ -215,7 +215,7 @@ class Riffer::Agent
|
|
|
215
215
|
#
|
|
216
216
|
# +tags:+ is an optional flat hash of attribution tags applied to this single
|
|
217
217
|
# call: they propagate to the provider's native request-metadata field, and
|
|
218
|
-
# are stamped as +riffer.tag.*+ on every span
|
|
218
|
+
# are stamped as +riffer.tag.*+ on every span the call emits. See
|
|
219
219
|
# +docs/03_AGENTS.md+ for the per-provider mapping. The reserved key
|
|
220
220
|
# +user_id+ also maps to the provider's native user identifier where one
|
|
221
221
|
# exists.
|
|
@@ -137,36 +137,6 @@ class Riffer::Config
|
|
|
137
137
|
def backend=: (untyped) -> void
|
|
138
138
|
end
|
|
139
139
|
|
|
140
|
-
# Metrics-related global configuration, independent of +config.tracing+ so a
|
|
141
|
-
# host can run one signal without the other.
|
|
142
|
-
class Metrics
|
|
143
|
-
# Whether riffer records OTEL metric instruments; defaults to +true+, a
|
|
144
|
-
# no-op until a host wires an OTEL metrics SDK.
|
|
145
|
-
attr_reader enabled: bool
|
|
146
|
-
|
|
147
|
-
# The backend riffer routes measurements through; defaults to +nil+, a no-op.
|
|
148
|
-
# Riffer auto-detects no backend; assigning one is opt-in.
|
|
149
|
-
attr_reader backend: untyped
|
|
150
|
-
|
|
151
|
-
# --
|
|
152
|
-
# : () -> void
|
|
153
|
-
def initialize: () -> void
|
|
154
|
-
|
|
155
|
-
# Sets the enabled flag, coercing boolean-ish values so an env-var
|
|
156
|
-
# +"false"+ (truthy in Ruby) doesn't silently keep metrics on. Raises
|
|
157
|
-
# Riffer::ArgumentError on an unrecognized value.
|
|
158
|
-
# --
|
|
159
|
-
# : (untyped) -> void
|
|
160
|
-
def enabled=: (untyped) -> void
|
|
161
|
-
|
|
162
|
-
# Sets the metrics backend riffer routes measurements through. Raises
|
|
163
|
-
# Riffer::ArgumentError unless the value is +nil+ or responds to
|
|
164
|
-
# +record_histogram+.
|
|
165
|
-
# --
|
|
166
|
-
# : (untyped) -> void
|
|
167
|
-
def backend=: (untyped) -> void
|
|
168
|
-
end
|
|
169
|
-
|
|
170
140
|
# Consumer-configured token pricing, keyed by +provider/model+ id. Riffer
|
|
171
141
|
# ships no price table, so an unconfigured model carries no cost.
|
|
172
142
|
class Pricing
|
|
@@ -276,9 +246,6 @@ class Riffer::Config
|
|
|
276
246
|
# Tracing-related global configuration.
|
|
277
247
|
attr_reader tracing: Riffer::Config::Tracing
|
|
278
248
|
|
|
279
|
-
# Metrics-related global configuration.
|
|
280
|
-
attr_reader metrics: Riffer::Config::Metrics
|
|
281
|
-
|
|
282
249
|
# Consumer-configured per-model token pricing.
|
|
283
250
|
attr_reader pricing: Riffer::Config::Pricing
|
|
284
251
|
|
|
@@ -40,10 +40,6 @@ class Riffer::Guardrails::Runner
|
|
|
40
40
|
# : (Riffer::Guardrail, untyped, messages: Array[Riffer::Messages::Base]?) -> Riffer::Guardrails::Result
|
|
41
41
|
def execute_guardrail: (Riffer::Guardrail, untyped, messages: Array[Riffer::Messages::Base]?) -> Riffer::Guardrails::Result
|
|
42
42
|
|
|
43
|
-
# --
|
|
44
|
-
# : (Riffer::Guardrail) { () -> Riffer::Guardrails::Result } -> Riffer::Guardrails::Result
|
|
45
|
-
def instrument_guardrail: (Riffer::Guardrail) { () -> Riffer::Guardrails::Result } -> Riffer::Guardrails::Result
|
|
46
|
-
|
|
47
43
|
# --
|
|
48
44
|
# : (Riffer::Guardrail, untyped, messages: Array[Riffer::Messages::Base]?) -> Riffer::Guardrails::Result
|
|
49
45
|
def run_guardrail_phase: (Riffer::Guardrail, untyped, messages: Array[Riffer::Messages::Base]?) -> Riffer::Guardrails::Result
|
|
@@ -52,10 +48,6 @@ class Riffer::Guardrails::Runner
|
|
|
52
48
|
# : (Riffer::Guardrail) -> Hash[String, untyped]
|
|
53
49
|
def guardrail_span_attributes: (Riffer::Guardrail) -> Hash[String, untyped]
|
|
54
50
|
|
|
55
|
-
# --
|
|
56
|
-
# : (Riffer::Guardrail, String?) -> Hash[String, untyped]
|
|
57
|
-
def guardrail_metric_attributes: (Riffer::Guardrail, String?) -> Hash[String, untyped]
|
|
58
|
-
|
|
59
51
|
# A block is a handled outcome, so its span status stays unset — an error
|
|
60
52
|
# span status is reserved for a raised exception.
|
|
61
53
|
# --
|
|
@@ -98,30 +98,12 @@ class Riffer::Providers::Base
|
|
|
98
98
|
# : (String?, Hash[Symbol, untyped]) -> Hash[String, untyped]
|
|
99
99
|
def chat_span_attributes: (String?, Hash[Symbol, untyped]) -> Hash[String, untyped]
|
|
100
100
|
|
|
101
|
-
#
|
|
102
|
-
#
|
|
103
|
-
def chat_metric_base_attributes: (String?, ?Hash[String, String]) -> Hash[String, untyped]
|
|
104
|
-
|
|
105
|
-
# --
|
|
106
|
-
# : (String?, String?, ?Hash[String, String]) -> Hash[String, untyped]
|
|
107
|
-
def chat_metric_attributes: (String?, String?, ?Hash[String, String]) -> Hash[String, untyped]
|
|
108
|
-
|
|
109
|
-
# Maps normalized tags to their namespaced span/metric attribute form. An
|
|
110
|
-
# empty map yields an empty hash, so merging it is a no-op.
|
|
101
|
+
# Maps normalized tags to their namespaced span attribute form. An empty map
|
|
102
|
+
# yields an empty hash, so merging it is a no-op.
|
|
111
103
|
# --
|
|
112
104
|
# : (Hash[String, String]) -> Hash[String, String]
|
|
113
105
|
def tag_attributes: (Hash[String, String]) -> Hash[String, String]
|
|
114
106
|
|
|
115
|
-
# Per-call only — the run level would double-count an aggregate.
|
|
116
|
-
# --
|
|
117
|
-
# : (String?, Riffer::Providers::TokenUsage?, ?Hash[String, String]) -> void
|
|
118
|
-
def record_token_usage_metric: (String?, Riffer::Providers::TokenUsage?, ?Hash[String, String]) -> void
|
|
119
|
-
|
|
120
|
-
# Per-call only — the run level would double-count an aggregate.
|
|
121
|
-
# --
|
|
122
|
-
# : (String?, Riffer::Providers::TokenUsage?, ?Hash[String, String]) -> void
|
|
123
|
-
def record_cost_metric: (String?, Riffer::Providers::TokenUsage?, ?Hash[String, String]) -> void
|
|
124
|
-
|
|
125
107
|
# --
|
|
126
108
|
# : ((Riffer::Tracing::Otel::Span | Riffer::Tracing::NoOp::Span), Symbol?, String?) -> void
|
|
127
109
|
def record_finish_reason: (Riffer::Tracing::Otel::Span | Riffer::Tracing::NoOp::Span, Symbol?, String?) -> void
|
|
@@ -1,12 +1,34 @@
|
|
|
1
1
|
# Generated from lib/riffer/providers/repository.rb with RBS::Inline
|
|
2
2
|
|
|
3
|
-
#
|
|
3
|
+
# Resolves provider classes by identifier, combining the built-in REPO with
|
|
4
|
+
# consumer registrations added through +register+. Registration is not
|
|
5
|
+
# synchronized — register during boot, before concurrent generation begins.
|
|
4
6
|
module Riffer::Providers::Repository
|
|
7
|
+
@registrations: Hash[Symbol, ^() -> singleton(Riffer::Providers::Base)]
|
|
8
|
+
|
|
5
9
|
@key_for: Hash[singleton(Riffer::Providers::Base), Symbol]?
|
|
6
10
|
|
|
7
11
|
REPO: Hash[Symbol, ^() -> singleton(Riffer::Providers::Base)]
|
|
8
12
|
|
|
9
|
-
#
|
|
13
|
+
# Registers a custom provider under +identifier+, resolved lazily by the block.
|
|
14
|
+
# Takes precedence over a built-in sharing the identifier.
|
|
15
|
+
#
|
|
16
|
+
# Riffer::Providers::Repository.register(:jane) { MyApp::JaneProvider }
|
|
17
|
+
#
|
|
18
|
+
# Raises Riffer::ArgumentError when called without a block.
|
|
19
|
+
#
|
|
20
|
+
# --
|
|
21
|
+
# : ((String | Symbol)) { () -> singleton(Riffer::Providers::Base) } -> void
|
|
22
|
+
def register: (String | Symbol) { () -> singleton(Riffer::Providers::Base) } -> void
|
|
23
|
+
|
|
24
|
+
# Removes a custom registration by identifier, leaving any built-in of the
|
|
25
|
+
# same name intact.
|
|
26
|
+
# --
|
|
27
|
+
# : ((String | Symbol)) -> void
|
|
28
|
+
def unregister: (String | Symbol) -> void
|
|
29
|
+
|
|
30
|
+
# Finds a provider class by identifier, preferring a custom registration over
|
|
31
|
+
# a built-in of the same name.
|
|
10
32
|
#
|
|
11
33
|
# --
|
|
12
34
|
# : ((String | Symbol)) -> singleton(Riffer::Providers::Base)?
|
|
@@ -16,4 +38,10 @@ module Riffer::Providers::Repository
|
|
|
16
38
|
# --
|
|
17
39
|
# : (singleton(Riffer::Providers::Base)) -> Symbol?
|
|
18
40
|
def key_for: (singleton(Riffer::Providers::Base)) -> Symbol?
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
# --
|
|
45
|
+
# : () -> Hash[singleton(Riffer::Providers::Base), Symbol]
|
|
46
|
+
def build_key_index: () -> Hash[singleton(Riffer::Providers::Base), Symbol]
|
|
19
47
|
end
|
|
@@ -56,12 +56,8 @@ class Riffer::Tools::Runtime
|
|
|
56
56
|
# : (Riffer::Messages::Assistant::ToolCall, ?Hash[String, String]) -> Hash[String, untyped]
|
|
57
57
|
def tool_span_attributes: (Riffer::Messages::Assistant::ToolCall, ?Hash[String, String]) -> Hash[String, untyped]
|
|
58
58
|
|
|
59
|
-
#
|
|
60
|
-
#
|
|
61
|
-
def tool_metric_attributes: (Riffer::Messages::Assistant::ToolCall, String?, ?Hash[String, String]) -> Hash[String, untyped]
|
|
62
|
-
|
|
63
|
-
# Maps normalized tags to their namespaced span/metric attribute form. An
|
|
64
|
-
# empty map yields an empty hash, so merging it is a no-op.
|
|
59
|
+
# Maps normalized tags to their namespaced span attribute form. An empty map
|
|
60
|
+
# yields an empty hash, so merging it is a no-op.
|
|
65
61
|
# --
|
|
66
62
|
# : (Hash[String, String]) -> Hash[String, String]
|
|
67
63
|
def tag_attributes: (Hash[String, String]) -> Hash[String, String]
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: riffer
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.37.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Jake Bottrall
|
|
@@ -35,14 +35,14 @@ dependencies:
|
|
|
35
35
|
requirements:
|
|
36
36
|
- - "~>"
|
|
37
37
|
- !ruby/object:Gem::Version
|
|
38
|
-
version: 1.
|
|
38
|
+
version: 1.55.0
|
|
39
39
|
type: :development
|
|
40
40
|
prerelease: false
|
|
41
41
|
version_requirements: !ruby/object:Gem::Requirement
|
|
42
42
|
requirements:
|
|
43
43
|
- - "~>"
|
|
44
44
|
- !ruby/object:Gem::Version
|
|
45
|
-
version: 1.
|
|
45
|
+
version: 1.55.0
|
|
46
46
|
- !ruby/object:Gem::Dependency
|
|
47
47
|
name: aws-sdk-bedrockruntime
|
|
48
48
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -108,7 +108,7 @@ dependencies:
|
|
|
108
108
|
version: '2.25'
|
|
109
109
|
- - "<"
|
|
110
110
|
- !ruby/object:Gem::Version
|
|
111
|
-
version: '2.
|
|
111
|
+
version: '2.43'
|
|
112
112
|
type: :development
|
|
113
113
|
prerelease: false
|
|
114
114
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -118,21 +118,21 @@ dependencies:
|
|
|
118
118
|
version: '2.25'
|
|
119
119
|
- - "<"
|
|
120
120
|
- !ruby/object:Gem::Version
|
|
121
|
-
version: '2.
|
|
121
|
+
version: '2.43'
|
|
122
122
|
- !ruby/object:Gem::Dependency
|
|
123
123
|
name: io-event
|
|
124
124
|
requirement: !ruby/object:Gem::Requirement
|
|
125
125
|
requirements:
|
|
126
126
|
- - "<"
|
|
127
127
|
- !ruby/object:Gem::Version
|
|
128
|
-
version: '1.
|
|
128
|
+
version: '1.20'
|
|
129
129
|
type: :development
|
|
130
130
|
prerelease: false
|
|
131
131
|
version_requirements: !ruby/object:Gem::Requirement
|
|
132
132
|
requirements:
|
|
133
133
|
- - "<"
|
|
134
134
|
- !ruby/object:Gem::Version
|
|
135
|
-
version: '1.
|
|
135
|
+
version: '1.20'
|
|
136
136
|
- !ruby/object:Gem::Dependency
|
|
137
137
|
name: minitest
|
|
138
138
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -278,7 +278,6 @@ files:
|
|
|
278
278
|
- docs/14_MCP.md
|
|
279
279
|
- docs/15_SERIALIZATION.md
|
|
280
280
|
- docs/16_TRACING.md
|
|
281
|
-
- docs/17_METRICS.md
|
|
282
281
|
- docs/providers/01_PROVIDERS.md
|
|
283
282
|
- docs/providers/02_AMAZON_BEDROCK.md
|
|
284
283
|
- docs/providers/03_ANTHROPIC.md
|
|
@@ -334,10 +333,6 @@ files:
|
|
|
334
333
|
- lib/riffer/messages/system.rb
|
|
335
334
|
- lib/riffer/messages/tool.rb
|
|
336
335
|
- lib/riffer/messages/user.rb
|
|
337
|
-
- lib/riffer/metrics.rb
|
|
338
|
-
- lib/riffer/metrics/instruments.rb
|
|
339
|
-
- lib/riffer/metrics/no_op.rb
|
|
340
|
-
- lib/riffer/metrics/otel.rb
|
|
341
336
|
- lib/riffer/params.rb
|
|
342
337
|
- lib/riffer/params/boolean.rb
|
|
343
338
|
- lib/riffer/params/param.rb
|
|
@@ -455,10 +450,6 @@ files:
|
|
|
455
450
|
- sig/generated/riffer/messages/system.rbs
|
|
456
451
|
- sig/generated/riffer/messages/tool.rbs
|
|
457
452
|
- sig/generated/riffer/messages/user.rbs
|
|
458
|
-
- sig/generated/riffer/metrics.rbs
|
|
459
|
-
- sig/generated/riffer/metrics/instruments.rbs
|
|
460
|
-
- sig/generated/riffer/metrics/no_op.rbs
|
|
461
|
-
- sig/generated/riffer/metrics/otel.rbs
|
|
462
453
|
- sig/generated/riffer/params.rbs
|
|
463
454
|
- sig/generated/riffer/params/boolean.rbs
|
|
464
455
|
- sig/generated/riffer/params/param.rbs
|
|
@@ -532,8 +523,6 @@ files:
|
|
|
532
523
|
- sig/manual/riffer/mcp/authenticated_tool.rbs
|
|
533
524
|
- sig/manual/riffer/mcp/registry.rbs
|
|
534
525
|
- sig/manual/riffer/mcp/tool_factory.rbs
|
|
535
|
-
- sig/manual/riffer/metrics.rbs
|
|
536
|
-
- sig/manual/riffer/metrics/no_op.rbs
|
|
537
526
|
- sig/manual/riffer/providers.rbs
|
|
538
527
|
- sig/manual/riffer/providers/repository.rbs
|
|
539
528
|
- sig/manual/riffer/tracing.rbs
|
data/docs/17_METRICS.md
DELETED
|
@@ -1,155 +0,0 @@
|
|
|
1
|
-
# Metrics
|
|
2
|
-
|
|
3
|
-
Riffer can record [OpenTelemetry](https://opentelemetry.io/) metric instruments alongside its [spans](16_TRACING.md), following the [GenAI semantic conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/). Metric names, instrument types, units, and attributes are a public, versioned contract you can build dashboards and alerts against. This page is the reference for that contract.
|
|
4
|
-
|
|
5
|
-
As with tracing, Riffer only _records_ instruments, and only through a backend you assign to `config.metrics.backend` — OpenTelemetry is the built-in option you opt into (the host application owns the SDK, metric reader, exporter, and aggregation — the standard OTEL split), but never a default. With no backend assigned, every measurement is a silent no-op and Riffer carries no OpenTelemetry gem dependency.
|
|
6
|
-
|
|
7
|
-
> **OpenTelemetry metrics for Ruby is still pre-1.0.** The metrics API and SDK ship as separate, experimental gems (`opentelemetry-metrics-api`, `opentelemetry-metrics-sdk`) from the stable 1.x traces API. Riffer guards against an incompatible API and falls back to a no-op outside the supported range, but expect the host-side wiring below to evolve with those gems.
|
|
8
|
-
|
|
9
|
-
## Enabling metrics
|
|
10
|
-
|
|
11
|
-
Riffer records measurements only through a backend you assign to `config.metrics.backend` — it does **not** auto-detect OpenTelemetry. To use OTEL, add the metrics SDK, register a metric reader with an exporter, and assign Riffer's built-in OTEL backend with `Riffer::Metrics::Otel.build`.
|
|
12
|
-
|
|
13
|
-
```ruby
|
|
14
|
-
# Gemfile
|
|
15
|
-
gem "opentelemetry-metrics-sdk"
|
|
16
|
-
```
|
|
17
|
-
|
|
18
|
-
```ruby
|
|
19
|
-
require "opentelemetry-metrics-sdk"
|
|
20
|
-
|
|
21
|
-
OpenTelemetry::SDK.configure do |c|
|
|
22
|
-
c.service_name = "my-agent-host"
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
Riffer.configure do |config|
|
|
26
|
-
config.metrics.backend = Riffer::Metrics::Otel.build
|
|
27
|
-
end
|
|
28
|
-
```
|
|
29
|
-
|
|
30
|
-
The metrics SDK is **separate** from the traces SDK (`opentelemetry-sdk`); add it explicitly even if you already trace. `Riffer::Metrics::Otel.build` wraps the global `OpenTelemetry.meter_provider` by default; pass `provider:` to wrap a specific one. It returns `nil` — leaving metrics a no-op rather than raising — when the `opentelemetry-metrics-api` gem is absent or outside the supported range (>= 0.2, < 1.0). Any backend implementing the OpenTelemetry Metrics API then ingests Riffer's instruments. For real reader and exporter setup (OTLP, periodic export, Views), see the [OpenTelemetry Ruby docs](https://opentelemetry.io/docs/languages/ruby/).
|
|
31
|
-
|
|
32
|
-
The metrics knobs — the `enabled` kill switch and the `backend` itself — live in [Configuration — Metrics](10_CONFIGURATION.md#metrics). They are **independent** of the tracing knobs (each signal has its own backend): you can run tracing while metrics are off, or the reverse.
|
|
33
|
-
|
|
34
|
-
Instruments are recorded under the instrumentation scope named `riffer`, versioned with the Riffer gem version — the runtime signal for which release produced a measurement; see [Stability](#stability).
|
|
35
|
-
|
|
36
|
-
## Routing to a non-OpenTelemetry backend
|
|
37
|
-
|
|
38
|
-
OpenTelemetry is one backend, not the only one. A host that already runs another metrics stack — DogStatsD, say — can route Riffer's measurements into it with **no `opentelemetry-*` gem installed** by assigning its own backend to `config.metrics.backend` in place of `Riffer::Metrics::Otel.build`. Whatever you assign is the backend; there is no fallback and no auto-detection — an unset backend is a no-op.
|
|
39
|
-
|
|
40
|
-
```ruby
|
|
41
|
-
Riffer.configure do |config|
|
|
42
|
-
config.metrics.backend = MyDogStatsdMetricsBackend.new
|
|
43
|
-
end
|
|
44
|
-
```
|
|
45
|
-
|
|
46
|
-
The backend is duck-typed: any object that responds to `record_histogram(name, value, unit:, description:, attributes:)` works, and the setter validates only that (otherwise it raises `Riffer::ArgumentError`). `Riffer::Metrics::NoOp` is the reference shape. All four instruments are histograms, so the single `record_histogram` method covers the full contract; tell them apart by `name`. Two carry unit `s` — `gen_ai.client.operation.duration` and `riffer.guardrail.duration` — so `name` is the reliable discriminator; `{token}` is `gen_ai.client.token.usage` and `USD` is `riffer.gen_ai.cost`.
|
|
47
|
-
|
|
48
|
-
A custom backend counts as a **live** sink, so the providers still compute the token counts and cost that feed it — the same data that, under OTEL, populates `gen_ai.client.token.usage` and `riffer.gen_ai.cost`. The `enabled` kill switch is honoured ahead of the backend: with `config.metrics.enabled = false`, measurements short-circuit to the no-op without ever reaching a custom backend.
|
|
49
|
-
|
|
50
|
-
### Bucket boundaries
|
|
51
|
-
|
|
52
|
-
Histogram bucket boundaries are a **host-side** concern. The OpenTelemetry metrics API does not let an instrumenting library attach bucket boundaries at instrument creation, so Riffer does not set them — the SDK's default buckets apply unless you override them. To match the GenAI semantic conventions' recommended boundaries (or your own), register a [View](https://opentelemetry.io/docs/specs/otel/metrics/sdk/#view) on the meter provider that targets the instrument by name and sets explicit bucket boundaries.
|
|
53
|
-
|
|
54
|
-
The convention recommends boundaries scaled to each instrument, so register one View per histogram — the token-count buckets below are for `gen_ai.client.token.usage`; `gen_ai.client.operation.duration` wants its own latency-scaled set, and `riffer.gen_ai.cost` a USD-scaled one.
|
|
55
|
-
|
|
56
|
-
```ruby
|
|
57
|
-
require "opentelemetry-metrics-sdk"
|
|
58
|
-
|
|
59
|
-
OpenTelemetry::SDK.configure do |c|
|
|
60
|
-
c.service_name = "my-agent-host"
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
# The GenAI semconv's recommended token-count boundaries. Register the View
|
|
64
|
-
# before Riffer records its first measurement.
|
|
65
|
-
OpenTelemetry.meter_provider.add_view(
|
|
66
|
-
"gen_ai.client.token.usage",
|
|
67
|
-
aggregation: OpenTelemetry::SDK::Metrics::Aggregation::ExplicitBucketHistogram.new(
|
|
68
|
-
boundaries: [1, 4, 16, 64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864]
|
|
69
|
-
)
|
|
70
|
-
)
|
|
71
|
-
```
|
|
72
|
-
|
|
73
|
-
## Instruments
|
|
74
|
-
|
|
75
|
-
Each instrument is documented here as a row carrying its name, instrument type, unit, and attribute set.
|
|
76
|
-
|
|
77
|
-
> **Per-call tags become dimensions.** Any tags passed to `#generate` / `#stream` via `tags:` are added as `riffer.tag.<key>` attributes to **every** instrument below — `operation.duration` (each operation), `token.usage` (each point), `cost`, and `guardrail.duration` — so the attribute lists omit them. Tags are promoted verbatim with no allow-list — a high-cardinality value like a unique `user_id` per request multiplies time series accordingly; drop it with a [View](https://opentelemetry.io/docs/specs/otel/metrics/sdk/#view) or keep metric-bound tags low-cardinality. See [Per-Call Tags](03_AGENTS.md#per-call-tags).
|
|
78
|
-
|
|
79
|
-
### `gen_ai.client.operation.duration`
|
|
80
|
-
|
|
81
|
-
Histogram, unit `s`. The latency of a single GenAI operation, recorded around the same wrap as the matching [span](16_TRACING.md) on both the success and error paths and timed with a monotonic clock. Recording is independent of tracing — the metric fires even with `config.tracing.enabled = false`. Tell the three operations apart by `gen_ai.operation.name`.
|
|
82
|
-
|
|
83
|
-
| `gen_ai.operation.name` | Recorded around | Attributes |
|
|
84
|
-
| ----------------------- | -------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- |
|
|
85
|
-
| `chat` | each provider call (`generate_text`/`stream_text`) | `gen_ai.operation.name`, `gen_ai.provider.name`, `gen_ai.request.model` (when set), `error.type` (on error) |
|
|
86
|
-
| `invoke_agent` | each agent run (`generate`/`stream`) | `gen_ai.operation.name`, `gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.agent.name`, `error.type` (on error) |
|
|
87
|
-
| `execute_tool` | each tool call | `gen_ai.operation.name`, `gen_ai.tool.name`, `error.type` (on error) |
|
|
88
|
-
|
|
89
|
-
`error.type` carries the exception class for a raised error; for `execute_tool` it carries the handled error category (e.g. `validation_error`, `timeout_error`) when a tool returns an error result instead of raising — matching the span. `gen_ai.response.model` is not recorded yet; it will land once it is also captured on the chat span.
|
|
90
|
-
|
|
91
|
-
> **Streamed operations are consumption-paced.** A streamed `chat` or `invoke_agent` records its duration when the stream drains, so the value includes the time your consumer takes to iterate the events, not just provider latency. The matching span behaves the same way.
|
|
92
|
-
|
|
93
|
-
> **`gen_ai.tool.name` cardinality.** One time series exists per distinct tool name. With a large or dynamic tool set (for example MCP-discovered tools) that can grow unbounded — drop the attribute with a [View](https://opentelemetry.io/docs/specs/otel/metrics/sdk/#view) if your backend strains.
|
|
94
|
-
|
|
95
|
-
### `gen_ai.client.token.usage`
|
|
96
|
-
|
|
97
|
-
Histogram, unit `{token}`. Token volume for a single `chat` call, recorded from the normalized token usage after the provider responds. Each call emits **two data points** — one `input`, one `output` — distinguished by `gen_ai.token.type`. Recording is independent of tracing (it fires with `config.tracing.enabled = false`); for a streamed call it fires when the stream drains. `gen_ai.response.model` is not recorded yet, for the same reason as `operation.duration`.
|
|
98
|
-
|
|
99
|
-
| `gen_ai.token.type` | Value | Attributes |
|
|
100
|
-
| ------------------- | ----------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
|
|
101
|
-
| `input` | total prompt tokens for the call, **cache-inclusive** | `gen_ai.operation.name` (always `chat`), `gen_ai.provider.name`, `gen_ai.token.type`, `gen_ai.request.model` (when set) |
|
|
102
|
-
| `output` | tokens generated, including reasoning/thinking tokens | same |
|
|
103
|
-
|
|
104
|
-
> **Per-call only.** Token usage is never recorded at the run (`invoke_agent`) level. Metrics pre-aggregate, so emitting both the per-call points and a run total would double-count — sum the per-call points in your backend if you want a run total. This is the metric-side counterpart of the span-level [double-count trap](16_TRACING.md#token-usage-and-cost).
|
|
105
|
-
|
|
106
|
-
> **Cache buckets stay on spans.** The semconv `gen_ai.token.type` defines only `input` and `output`, so the prompt-cache subsets (`cache_read` / `cache_creation`) live on [spans](16_TRACING.md#token-usage-and-cost), not this metric. The `input` value is the cache-inclusive total, matching the span's `gen_ai.usage.input_tokens`.
|
|
107
|
-
|
|
108
|
-
A call that reports no usage records no data points, and a failed call has nothing to count — so this metric carries no `error.type` (the semconv marks it not applicable here, unlike `operation.duration`).
|
|
109
|
-
|
|
110
|
-
### `riffer.gen_ai.cost`
|
|
111
|
-
|
|
112
|
-
Histogram, unit `USD`. The cost of a single `chat` call, recorded from the [cost](16_TRACING.md#token-usage-and-cost) on the normalized token usage after the provider responds — the same source as the cost span attribute, a different sink. This instrument is Riffer-owned (`riffer.*`, not `gen_ai.*`) so it won't collide if the semantic conventions later define a cost instrument; see [Stability](#stability). Recording is independent of tracing (it fires with `config.tracing.enabled = false`); for a streamed call it fires when the stream drains.
|
|
113
|
-
|
|
114
|
-
| Value | Attributes |
|
|
115
|
-
| ---------------- | -------------------------------------------------------------------------------------------------- |
|
|
116
|
-
| cost of the call | `gen_ai.operation.name` (always `chat`), `gen_ai.provider.name`, `gen_ai.request.model` (when set) |
|
|
117
|
-
|
|
118
|
-
Pricing is **consumer-configured** — no price table ships with the gem (see [Configuration — Pricing](10_CONFIGURATION.md#pricing)). A call whose model has no configured price records **no** data point, so this metric covers only priced calls; `operation.duration` and `token.usage` still record. A priced call that computes to `0.0` does record a zero data point — only an absent price means there is nothing to measure.
|
|
119
|
-
|
|
120
|
-
> **Per-call only.** Cost is never recorded at the run (`invoke_agent`) level, for the same reason as token usage: metrics pre-aggregate, so emitting both per-call points and a run total would double-count. Sum the per-call points in your backend for a run total.
|
|
121
|
-
|
|
122
|
-
### `riffer.guardrail.duration`
|
|
123
|
-
|
|
124
|
-
Histogram, unit `s`. The latency of a single guardrail execution, recorded around the same wrap as the [`execute_guardrail` span](16_TRACING.md) on both the success and raise paths and timed with a monotonic clock. Guardrails run on the request hot path — before every model turn and after every response — so this is the guardrail counterpart to `gen_ai.client.operation.duration`. Recording is independent of tracing — the metric fires even with `config.tracing.enabled = false`.
|
|
125
|
-
|
|
126
|
-
This instrument is Riffer-owned (`riffer.*`, not `gen_ai.*`) by the same reasoning as its span: a guardrail is not a GenAI semantic-convention operation, so the `execute_guardrail` span deliberately carries no `gen_ai.operation.name` and lives in riffer's own `riffer.guardrail.*` namespace (see [Tracing](16_TRACING.md)). Folding the metric into `gen_ai.client.operation.duration` would contradict that, so the duration is its own riffer-owned histogram instead; see [Stability](#stability).
|
|
127
|
-
|
|
128
|
-
| Value | Attributes |
|
|
129
|
-
| ----------------------------------- | -------------------------------------------------------------------------- |
|
|
130
|
-
| duration of the guardrail execution | `riffer.guardrail.name`, `riffer.guardrail.phase`, `error.type` (on raise) |
|
|
131
|
-
|
|
132
|
-
`riffer.guardrail.phase` is `before` or `after`. A pass, transform, or block is a **handled** outcome and records no `error.type` — that attribute carries the exception class only when a guardrail raises, mirroring the span. One time series exists per `riffer.guardrail.name` × `riffer.guardrail.phase`; guardrail names are bounded by the guardrails you configure, so cardinality is safe — unlike the dynamic tool names on `gen_ai.client.operation.duration`.
|
|
133
|
-
|
|
134
|
-
## Stability
|
|
135
|
-
|
|
136
|
-
The instrument shape is a public, versioned contract, in two tiers — mirroring the [tracing contract](16_TRACING.md#stability):
|
|
137
|
-
|
|
138
|
-
- **`gen_ai.*`** tracks the OpenTelemetry GenAI semantic conventions, pinned to schema version `1.37.0`. That convention is still "Development" status upstream and its names may change; Riffer absorbs such renames deliberately in a release, never silently, with a CHANGELOG entry.
|
|
139
|
-
- **`riffer.*`** is Riffer-owned and changes only through a normal version bump and CHANGELOG entry. Riffer-owned metrics live here so they won't collide if semconv later defines an equivalent.
|
|
140
|
-
|
|
141
|
-
The semantic-convention schema version is a documented pin rather than an instrument attribute — the OpenTelemetry Ruby API can't attach a schema URL to a meter. The runtime version signal is the instrumentation scope: every measurement carries scope name `riffer` at the gem version that recorded it. Pin the Riffer version your dashboards depend on, and watch the CHANGELOG for metrics entries before upgrading.
|
|
142
|
-
|
|
143
|
-
## Avoid double instrumentation
|
|
144
|
-
|
|
145
|
-
Riffer records its agent loop's metrics natively. Running a provider-level GenAI instrumentation gem (for example an OpenTelemetry contrib instrumentation for the underlying Anthropic or OpenAI client) _alongside_ Riffer records the same `gen_ai.client.*` metrics twice. Because metrics pre-aggregate, that duplication is **silent** — it inflates counts and distorts distributions in your dashboards without any obvious per-event trace to inspect.
|
|
146
|
-
|
|
147
|
-
Record one or the other, not both. Since the metrics kill switch is independent of tracing, the usual resolution is to keep Riffer's spans and turn _Riffer's metrics_ off, letting the provider-level instrumentation own the metrics:
|
|
148
|
-
|
|
149
|
-
```ruby
|
|
150
|
-
Riffer.configure do |config|
|
|
151
|
-
config.metrics.enabled = false
|
|
152
|
-
end
|
|
153
|
-
```
|
|
154
|
-
|
|
155
|
-
— or disable the provider-level metric instrumentation and let Riffer own them.
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
# rbs_inline: enabled
|
|
3
|
-
|
|
4
|
-
# The catalog of metric instruments riffer records. Each handle is a constant
|
|
5
|
-
# that resolves its backend at record time, so it survives a meter-provider swap
|
|
6
|
-
# or a runtime +enabled+ flip.
|
|
7
|
-
module Riffer::Metrics::Instruments # :nodoc: all
|
|
8
|
-
OPERATION_DURATION = Riffer::Metrics.create_histogram(
|
|
9
|
-
"gen_ai.client.operation.duration",
|
|
10
|
-
unit: "s",
|
|
11
|
-
description: "Duration of GenAI client operations"
|
|
12
|
-
) #: Riffer::Metrics::Histogram
|
|
13
|
-
|
|
14
|
-
TOKEN_USAGE = Riffer::Metrics.create_histogram(
|
|
15
|
-
"gen_ai.client.token.usage",
|
|
16
|
-
unit: "{token}",
|
|
17
|
-
description: "Number of input and output tokens used in GenAI operations"
|
|
18
|
-
) #: Riffer::Metrics::Histogram
|
|
19
|
-
|
|
20
|
-
COST = Riffer::Metrics.create_histogram(
|
|
21
|
-
"riffer.gen_ai.cost",
|
|
22
|
-
unit: "USD",
|
|
23
|
-
description: "Cost of GenAI client operations in USD"
|
|
24
|
-
) #: Riffer::Metrics::Histogram
|
|
25
|
-
|
|
26
|
-
GUARDRAIL_DURATION = Riffer::Metrics.create_histogram(
|
|
27
|
-
"riffer.guardrail.duration",
|
|
28
|
-
unit: "s",
|
|
29
|
-
description: "Duration of guardrail execution"
|
|
30
|
-
) #: Riffer::Metrics::Histogram
|
|
31
|
-
end
|
data/lib/riffer/metrics/no_op.rb
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
# rbs_inline: enabled
|
|
3
|
-
|
|
4
|
-
# No-op metrics backend, used when the OpenTelemetry metrics API is unavailable
|
|
5
|
-
# or metrics are disabled.
|
|
6
|
-
module Riffer::Metrics::NoOp # :nodoc: all
|
|
7
|
-
extend self
|
|
8
|
-
|
|
9
|
-
# Ignores the measurement; there is no meter without the OTEL metrics API.
|
|
10
|
-
#--
|
|
11
|
-
#: (String, Numeric, unit: String?, description: String?, attributes: Hash[String, untyped]?) -> void
|
|
12
|
-
def record_histogram(name, value, unit:, description:, attributes:)
|
|
13
|
-
end
|
|
14
|
-
end
|
data/lib/riffer/metrics/otel.rb
DELETED
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
# rbs_inline: enabled
|
|
3
|
-
|
|
4
|
-
# OTEL-backed metrics backend. <tt>::OpenTelemetry</tt> constants appear only
|
|
5
|
-
# inside method bodies here, so the gem loads and eager-loads cleanly when the
|
|
6
|
-
# OpenTelemetry metrics API is absent.
|
|
7
|
-
class Riffer::Metrics::Otel # :nodoc: all
|
|
8
|
-
SUPPORTED_API_VERSIONS = Gem::Requirement.new(">= 0.2", "< 1.0") #: Gem::Requirement
|
|
9
|
-
|
|
10
|
-
class << self
|
|
11
|
-
# Builds a backend when the OpenTelemetry metrics API is loadable at a
|
|
12
|
-
# supported version; returns +nil+ so resolution falls back to NoOp.
|
|
13
|
-
# +provider+ defaults to the global <tt>OpenTelemetry.meter_provider</tt>.
|
|
14
|
-
#--
|
|
15
|
-
#: (?provider: untyped) -> Riffer::Metrics::Otel?
|
|
16
|
-
def build(provider: nil)
|
|
17
|
-
version = api_version
|
|
18
|
-
return nil unless version
|
|
19
|
-
|
|
20
|
-
unless supported?(version)
|
|
21
|
-
Kernel.warn "riffer: opentelemetry-metrics-api #{version} is outside the supported range (#{SUPPORTED_API_VERSIONS}); metrics are disabled"
|
|
22
|
-
return nil
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
new(provider: provider || ::OpenTelemetry.meter_provider)
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
# Whether the OpenTelemetry metrics API gem is loadable at a supported
|
|
29
|
-
# version.
|
|
30
|
-
#--
|
|
31
|
-
#: () -> bool
|
|
32
|
-
def available?
|
|
33
|
-
version = api_version
|
|
34
|
-
!version.nil? && supported?(version)
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
# Whether the given opentelemetry-metrics-api version is one riffer codes
|
|
38
|
-
# against. The gem is undeclared, so this guard is the only protection
|
|
39
|
-
# against an incompatible, still-pre-1.0 API.
|
|
40
|
-
#--
|
|
41
|
-
#: (Gem::Version) -> bool
|
|
42
|
-
def supported?(version)
|
|
43
|
-
SUPPORTED_API_VERSIONS.satisfied_by?(version)
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
private
|
|
47
|
-
|
|
48
|
-
#--
|
|
49
|
-
#: () -> Gem::Version?
|
|
50
|
-
def api_version
|
|
51
|
-
require "opentelemetry-metrics-api"
|
|
52
|
-
spec = Gem.loaded_specs["opentelemetry-metrics-api"] #: untyped
|
|
53
|
-
spec&.version
|
|
54
|
-
rescue ::LoadError
|
|
55
|
-
nil
|
|
56
|
-
end
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
# @rbs @meter: untyped
|
|
60
|
-
# @rbs @instruments: Hash[String, untyped]
|
|
61
|
-
# @rbs @mutex: Mutex
|
|
62
|
-
|
|
63
|
-
#--
|
|
64
|
-
#: (provider: untyped) -> void
|
|
65
|
-
def initialize(provider:)
|
|
66
|
-
@meter = provider.meter("riffer", version: Riffer::VERSION)
|
|
67
|
-
@instruments = {}
|
|
68
|
-
@mutex = Mutex.new
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
# Records a value onto the named histogram.
|
|
72
|
-
#--
|
|
73
|
-
#: (String, Numeric, unit: String?, description: String?, attributes: Hash[String, untyped]?) -> void
|
|
74
|
-
def record_histogram(name, value, unit:, description:, attributes:)
|
|
75
|
-
histogram = @mutex.synchronize do
|
|
76
|
-
@instruments[name] ||= @meter.create_histogram(name, unit: unit, description: description)
|
|
77
|
-
end
|
|
78
|
-
histogram.record(value, attributes: attributes)
|
|
79
|
-
end
|
|
80
|
-
end
|