riffer 0.35.0 → 0.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.release-please-manifest.json +1 -1
- data/CHANGELOG.md +11 -0
- data/README.md +0 -1
- data/docs/01_OVERVIEW.md +0 -1
- data/docs/03_AGENTS.md +3 -7
- data/docs/10_CONFIGURATION.md +0 -17
- data/lib/riffer/agent/run.rb +11 -35
- data/lib/riffer/agent.rb +1 -1
- data/lib/riffer/config.rb +0 -45
- data/lib/riffer/guardrails/runner.rb +7 -32
- data/lib/riffer/providers/anthropic.rb +1 -1
- data/lib/riffer/providers/base.rb +11 -75
- data/lib/riffer/tools/runtime.rb +8 -29
- data/lib/riffer/version.rb +1 -1
- data/sig/_private/opentelemetry.rbs +3 -6
- data/sig/generated/riffer/agent/run.rbs +4 -8
- data/sig/generated/riffer/agent.rbs +1 -1
- data/sig/generated/riffer/config.rbs +0 -33
- data/sig/generated/riffer/guardrails/runner.rbs +0 -8
- data/sig/generated/riffer/providers/base.rbs +2 -20
- data/sig/generated/riffer/tools/runtime.rbs +2 -6
- metadata +5 -16
- data/docs/17_METRICS.md +0 -155
- data/lib/riffer/metrics/instruments.rb +0 -31
- data/lib/riffer/metrics/no_op.rb +0 -14
- data/lib/riffer/metrics/otel.rb +0 -80
- data/lib/riffer/metrics.rb +0 -94
- data/sig/generated/riffer/metrics/instruments.rbs +0 -15
- data/sig/generated/riffer/metrics/no_op.rbs +0 -10
- data/sig/generated/riffer/metrics/otel.rbs +0 -48
- data/sig/generated/riffer/metrics.rbs +0 -72
- data/sig/manual/riffer/metrics/no_op.rbs +0 -5
- data/sig/manual/riffer/metrics.rbs +0 -5
|
@@ -40,10 +40,6 @@ class Riffer::Guardrails::Runner
|
|
|
40
40
|
# : (Riffer::Guardrail, untyped, messages: Array[Riffer::Messages::Base]?) -> Riffer::Guardrails::Result
|
|
41
41
|
def execute_guardrail: (Riffer::Guardrail, untyped, messages: Array[Riffer::Messages::Base]?) -> Riffer::Guardrails::Result
|
|
42
42
|
|
|
43
|
-
# --
|
|
44
|
-
# : (Riffer::Guardrail) { () -> Riffer::Guardrails::Result } -> Riffer::Guardrails::Result
|
|
45
|
-
def instrument_guardrail: (Riffer::Guardrail) { () -> Riffer::Guardrails::Result } -> Riffer::Guardrails::Result
|
|
46
|
-
|
|
47
43
|
# --
|
|
48
44
|
# : (Riffer::Guardrail, untyped, messages: Array[Riffer::Messages::Base]?) -> Riffer::Guardrails::Result
|
|
49
45
|
def run_guardrail_phase: (Riffer::Guardrail, untyped, messages: Array[Riffer::Messages::Base]?) -> Riffer::Guardrails::Result
|
|
@@ -52,10 +48,6 @@ class Riffer::Guardrails::Runner
|
|
|
52
48
|
# : (Riffer::Guardrail) -> Hash[String, untyped]
|
|
53
49
|
def guardrail_span_attributes: (Riffer::Guardrail) -> Hash[String, untyped]
|
|
54
50
|
|
|
55
|
-
# --
|
|
56
|
-
# : (Riffer::Guardrail, String?) -> Hash[String, untyped]
|
|
57
|
-
def guardrail_metric_attributes: (Riffer::Guardrail, String?) -> Hash[String, untyped]
|
|
58
|
-
|
|
59
51
|
# A block is a handled outcome, so its span status stays unset — an error
|
|
60
52
|
# span status is reserved for a raised exception.
|
|
61
53
|
# --
|
|
@@ -98,30 +98,12 @@ class Riffer::Providers::Base
|
|
|
98
98
|
# : (String?, Hash[Symbol, untyped]) -> Hash[String, untyped]
|
|
99
99
|
def chat_span_attributes: (String?, Hash[Symbol, untyped]) -> Hash[String, untyped]
|
|
100
100
|
|
|
101
|
-
#
|
|
102
|
-
#
|
|
103
|
-
def chat_metric_base_attributes: (String?, ?Hash[String, String]) -> Hash[String, untyped]
|
|
104
|
-
|
|
105
|
-
# --
|
|
106
|
-
# : (String?, String?, ?Hash[String, String]) -> Hash[String, untyped]
|
|
107
|
-
def chat_metric_attributes: (String?, String?, ?Hash[String, String]) -> Hash[String, untyped]
|
|
108
|
-
|
|
109
|
-
# Maps normalized tags to their namespaced span/metric attribute form. An
|
|
110
|
-
# empty map yields an empty hash, so merging it is a no-op.
|
|
101
|
+
# Maps normalized tags to their namespaced span attribute form. An empty map
|
|
102
|
+
# yields an empty hash, so merging it is a no-op.
|
|
111
103
|
# --
|
|
112
104
|
# : (Hash[String, String]) -> Hash[String, String]
|
|
113
105
|
def tag_attributes: (Hash[String, String]) -> Hash[String, String]
|
|
114
106
|
|
|
115
|
-
# Per-call only — the run level would double-count an aggregate.
|
|
116
|
-
# --
|
|
117
|
-
# : (String?, Riffer::Providers::TokenUsage?, ?Hash[String, String]) -> void
|
|
118
|
-
def record_token_usage_metric: (String?, Riffer::Providers::TokenUsage?, ?Hash[String, String]) -> void
|
|
119
|
-
|
|
120
|
-
# Per-call only — the run level would double-count an aggregate.
|
|
121
|
-
# --
|
|
122
|
-
# : (String?, Riffer::Providers::TokenUsage?, ?Hash[String, String]) -> void
|
|
123
|
-
def record_cost_metric: (String?, Riffer::Providers::TokenUsage?, ?Hash[String, String]) -> void
|
|
124
|
-
|
|
125
107
|
# --
|
|
126
108
|
# : ((Riffer::Tracing::Otel::Span | Riffer::Tracing::NoOp::Span), Symbol?, String?) -> void
|
|
127
109
|
def record_finish_reason: (Riffer::Tracing::Otel::Span | Riffer::Tracing::NoOp::Span, Symbol?, String?) -> void
|
|
@@ -56,12 +56,8 @@ class Riffer::Tools::Runtime
|
|
|
56
56
|
# : (Riffer::Messages::Assistant::ToolCall, ?Hash[String, String]) -> Hash[String, untyped]
|
|
57
57
|
def tool_span_attributes: (Riffer::Messages::Assistant::ToolCall, ?Hash[String, String]) -> Hash[String, untyped]
|
|
58
58
|
|
|
59
|
-
#
|
|
60
|
-
#
|
|
61
|
-
def tool_metric_attributes: (Riffer::Messages::Assistant::ToolCall, String?, ?Hash[String, String]) -> Hash[String, untyped]
|
|
62
|
-
|
|
63
|
-
# Maps normalized tags to their namespaced span/metric attribute form. An
|
|
64
|
-
# empty map yields an empty hash, so merging it is a no-op.
|
|
59
|
+
# Maps normalized tags to their namespaced span attribute form. An empty map
|
|
60
|
+
# yields an empty hash, so merging it is a no-op.
|
|
65
61
|
# --
|
|
66
62
|
# : (Hash[String, String]) -> Hash[String, String]
|
|
67
63
|
def tag_attributes: (Hash[String, String]) -> Hash[String, String]
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: riffer
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.36.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Jake Bottrall
|
|
@@ -35,14 +35,14 @@ dependencies:
|
|
|
35
35
|
requirements:
|
|
36
36
|
- - "~>"
|
|
37
37
|
- !ruby/object:Gem::Version
|
|
38
|
-
version: 1.
|
|
38
|
+
version: 1.51.0
|
|
39
39
|
type: :development
|
|
40
40
|
prerelease: false
|
|
41
41
|
version_requirements: !ruby/object:Gem::Requirement
|
|
42
42
|
requirements:
|
|
43
43
|
- - "~>"
|
|
44
44
|
- !ruby/object:Gem::Version
|
|
45
|
-
version: 1.
|
|
45
|
+
version: 1.51.0
|
|
46
46
|
- !ruby/object:Gem::Dependency
|
|
47
47
|
name: aws-sdk-bedrockruntime
|
|
48
48
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -108,7 +108,7 @@ dependencies:
|
|
|
108
108
|
version: '2.25'
|
|
109
109
|
- - "<"
|
|
110
110
|
- !ruby/object:Gem::Version
|
|
111
|
-
version: '2.
|
|
111
|
+
version: '2.42'
|
|
112
112
|
type: :development
|
|
113
113
|
prerelease: false
|
|
114
114
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -118,7 +118,7 @@ dependencies:
|
|
|
118
118
|
version: '2.25'
|
|
119
119
|
- - "<"
|
|
120
120
|
- !ruby/object:Gem::Version
|
|
121
|
-
version: '2.
|
|
121
|
+
version: '2.42'
|
|
122
122
|
- !ruby/object:Gem::Dependency
|
|
123
123
|
name: io-event
|
|
124
124
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -278,7 +278,6 @@ files:
|
|
|
278
278
|
- docs/14_MCP.md
|
|
279
279
|
- docs/15_SERIALIZATION.md
|
|
280
280
|
- docs/16_TRACING.md
|
|
281
|
-
- docs/17_METRICS.md
|
|
282
281
|
- docs/providers/01_PROVIDERS.md
|
|
283
282
|
- docs/providers/02_AMAZON_BEDROCK.md
|
|
284
283
|
- docs/providers/03_ANTHROPIC.md
|
|
@@ -334,10 +333,6 @@ files:
|
|
|
334
333
|
- lib/riffer/messages/system.rb
|
|
335
334
|
- lib/riffer/messages/tool.rb
|
|
336
335
|
- lib/riffer/messages/user.rb
|
|
337
|
-
- lib/riffer/metrics.rb
|
|
338
|
-
- lib/riffer/metrics/instruments.rb
|
|
339
|
-
- lib/riffer/metrics/no_op.rb
|
|
340
|
-
- lib/riffer/metrics/otel.rb
|
|
341
336
|
- lib/riffer/params.rb
|
|
342
337
|
- lib/riffer/params/boolean.rb
|
|
343
338
|
- lib/riffer/params/param.rb
|
|
@@ -455,10 +450,6 @@ files:
|
|
|
455
450
|
- sig/generated/riffer/messages/system.rbs
|
|
456
451
|
- sig/generated/riffer/messages/tool.rbs
|
|
457
452
|
- sig/generated/riffer/messages/user.rbs
|
|
458
|
-
- sig/generated/riffer/metrics.rbs
|
|
459
|
-
- sig/generated/riffer/metrics/instruments.rbs
|
|
460
|
-
- sig/generated/riffer/metrics/no_op.rbs
|
|
461
|
-
- sig/generated/riffer/metrics/otel.rbs
|
|
462
453
|
- sig/generated/riffer/params.rbs
|
|
463
454
|
- sig/generated/riffer/params/boolean.rbs
|
|
464
455
|
- sig/generated/riffer/params/param.rbs
|
|
@@ -532,8 +523,6 @@ files:
|
|
|
532
523
|
- sig/manual/riffer/mcp/authenticated_tool.rbs
|
|
533
524
|
- sig/manual/riffer/mcp/registry.rbs
|
|
534
525
|
- sig/manual/riffer/mcp/tool_factory.rbs
|
|
535
|
-
- sig/manual/riffer/metrics.rbs
|
|
536
|
-
- sig/manual/riffer/metrics/no_op.rbs
|
|
537
526
|
- sig/manual/riffer/providers.rbs
|
|
538
527
|
- sig/manual/riffer/providers/repository.rbs
|
|
539
528
|
- sig/manual/riffer/tracing.rbs
|
data/docs/17_METRICS.md
DELETED
|
@@ -1,155 +0,0 @@
|
|
|
1
|
-
# Metrics
|
|
2
|
-
|
|
3
|
-
Riffer can record [OpenTelemetry](https://opentelemetry.io/) metric instruments alongside its [spans](16_TRACING.md), following the [GenAI semantic conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/). Metric names, instrument types, units, and attributes are a public, versioned contract you can build dashboards and alerts against. This page is the reference for that contract.
|
|
4
|
-
|
|
5
|
-
As with tracing, Riffer only _records_ instruments, and only through a backend you assign to `config.metrics.backend` — OpenTelemetry is the built-in option you opt into (the host application owns the SDK, metric reader, exporter, and aggregation — the standard OTEL split), but never a default. With no backend assigned, every measurement is a silent no-op and Riffer carries no OpenTelemetry gem dependency.
|
|
6
|
-
|
|
7
|
-
> **OpenTelemetry metrics for Ruby is still pre-1.0.** The metrics API and SDK ship as separate, experimental gems (`opentelemetry-metrics-api`, `opentelemetry-metrics-sdk`) from the stable 1.x traces API. Riffer guards against an incompatible API and falls back to a no-op outside the supported range, but expect the host-side wiring below to evolve with those gems.
|
|
8
|
-
|
|
9
|
-
## Enabling metrics
|
|
10
|
-
|
|
11
|
-
Riffer records measurements only through a backend you assign to `config.metrics.backend` — it does **not** auto-detect OpenTelemetry. To use OTEL, add the metrics SDK, register a metric reader with an exporter, and assign Riffer's built-in OTEL backend with `Riffer::Metrics::Otel.build`.
|
|
12
|
-
|
|
13
|
-
```ruby
|
|
14
|
-
# Gemfile
|
|
15
|
-
gem "opentelemetry-metrics-sdk"
|
|
16
|
-
```
|
|
17
|
-
|
|
18
|
-
```ruby
|
|
19
|
-
require "opentelemetry-metrics-sdk"
|
|
20
|
-
|
|
21
|
-
OpenTelemetry::SDK.configure do |c|
|
|
22
|
-
c.service_name = "my-agent-host"
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
Riffer.configure do |config|
|
|
26
|
-
config.metrics.backend = Riffer::Metrics::Otel.build
|
|
27
|
-
end
|
|
28
|
-
```
|
|
29
|
-
|
|
30
|
-
The metrics SDK is **separate** from the traces SDK (`opentelemetry-sdk`); add it explicitly even if you already trace. `Riffer::Metrics::Otel.build` wraps the global `OpenTelemetry.meter_provider` by default; pass `provider:` to wrap a specific one. It returns `nil` — leaving metrics a no-op rather than raising — when the `opentelemetry-metrics-api` gem is absent or outside the supported range (>= 0.2, < 1.0). Any backend implementing the OpenTelemetry Metrics API then ingests Riffer's instruments. For real reader and exporter setup (OTLP, periodic export, Views), see the [OpenTelemetry Ruby docs](https://opentelemetry.io/docs/languages/ruby/).
|
|
31
|
-
|
|
32
|
-
The metrics knobs — the `enabled` kill switch and the `backend` itself — live in [Configuration — Metrics](10_CONFIGURATION.md#metrics). They are **independent** of the tracing knobs (each signal has its own backend): you can run tracing while metrics are off, or the reverse.
|
|
33
|
-
|
|
34
|
-
Instruments are recorded under the instrumentation scope named `riffer`, versioned with the Riffer gem version — the runtime signal for which release produced a measurement; see [Stability](#stability).
|
|
35
|
-
|
|
36
|
-
## Routing to a non-OpenTelemetry backend
|
|
37
|
-
|
|
38
|
-
OpenTelemetry is one backend, not the only one. A host that already runs another metrics stack — DogStatsD, say — can route Riffer's measurements into it with **no `opentelemetry-*` gem installed** by assigning its own backend to `config.metrics.backend` in place of `Riffer::Metrics::Otel.build`. Whatever you assign is the backend; there is no fallback and no auto-detection — an unset backend is a no-op.
|
|
39
|
-
|
|
40
|
-
```ruby
|
|
41
|
-
Riffer.configure do |config|
|
|
42
|
-
config.metrics.backend = MyDogStatsdMetricsBackend.new
|
|
43
|
-
end
|
|
44
|
-
```
|
|
45
|
-
|
|
46
|
-
The backend is duck-typed: any object that responds to `record_histogram(name, value, unit:, description:, attributes:)` works, and the setter validates only that (otherwise it raises `Riffer::ArgumentError`). `Riffer::Metrics::NoOp` is the reference shape. All four instruments are histograms, so the single `record_histogram` method covers the full contract; tell them apart by `name`. Two carry unit `s` — `gen_ai.client.operation.duration` and `riffer.guardrail.duration` — so `name` is the reliable discriminator; `{token}` is `gen_ai.client.token.usage` and `USD` is `riffer.gen_ai.cost`.
|
|
47
|
-
|
|
48
|
-
A custom backend counts as a **live** sink, so the providers still compute the token counts and cost that feed it — the same data that, under OTEL, populates `gen_ai.client.token.usage` and `riffer.gen_ai.cost`. The `enabled` kill switch is honoured ahead of the backend: with `config.metrics.enabled = false`, measurements short-circuit to the no-op without ever reaching a custom backend.
|
|
49
|
-
|
|
50
|
-
### Bucket boundaries
|
|
51
|
-
|
|
52
|
-
Histogram bucket boundaries are a **host-side** concern. The OpenTelemetry metrics API does not let an instrumenting library attach bucket boundaries at instrument creation, so Riffer does not set them — the SDK's default buckets apply unless you override them. To match the GenAI semantic conventions' recommended boundaries (or your own), register a [View](https://opentelemetry.io/docs/specs/otel/metrics/sdk/#view) on the meter provider that targets the instrument by name and sets explicit bucket boundaries.
|
|
53
|
-
|
|
54
|
-
The convention recommends boundaries scaled to each instrument, so register one View per histogram — the token-count buckets below are for `gen_ai.client.token.usage`; `gen_ai.client.operation.duration` wants its own latency-scaled set, and `riffer.gen_ai.cost` a USD-scaled one.
|
|
55
|
-
|
|
56
|
-
```ruby
|
|
57
|
-
require "opentelemetry-metrics-sdk"
|
|
58
|
-
|
|
59
|
-
OpenTelemetry::SDK.configure do |c|
|
|
60
|
-
c.service_name = "my-agent-host"
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
# The GenAI semconv's recommended token-count boundaries. Register the View
|
|
64
|
-
# before Riffer records its first measurement.
|
|
65
|
-
OpenTelemetry.meter_provider.add_view(
|
|
66
|
-
"gen_ai.client.token.usage",
|
|
67
|
-
aggregation: OpenTelemetry::SDK::Metrics::Aggregation::ExplicitBucketHistogram.new(
|
|
68
|
-
boundaries: [1, 4, 16, 64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864]
|
|
69
|
-
)
|
|
70
|
-
)
|
|
71
|
-
```
|
|
72
|
-
|
|
73
|
-
## Instruments
|
|
74
|
-
|
|
75
|
-
Each instrument is documented here as a row carrying its name, instrument type, unit, and attribute set.
|
|
76
|
-
|
|
77
|
-
> **Per-call tags become dimensions.** Any tags passed to `#generate` / `#stream` via `tags:` are added as `riffer.tag.<key>` attributes to **every** instrument below — `operation.duration` (each operation), `token.usage` (each point), `cost`, and `guardrail.duration` — so the attribute lists omit them. Tags are promoted verbatim with no allow-list — a high-cardinality value like a unique `user_id` per request multiplies time series accordingly; drop it with a [View](https://opentelemetry.io/docs/specs/otel/metrics/sdk/#view) or keep metric-bound tags low-cardinality. See [Per-Call Tags](03_AGENTS.md#per-call-tags).
|
|
78
|
-
|
|
79
|
-
### `gen_ai.client.operation.duration`
|
|
80
|
-
|
|
81
|
-
Histogram, unit `s`. The latency of a single GenAI operation, recorded around the same wrap as the matching [span](16_TRACING.md) on both the success and error paths and timed with a monotonic clock. Recording is independent of tracing — the metric fires even with `config.tracing.enabled = false`. Tell the three operations apart by `gen_ai.operation.name`.
|
|
82
|
-
|
|
83
|
-
| `gen_ai.operation.name` | Recorded around | Attributes |
|
|
84
|
-
| ----------------------- | -------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- |
|
|
85
|
-
| `chat` | each provider call (`generate_text`/`stream_text`) | `gen_ai.operation.name`, `gen_ai.provider.name`, `gen_ai.request.model` (when set), `error.type` (on error) |
|
|
86
|
-
| `invoke_agent` | each agent run (`generate`/`stream`) | `gen_ai.operation.name`, `gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.agent.name`, `error.type` (on error) |
|
|
87
|
-
| `execute_tool` | each tool call | `gen_ai.operation.name`, `gen_ai.tool.name`, `error.type` (on error) |
|
|
88
|
-
|
|
89
|
-
`error.type` carries the exception class for a raised error; for `execute_tool` it carries the handled error category (e.g. `validation_error`, `timeout_error`) when a tool returns an error result instead of raising — matching the span. `gen_ai.response.model` is not recorded yet; it will land once it is also captured on the chat span.
|
|
90
|
-
|
|
91
|
-
> **Streamed operations are consumption-paced.** A streamed `chat` or `invoke_agent` records its duration when the stream drains, so the value includes the time your consumer takes to iterate the events, not just provider latency. The matching span behaves the same way.
|
|
92
|
-
|
|
93
|
-
> **`gen_ai.tool.name` cardinality.** One time series exists per distinct tool name. With a large or dynamic tool set (for example MCP-discovered tools) that can grow unbounded — drop the attribute with a [View](https://opentelemetry.io/docs/specs/otel/metrics/sdk/#view) if your backend strains.
|
|
94
|
-
|
|
95
|
-
### `gen_ai.client.token.usage`
|
|
96
|
-
|
|
97
|
-
Histogram, unit `{token}`. Token volume for a single `chat` call, recorded from the normalized token usage after the provider responds. Each call emits **two data points** — one `input`, one `output` — distinguished by `gen_ai.token.type`. Recording is independent of tracing (it fires with `config.tracing.enabled = false`); for a streamed call it fires when the stream drains. `gen_ai.response.model` is not recorded yet, for the same reason as `operation.duration`.
|
|
98
|
-
|
|
99
|
-
| `gen_ai.token.type` | Value | Attributes |
|
|
100
|
-
| ------------------- | ----------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
|
|
101
|
-
| `input` | total prompt tokens for the call, **cache-inclusive** | `gen_ai.operation.name` (always `chat`), `gen_ai.provider.name`, `gen_ai.token.type`, `gen_ai.request.model` (when set) |
|
|
102
|
-
| `output` | tokens generated, including reasoning/thinking tokens | same |
|
|
103
|
-
|
|
104
|
-
> **Per-call only.** Token usage is never recorded at the run (`invoke_agent`) level. Metrics pre-aggregate, so emitting both the per-call points and a run total would double-count — sum the per-call points in your backend if you want a run total. This is the metric-side counterpart of the span-level [double-count trap](16_TRACING.md#token-usage-and-cost).
|
|
105
|
-
|
|
106
|
-
> **Cache buckets stay on spans.** The semconv `gen_ai.token.type` defines only `input` and `output`, so the prompt-cache subsets (`cache_read` / `cache_creation`) live on [spans](16_TRACING.md#token-usage-and-cost), not this metric. The `input` value is the cache-inclusive total, matching the span's `gen_ai.usage.input_tokens`.
|
|
107
|
-
|
|
108
|
-
A call that reports no usage records no data points, and a failed call has nothing to count — so this metric carries no `error.type` (the semconv marks it not applicable here, unlike `operation.duration`).
|
|
109
|
-
|
|
110
|
-
### `riffer.gen_ai.cost`
|
|
111
|
-
|
|
112
|
-
Histogram, unit `USD`. The cost of a single `chat` call, recorded from the [cost](16_TRACING.md#token-usage-and-cost) on the normalized token usage after the provider responds — the same source as the cost span attribute, a different sink. This instrument is Riffer-owned (`riffer.*`, not `gen_ai.*`) so it won't collide if the semantic conventions later define a cost instrument; see [Stability](#stability). Recording is independent of tracing (it fires with `config.tracing.enabled = false`); for a streamed call it fires when the stream drains.
|
|
113
|
-
|
|
114
|
-
| Value | Attributes |
|
|
115
|
-
| ---------------- | -------------------------------------------------------------------------------------------------- |
|
|
116
|
-
| cost of the call | `gen_ai.operation.name` (always `chat`), `gen_ai.provider.name`, `gen_ai.request.model` (when set) |
|
|
117
|
-
|
|
118
|
-
Pricing is **consumer-configured** — no price table ships with the gem (see [Configuration — Pricing](10_CONFIGURATION.md#pricing)). A call whose model has no configured price records **no** data point, so this metric covers only priced calls; `operation.duration` and `token.usage` still record. A priced call that computes to `0.0` does record a zero data point — only an absent price means there is nothing to measure.
|
|
119
|
-
|
|
120
|
-
> **Per-call only.** Cost is never recorded at the run (`invoke_agent`) level, for the same reason as token usage: metrics pre-aggregate, so emitting both per-call points and a run total would double-count. Sum the per-call points in your backend for a run total.
|
|
121
|
-
|
|
122
|
-
### `riffer.guardrail.duration`
|
|
123
|
-
|
|
124
|
-
Histogram, unit `s`. The latency of a single guardrail execution, recorded around the same wrap as the [`execute_guardrail` span](16_TRACING.md) on both the success and raise paths and timed with a monotonic clock. Guardrails run on the request hot path — before every model turn and after every response — so this is the guardrail counterpart to `gen_ai.client.operation.duration`. Recording is independent of tracing — the metric fires even with `config.tracing.enabled = false`.
|
|
125
|
-
|
|
126
|
-
This instrument is Riffer-owned (`riffer.*`, not `gen_ai.*`) by the same reasoning as its span: a guardrail is not a GenAI semantic-convention operation, so the `execute_guardrail` span deliberately carries no `gen_ai.operation.name` and lives in riffer's own `riffer.guardrail.*` namespace (see [Tracing](16_TRACING.md)). Folding the metric into `gen_ai.client.operation.duration` would contradict that, so the duration is its own riffer-owned histogram instead; see [Stability](#stability).
|
|
127
|
-
|
|
128
|
-
| Value | Attributes |
|
|
129
|
-
| ----------------------------------- | -------------------------------------------------------------------------- |
|
|
130
|
-
| duration of the guardrail execution | `riffer.guardrail.name`, `riffer.guardrail.phase`, `error.type` (on raise) |
|
|
131
|
-
|
|
132
|
-
`riffer.guardrail.phase` is `before` or `after`. A pass, transform, or block is a **handled** outcome and records no `error.type` — that attribute carries the exception class only when a guardrail raises, mirroring the span. One time series exists per `riffer.guardrail.name` × `riffer.guardrail.phase`; guardrail names are bounded by the guardrails you configure, so cardinality is safe — unlike the dynamic tool names on `gen_ai.client.operation.duration`.
|
|
133
|
-
|
|
134
|
-
## Stability
|
|
135
|
-
|
|
136
|
-
The instrument shape is a public, versioned contract, in two tiers — mirroring the [tracing contract](16_TRACING.md#stability):
|
|
137
|
-
|
|
138
|
-
- **`gen_ai.*`** tracks the OpenTelemetry GenAI semantic conventions, pinned to schema version `1.37.0`. That convention is still "Development" status upstream and its names may change; Riffer absorbs such renames deliberately in a release, never silently, with a CHANGELOG entry.
|
|
139
|
-
- **`riffer.*`** is Riffer-owned and changes only through a normal version bump and CHANGELOG entry. Riffer-owned metrics live here so they won't collide if semconv later defines an equivalent.
|
|
140
|
-
|
|
141
|
-
The semantic-convention schema version is a documented pin rather than an instrument attribute — the OpenTelemetry Ruby API can't attach a schema URL to a meter. The runtime version signal is the instrumentation scope: every measurement carries scope name `riffer` at the gem version that recorded it. Pin the Riffer version your dashboards depend on, and watch the CHANGELOG for metrics entries before upgrading.
|
|
142
|
-
|
|
143
|
-
## Avoid double instrumentation
|
|
144
|
-
|
|
145
|
-
Riffer records its agent loop's metrics natively. Running a provider-level GenAI instrumentation gem (for example an OpenTelemetry contrib instrumentation for the underlying Anthropic or OpenAI client) _alongside_ Riffer records the same `gen_ai.client.*` metrics twice. Because metrics pre-aggregate, that duplication is **silent** — it inflates counts and distorts distributions in your dashboards without any obvious per-event trace to inspect.
|
|
146
|
-
|
|
147
|
-
Record one or the other, not both. Since the metrics kill switch is independent of tracing, the usual resolution is to keep Riffer's spans and turn _Riffer's metrics_ off, letting the provider-level instrumentation own the metrics:
|
|
148
|
-
|
|
149
|
-
```ruby
|
|
150
|
-
Riffer.configure do |config|
|
|
151
|
-
config.metrics.enabled = false
|
|
152
|
-
end
|
|
153
|
-
```
|
|
154
|
-
|
|
155
|
-
— or disable the provider-level metric instrumentation and let Riffer own them.
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
# rbs_inline: enabled
|
|
3
|
-
|
|
4
|
-
# The catalog of metric instruments riffer records. Each handle is a constant
|
|
5
|
-
# that resolves its backend at record time, so it survives a meter-provider swap
|
|
6
|
-
# or a runtime +enabled+ flip.
|
|
7
|
-
module Riffer::Metrics::Instruments # :nodoc: all
|
|
8
|
-
OPERATION_DURATION = Riffer::Metrics.create_histogram(
|
|
9
|
-
"gen_ai.client.operation.duration",
|
|
10
|
-
unit: "s",
|
|
11
|
-
description: "Duration of GenAI client operations"
|
|
12
|
-
) #: Riffer::Metrics::Histogram
|
|
13
|
-
|
|
14
|
-
TOKEN_USAGE = Riffer::Metrics.create_histogram(
|
|
15
|
-
"gen_ai.client.token.usage",
|
|
16
|
-
unit: "{token}",
|
|
17
|
-
description: "Number of input and output tokens used in GenAI operations"
|
|
18
|
-
) #: Riffer::Metrics::Histogram
|
|
19
|
-
|
|
20
|
-
COST = Riffer::Metrics.create_histogram(
|
|
21
|
-
"riffer.gen_ai.cost",
|
|
22
|
-
unit: "USD",
|
|
23
|
-
description: "Cost of GenAI client operations in USD"
|
|
24
|
-
) #: Riffer::Metrics::Histogram
|
|
25
|
-
|
|
26
|
-
GUARDRAIL_DURATION = Riffer::Metrics.create_histogram(
|
|
27
|
-
"riffer.guardrail.duration",
|
|
28
|
-
unit: "s",
|
|
29
|
-
description: "Duration of guardrail execution"
|
|
30
|
-
) #: Riffer::Metrics::Histogram
|
|
31
|
-
end
|
data/lib/riffer/metrics/no_op.rb
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
# rbs_inline: enabled
|
|
3
|
-
|
|
4
|
-
# No-op metrics backend, used when the OpenTelemetry metrics API is unavailable
|
|
5
|
-
# or metrics are disabled.
|
|
6
|
-
module Riffer::Metrics::NoOp # :nodoc: all
|
|
7
|
-
extend self
|
|
8
|
-
|
|
9
|
-
# Ignores the measurement; there is no meter without the OTEL metrics API.
|
|
10
|
-
#--
|
|
11
|
-
#: (String, Numeric, unit: String?, description: String?, attributes: Hash[String, untyped]?) -> void
|
|
12
|
-
def record_histogram(name, value, unit:, description:, attributes:)
|
|
13
|
-
end
|
|
14
|
-
end
|
data/lib/riffer/metrics/otel.rb
DELETED
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
# rbs_inline: enabled
|
|
3
|
-
|
|
4
|
-
# OTEL-backed metrics backend. <tt>::OpenTelemetry</tt> constants appear only
|
|
5
|
-
# inside method bodies here, so the gem loads and eager-loads cleanly when the
|
|
6
|
-
# OpenTelemetry metrics API is absent.
|
|
7
|
-
class Riffer::Metrics::Otel # :nodoc: all
|
|
8
|
-
SUPPORTED_API_VERSIONS = Gem::Requirement.new(">= 0.2", "< 1.0") #: Gem::Requirement
|
|
9
|
-
|
|
10
|
-
class << self
|
|
11
|
-
# Builds a backend when the OpenTelemetry metrics API is loadable at a
|
|
12
|
-
# supported version; returns +nil+ so resolution falls back to NoOp.
|
|
13
|
-
# +provider+ defaults to the global <tt>OpenTelemetry.meter_provider</tt>.
|
|
14
|
-
#--
|
|
15
|
-
#: (?provider: untyped) -> Riffer::Metrics::Otel?
|
|
16
|
-
def build(provider: nil)
|
|
17
|
-
version = api_version
|
|
18
|
-
return nil unless version
|
|
19
|
-
|
|
20
|
-
unless supported?(version)
|
|
21
|
-
Kernel.warn "riffer: opentelemetry-metrics-api #{version} is outside the supported range (#{SUPPORTED_API_VERSIONS}); metrics are disabled"
|
|
22
|
-
return nil
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
new(provider: provider || ::OpenTelemetry.meter_provider)
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
# Whether the OpenTelemetry metrics API gem is loadable at a supported
|
|
29
|
-
# version.
|
|
30
|
-
#--
|
|
31
|
-
#: () -> bool
|
|
32
|
-
def available?
|
|
33
|
-
version = api_version
|
|
34
|
-
!version.nil? && supported?(version)
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
# Whether the given opentelemetry-metrics-api version is one riffer codes
|
|
38
|
-
# against. The gem is undeclared, so this guard is the only protection
|
|
39
|
-
# against an incompatible, still-pre-1.0 API.
|
|
40
|
-
#--
|
|
41
|
-
#: (Gem::Version) -> bool
|
|
42
|
-
def supported?(version)
|
|
43
|
-
SUPPORTED_API_VERSIONS.satisfied_by?(version)
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
private
|
|
47
|
-
|
|
48
|
-
#--
|
|
49
|
-
#: () -> Gem::Version?
|
|
50
|
-
def api_version
|
|
51
|
-
require "opentelemetry-metrics-api"
|
|
52
|
-
spec = Gem.loaded_specs["opentelemetry-metrics-api"] #: untyped
|
|
53
|
-
spec&.version
|
|
54
|
-
rescue ::LoadError
|
|
55
|
-
nil
|
|
56
|
-
end
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
# @rbs @meter: untyped
|
|
60
|
-
# @rbs @instruments: Hash[String, untyped]
|
|
61
|
-
# @rbs @mutex: Mutex
|
|
62
|
-
|
|
63
|
-
#--
|
|
64
|
-
#: (provider: untyped) -> void
|
|
65
|
-
def initialize(provider:)
|
|
66
|
-
@meter = provider.meter("riffer", version: Riffer::VERSION)
|
|
67
|
-
@instruments = {}
|
|
68
|
-
@mutex = Mutex.new
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
# Records a value onto the named histogram.
|
|
72
|
-
#--
|
|
73
|
-
#: (String, Numeric, unit: String?, description: String?, attributes: Hash[String, untyped]?) -> void
|
|
74
|
-
def record_histogram(name, value, unit:, description:, attributes:)
|
|
75
|
-
histogram = @mutex.synchronize do
|
|
76
|
-
@instruments[name] ||= @meter.create_histogram(name, unit: unit, description: description)
|
|
77
|
-
end
|
|
78
|
-
histogram.record(value, attributes: attributes)
|
|
79
|
-
end
|
|
80
|
-
end
|
data/lib/riffer/metrics.rb
DELETED
|
@@ -1,94 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
# rbs_inline: enabled
|
|
3
|
-
|
|
4
|
-
# Internal metrics port — records OTEL metric instruments when the host bundles
|
|
5
|
-
# the OpenTelemetry metrics API and no-ops otherwise, so riffer never declares
|
|
6
|
-
# an OTEL dependency.
|
|
7
|
-
module Riffer::Metrics # :nodoc: all
|
|
8
|
-
extend self
|
|
9
|
-
|
|
10
|
-
# @rbs @backend: untyped
|
|
11
|
-
|
|
12
|
-
MUTEX = Mutex.new #: Mutex
|
|
13
|
-
|
|
14
|
-
# The Ruby API cannot attach a schema URL to a meter, so the semconv pin
|
|
15
|
-
# lives here as the documented contract version.
|
|
16
|
-
SCHEMA_URL = "https://opentelemetry.io/schemas/1.37.0" #: String
|
|
17
|
-
|
|
18
|
-
# A handle to a named histogram, safe to hold as a constant: it defers backend
|
|
19
|
-
# resolution to record time, so it survives a meter-provider swap or a runtime
|
|
20
|
-
# +enabled+ flip.
|
|
21
|
-
class Histogram
|
|
22
|
-
# @rbs @name: String
|
|
23
|
-
# @rbs @unit: String?
|
|
24
|
-
# @rbs @description: String?
|
|
25
|
-
|
|
26
|
-
#--
|
|
27
|
-
#: (String, ?unit: String?, ?description: String?) -> void
|
|
28
|
-
def initialize(name, unit: nil, description: nil)
|
|
29
|
-
@name = name
|
|
30
|
-
@unit = unit
|
|
31
|
-
@description = description
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
#--
|
|
35
|
-
#: (Numeric, ?attributes: Hash[String, untyped]?) -> void
|
|
36
|
-
def record(value, attributes: nil)
|
|
37
|
-
Riffer::Metrics.record_histogram(@name, value, unit: @unit, description: @description, attributes: attributes)
|
|
38
|
-
end
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
# Returns a handle to the named histogram.
|
|
42
|
-
#--
|
|
43
|
-
#: (String, ?unit: String?, ?description: String?) -> Riffer::Metrics::Histogram
|
|
44
|
-
def create_histogram(name, unit: nil, description: nil)
|
|
45
|
-
Histogram.new(name, unit: unit, description: description)
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
# Records a value onto the named histogram.
|
|
49
|
-
#--
|
|
50
|
-
#: (String, Numeric, ?unit: String?, ?description: String?, ?attributes: Hash[String, untyped]?) -> void
|
|
51
|
-
def record_histogram(name, value, unit: nil, description: nil, attributes: nil)
|
|
52
|
-
return unless Riffer.config.metrics.enabled
|
|
53
|
-
backend.record_histogram(name, value, unit: unit, description: description, attributes: attributes)
|
|
54
|
-
end
|
|
55
|
-
|
|
56
|
-
# Mirrors a span's +recording?+ so a caller can skip work that exists only to
|
|
57
|
-
# feed a metric. True for any live backend — the OTEL backend or a
|
|
58
|
-
# consumer-supplied one — so a custom metrics sink still gets fed.
|
|
59
|
-
#--
|
|
60
|
-
#: () -> bool
|
|
61
|
-
def recording?
|
|
62
|
-
Riffer.config.metrics.enabled && !backend.equal?(NoOp)
|
|
63
|
-
end
|
|
64
|
-
|
|
65
|
-
# Reads the monotonic clock in seconds — the time source for duration metrics,
|
|
66
|
-
# immune to wall-clock adjustments.
|
|
67
|
-
#--
|
|
68
|
-
#: () -> Float
|
|
69
|
-
def monotonic_now
|
|
70
|
-
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
71
|
-
end
|
|
72
|
-
|
|
73
|
-
# Discards the resolved backend so the next record re-resolves it; cached
|
|
74
|
-
# instruments live on that backend, so this clears them too.
|
|
75
|
-
#--
|
|
76
|
-
#: () -> void
|
|
77
|
-
def reset!
|
|
78
|
-
MUTEX.synchronize { @backend = nil }
|
|
79
|
-
end
|
|
80
|
-
|
|
81
|
-
private
|
|
82
|
-
|
|
83
|
-
#--
|
|
84
|
-
#: () -> untyped
|
|
85
|
-
def backend
|
|
86
|
-
@backend || MUTEX.synchronize { @backend ||= resolve_backend }
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
#--
|
|
90
|
-
#: () -> untyped
|
|
91
|
-
def resolve_backend
|
|
92
|
-
Riffer.config.metrics.backend || NoOp
|
|
93
|
-
end
|
|
94
|
-
end
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
# Generated from lib/riffer/metrics/instruments.rb with RBS::Inline
|
|
2
|
-
|
|
3
|
-
# The catalog of metric instruments riffer records. Each handle is a constant
|
|
4
|
-
# that resolves its backend at record time, so it survives a meter-provider swap
|
|
5
|
-
# or a runtime +enabled+ flip.
|
|
6
|
-
module Riffer::Metrics::Instruments
|
|
7
|
-
# :nodoc: all
|
|
8
|
-
OPERATION_DURATION: Riffer::Metrics::Histogram
|
|
9
|
-
|
|
10
|
-
TOKEN_USAGE: Riffer::Metrics::Histogram
|
|
11
|
-
|
|
12
|
-
COST: Riffer::Metrics::Histogram
|
|
13
|
-
|
|
14
|
-
GUARDRAIL_DURATION: Riffer::Metrics::Histogram
|
|
15
|
-
end
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
# Generated from lib/riffer/metrics/no_op.rb with RBS::Inline
|
|
2
|
-
|
|
3
|
-
# No-op metrics backend, used when the OpenTelemetry metrics API is unavailable
|
|
4
|
-
# or metrics are disabled.
|
|
5
|
-
module Riffer::Metrics::NoOp
|
|
6
|
-
# Ignores the measurement; there is no meter without the OTEL metrics API.
|
|
7
|
-
# --
|
|
8
|
-
# : (String, Numeric, unit: String?, description: String?, attributes: Hash[String, untyped]?) -> void
|
|
9
|
-
def record_histogram: (String, Numeric, unit: String?, description: String?, attributes: Hash[String, untyped]?) -> void
|
|
10
|
-
end
|
|
@@ -1,48 +0,0 @@
|
|
|
1
|
-
# Generated from lib/riffer/metrics/otel.rb with RBS::Inline
|
|
2
|
-
|
|
3
|
-
# OTEL-backed metrics backend. <tt>::OpenTelemetry</tt> constants appear only
|
|
4
|
-
# inside method bodies here, so the gem loads and eager-loads cleanly when the
|
|
5
|
-
# OpenTelemetry metrics API is absent.
|
|
6
|
-
class Riffer::Metrics::Otel
|
|
7
|
-
# :nodoc: all
|
|
8
|
-
SUPPORTED_API_VERSIONS: Gem::Requirement
|
|
9
|
-
|
|
10
|
-
# Builds a backend when the OpenTelemetry metrics API is loadable at a
|
|
11
|
-
# supported version; returns +nil+ so resolution falls back to NoOp.
|
|
12
|
-
# +provider+ defaults to the global <tt>OpenTelemetry.meter_provider</tt>.
|
|
13
|
-
# --
|
|
14
|
-
# : (?provider: untyped) -> Riffer::Metrics::Otel?
|
|
15
|
-
def self.build: (?provider: untyped) -> Riffer::Metrics::Otel?
|
|
16
|
-
|
|
17
|
-
# Whether the OpenTelemetry metrics API gem is loadable at a supported
|
|
18
|
-
# version.
|
|
19
|
-
# --
|
|
20
|
-
# : () -> bool
|
|
21
|
-
def self.available?: () -> bool
|
|
22
|
-
|
|
23
|
-
# Whether the given opentelemetry-metrics-api version is one riffer codes
|
|
24
|
-
# against. The gem is undeclared, so this guard is the only protection
|
|
25
|
-
# against an incompatible, still-pre-1.0 API.
|
|
26
|
-
# --
|
|
27
|
-
# : (Gem::Version) -> bool
|
|
28
|
-
def self.supported?: (Gem::Version) -> bool
|
|
29
|
-
|
|
30
|
-
# --
|
|
31
|
-
# : () -> Gem::Version?
|
|
32
|
-
private def self.api_version: () -> Gem::Version?
|
|
33
|
-
|
|
34
|
-
@meter: untyped
|
|
35
|
-
|
|
36
|
-
@instruments: Hash[String, untyped]
|
|
37
|
-
|
|
38
|
-
@mutex: Mutex
|
|
39
|
-
|
|
40
|
-
# --
|
|
41
|
-
# : (provider: untyped) -> void
|
|
42
|
-
def initialize: (provider: untyped) -> void
|
|
43
|
-
|
|
44
|
-
# Records a value onto the named histogram.
|
|
45
|
-
# --
|
|
46
|
-
# : (String, Numeric, unit: String?, description: String?, attributes: Hash[String, untyped]?) -> void
|
|
47
|
-
def record_histogram: (String, Numeric, unit: String?, description: String?, attributes: Hash[String, untyped]?) -> void
|
|
48
|
-
end
|