riffer 0.47.2 → 0.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.claude/rules/comments.md +2 -4
- data/.claude/rules/rbs-inline.md +2 -12
- data/.release-please-manifest.json +1 -1
- data/.rubocop.yml +5 -0
- data/CHANGELOG.md +42 -0
- data/docs/AGENTS.md +39 -7
- data/docs/AGENT_LIFECYCLE.md +14 -16
- data/docs/CONFIGURATION.md +23 -34
- data/docs/EVALS.md +2 -1
- data/docs/MCP.md +0 -4
- data/docs/MESSAGES.md +85 -17
- data/docs/STREAM_EVENTS.md +8 -4
- data/docs/TOOL_ADVANCED.md +1 -3
- data/docs/TRACING.md +2 -2
- data/docs/providers/AMAZON_BEDROCK.md +32 -1
- data/docs/providers/CUSTOM_PROVIDERS.md +56 -4
- data/docs/providers/GEMINI.md +1 -1
- data/docs/providers/MOCK_PROVIDER.md +17 -0
- data/docs/providers/OPENROUTER.md +18 -1
- data/docs-site/build.rb +0 -6
- data/docs-site/check.rb +1 -5
- data/lib/riffer/agent/config.rb +27 -44
- data/lib/riffer/agent/context.rb +2 -26
- data/lib/riffer/agent/outcome.rb +0 -20
- data/lib/riffer/agent/response.rb +4 -35
- data/lib/riffer/agent/run.rb +32 -48
- data/lib/riffer/agent/serializer.rb +10 -39
- data/lib/riffer/agent/session/repair.rb +3 -15
- data/lib/riffer/agent/session.rb +26 -38
- data/lib/riffer/agent/structured_output/result.rb +0 -8
- data/lib/riffer/agent/structured_output.rb +0 -7
- data/lib/riffer/agent.rb +31 -127
- data/lib/riffer/config/amazon_bedrock.rb +30 -0
- data/lib/riffer/config/anthropic.rb +21 -0
- data/lib/riffer/config/azure_open_ai.rb +30 -0
- data/lib/riffer/config/evals.rb +18 -0
- data/lib/riffer/config/files.rb +61 -0
- data/lib/riffer/config/gemini.rb +21 -0
- data/lib/riffer/config/mcp.rb +31 -0
- data/lib/riffer/config/open_ai.rb +30 -0
- data/lib/riffer/config/open_router.rb +21 -0
- data/lib/riffer/config/pricing/rates.rb +36 -0
- data/lib/riffer/config/pricing.rb +64 -0
- data/lib/riffer/config/skills.rb +38 -0
- data/lib/riffer/config/tracing.rb +43 -0
- data/lib/riffer/config.rb +3 -362
- data/lib/riffer/evals/evaluator.rb +22 -23
- data/lib/riffer/evals/evaluator_runner.rb +0 -15
- data/lib/riffer/evals/judge.rb +9 -11
- data/lib/riffer/evals/result.rb +1 -11
- data/lib/riffer/evals/run_result.rb +0 -12
- data/lib/riffer/evals/scenario_result.rb +0 -19
- data/lib/riffer/files/downloader.rb +2 -3
- data/lib/riffer/files/resolver.rb +5 -10
- data/lib/riffer/guardrail.rb +2 -22
- data/lib/riffer/guardrails/modification.rb +0 -8
- data/lib/riffer/guardrails/result.rb +0 -17
- data/lib/riffer/guardrails/runner.rb +2 -11
- data/lib/riffer/guardrails/tripwire.rb +0 -8
- data/lib/riffer/guardrails.rb +0 -2
- data/lib/riffer/helpers/boolean.rb +0 -4
- data/lib/riffer/helpers/call_or_value.rb +0 -3
- data/lib/riffer/helpers/deep_dup.rb +37 -0
- data/lib/riffer/helpers/dependencies.rb +0 -4
- data/lib/riffer/helpers/identifier.rb +3 -12
- data/lib/riffer/helpers/validate.rb +42 -0
- data/lib/riffer/mcp/authenticated_tool.rb +4 -12
- data/lib/riffer/mcp/client.rb +0 -7
- data/lib/riffer/mcp/manifest.rb +3 -7
- data/lib/riffer/mcp/registration.rb +0 -10
- data/lib/riffer/mcp/registry.rb +0 -9
- data/lib/riffer/mcp/search_tool.rb +0 -4
- data/lib/riffer/mcp/tool.rb +1 -4
- data/lib/riffer/mcp/tool_factory.rb +3 -6
- data/lib/riffer/mcp.rb +2 -23
- data/lib/riffer/messages/assistant/reasoning_part.rb +73 -0
- data/lib/riffer/messages/assistant/tool_call.rb +55 -0
- data/lib/riffer/messages/assistant.rb +43 -15
- data/lib/riffer/messages/base.rb +5 -33
- data/lib/riffer/messages/system.rb +8 -1
- data/lib/riffer/messages/tool.rb +15 -13
- data/lib/riffer/messages/{file_part.rb → user/file_part.rb} +16 -40
- data/lib/riffer/messages/user.rb +11 -4
- data/lib/riffer/params/boolean.rb +1 -5
- data/lib/riffer/params/param.rb +15 -31
- data/lib/riffer/params.rb +15 -39
- data/lib/riffer/providers/amazon_bedrock.rb +104 -44
- data/lib/riffer/providers/anthropic.rb +11 -26
- data/lib/riffer/providers/azure_open_ai.rb +3 -8
- data/lib/riffer/providers/base.rb +30 -32
- data/lib/riffer/providers/finish_reason.rb +0 -6
- data/lib/riffer/providers/gemini/client.rb +4 -17
- data/lib/riffer/providers/gemini.rb +6 -12
- data/lib/riffer/providers/mock.rb +18 -27
- data/lib/riffer/providers/open_ai.rb +12 -21
- data/lib/riffer/providers/open_router.rb +109 -33
- data/lib/riffer/providers/repository.rb +2 -14
- data/lib/riffer/providers/token_usage.rb +19 -12
- data/lib/riffer/registrable.rb +11 -45
- data/lib/riffer/runner/fibers.rb +1 -6
- data/lib/riffer/runner/sequential.rb +0 -1
- data/lib/riffer/runner/threaded.rb +0 -3
- data/lib/riffer/runner.rb +0 -3
- data/lib/riffer/skills/activate_tool.rb +0 -3
- data/lib/riffer/skills/adapter.rb +0 -8
- data/lib/riffer/skills/backend.rb +2 -7
- data/lib/riffer/skills/config.rb +13 -17
- data/lib/riffer/skills/context.rb +0 -29
- data/lib/riffer/skills/filesystem_backend.rb +0 -7
- data/lib/riffer/skills/frontmatter.rb +2 -16
- data/lib/riffer/skills/markdown_adapter.rb +3 -6
- data/lib/riffer/skills/xml_adapter.rb +0 -3
- data/lib/riffer/stream_events/base.rb +0 -3
- data/lib/riffer/stream_events/finish_reason_done.rb +1 -6
- data/lib/riffer/stream_events/guardrail_modification.rb +0 -10
- data/lib/riffer/stream_events/guardrail_tripwire.rb +0 -10
- data/lib/riffer/stream_events/interrupt.rb +2 -12
- data/lib/riffer/stream_events/reasoning_delta.rb +0 -3
- data/lib/riffer/stream_events/reasoning_done.rb +6 -8
- data/lib/riffer/stream_events/skill_activation.rb +0 -3
- data/lib/riffer/stream_events/text_delta.rb +0 -2
- data/lib/riffer/stream_events/text_done.rb +0 -2
- data/lib/riffer/stream_events/token_usage_done.rb +0 -2
- data/lib/riffer/stream_events/tool_call_delta.rb +1 -5
- data/lib/riffer/stream_events/tool_call_done.rb +0 -5
- data/lib/riffer/stream_events/web_search_done.rb +0 -3
- data/lib/riffer/stream_events/web_search_status.rb +1 -5
- data/lib/riffer/testing/minitest.rb +4 -5
- data/lib/riffer/testing.rb +5 -38
- data/lib/riffer/tool.rb +2 -28
- data/lib/riffer/tools/response.rb +3 -32
- data/lib/riffer/tools/runtime/fibers.rb +0 -6
- data/lib/riffer/tools/runtime/inline.rb +0 -1
- data/lib/riffer/tools/runtime/threaded.rb +0 -6
- data/lib/riffer/tools/runtime.rb +8 -30
- data/lib/riffer/tools/toolable.rb +0 -37
- data/lib/riffer/tracing/capture.rb +5 -9
- data/lib/riffer/tracing/no_op.rb +0 -7
- data/lib/riffer/tracing/otel.rb +7 -16
- data/lib/riffer/tracing/stream_recorder.rb +0 -7
- data/lib/riffer/tracing.rb +4 -27
- data/lib/riffer/version.rb +1 -1
- data/lib/riffer.rb +2 -30
- data/sig/generated/riffer/agent/config.rbs +24 -46
- data/sig/generated/riffer/agent/context.rbs +2 -26
- data/sig/generated/riffer/agent/outcome.rbs +0 -20
- data/sig/generated/riffer/agent/response.rbs +4 -27
- data/sig/generated/riffer/agent/run.rbs +12 -31
- data/sig/generated/riffer/agent/serializer.rbs +2 -27
- data/sig/generated/riffer/agent/session/repair.rbs +2 -10
- data/sig/generated/riffer/agent/session.rbs +10 -36
- data/sig/generated/riffer/agent/structured_output/result.rbs +0 -7
- data/sig/generated/riffer/agent/structured_output.rbs +0 -7
- data/sig/generated/riffer/agent.rbs +25 -125
- data/sig/generated/riffer/config/amazon_bedrock.rbs +21 -0
- data/sig/generated/riffer/config/anthropic.rbs +15 -0
- data/sig/generated/riffer/config/azure_open_ai.rbs +21 -0
- data/sig/generated/riffer/config/evals.rbs +13 -0
- data/sig/generated/riffer/config/files.rbs +43 -0
- data/sig/generated/riffer/config/gemini.rbs +15 -0
- data/sig/generated/riffer/config/mcp.rbs +19 -0
- data/sig/generated/riffer/config/open_ai.rbs +21 -0
- data/sig/generated/riffer/config/open_router.rbs +15 -0
- data/sig/generated/riffer/config/pricing/rates.rbs +19 -0
- data/sig/generated/riffer/config/pricing.rbs +31 -0
- data/sig/generated/riffer/config/skills.rbs +19 -0
- data/sig/generated/riffer/config/tracing.rbs +25 -0
- data/sig/generated/riffer/config.rbs +2 -307
- data/sig/generated/riffer/evals/evaluator.rbs +12 -21
- data/sig/generated/riffer/evals/evaluator_runner.rbs +0 -14
- data/sig/generated/riffer/evals/judge.rbs +4 -8
- data/sig/generated/riffer/evals/result.rbs +1 -11
- data/sig/generated/riffer/evals/run_result.rbs +0 -12
- data/sig/generated/riffer/evals/scenario_result.rbs +0 -14
- data/sig/generated/riffer/files/resolver.rbs +6 -13
- data/sig/generated/riffer/guardrail.rbs +2 -22
- data/sig/generated/riffer/guardrails/modification.rbs +0 -6
- data/sig/generated/riffer/guardrails/result.rbs +0 -15
- data/sig/generated/riffer/guardrails/runner.rbs +0 -11
- data/sig/generated/riffer/guardrails/tripwire.rbs +0 -8
- data/sig/generated/riffer/guardrails.rbs +0 -2
- data/sig/generated/riffer/helpers/boolean.rbs +0 -4
- data/sig/generated/riffer/helpers/call_or_value.rbs +0 -3
- data/sig/generated/riffer/helpers/deep_dup.rbs +13 -0
- data/sig/generated/riffer/helpers/dependencies.rbs +0 -4
- data/sig/generated/riffer/helpers/identifier.rbs +0 -12
- data/sig/generated/riffer/helpers/validate.rbs +21 -0
- data/sig/generated/riffer/mcp/authenticated_tool.rbs +0 -5
- data/sig/generated/riffer/mcp/client.rbs +0 -7
- data/sig/generated/riffer/mcp/manifest.rbs +3 -7
- data/sig/generated/riffer/mcp/registration.rbs +0 -10
- data/sig/generated/riffer/mcp/registry.rbs +0 -9
- data/sig/generated/riffer/mcp/search_tool.rbs +0 -4
- data/sig/generated/riffer/mcp/tool.rbs +0 -4
- data/sig/generated/riffer/mcp/tool_factory.rbs +0 -6
- data/sig/generated/riffer/mcp.rbs +2 -22
- data/sig/generated/riffer/messages/assistant/reasoning_part.rbs +45 -0
- data/sig/generated/riffer/messages/assistant/tool_call.rbs +34 -0
- data/sig/generated/riffer/messages/assistant.rbs +25 -31
- data/sig/generated/riffer/messages/base.rbs +0 -12
- data/sig/generated/riffer/messages/system.rbs +4 -1
- data/sig/generated/riffer/messages/tool.rbs +4 -10
- data/sig/generated/riffer/messages/user/file_part.rbs +76 -0
- data/sig/generated/riffer/messages/user.rbs +7 -5
- data/sig/generated/riffer/params/boolean.rbs +1 -4
- data/sig/generated/riffer/params/param.rbs +7 -31
- data/sig/generated/riffer/params.rbs +8 -38
- data/sig/generated/riffer/providers/amazon_bedrock.rbs +33 -29
- data/sig/generated/riffer/providers/anthropic.rbs +2 -12
- data/sig/generated/riffer/providers/azure_open_ai.rbs +0 -8
- data/sig/generated/riffer/providers/base.rbs +18 -34
- data/sig/generated/riffer/providers/finish_reason.rbs +0 -6
- data/sig/generated/riffer/providers/gemini/client.rbs +4 -17
- data/sig/generated/riffer/providers/gemini.rbs +4 -10
- data/sig/generated/riffer/providers/mock.rbs +6 -27
- data/sig/generated/riffer/providers/open_ai.rbs +6 -13
- data/sig/generated/riffer/providers/open_router.rbs +37 -18
- data/sig/generated/riffer/providers/repository.rbs +2 -14
- data/sig/generated/riffer/providers/token_usage.rbs +9 -12
- data/sig/generated/riffer/registrable.rbs +0 -45
- data/sig/generated/riffer/runner/fibers.rbs +0 -6
- data/sig/generated/riffer/runner/sequential.rbs +0 -1
- data/sig/generated/riffer/runner/threaded.rbs +0 -3
- data/sig/generated/riffer/runner.rbs +0 -3
- data/sig/generated/riffer/skills/activate_tool.rbs +0 -3
- data/sig/generated/riffer/skills/adapter.rbs +0 -8
- data/sig/generated/riffer/skills/backend.rbs +2 -7
- data/sig/generated/riffer/skills/config.rbs +6 -17
- data/sig/generated/riffer/skills/context.rbs +0 -29
- data/sig/generated/riffer/skills/filesystem_backend.rbs +0 -7
- data/sig/generated/riffer/skills/frontmatter.rbs +2 -16
- data/sig/generated/riffer/skills/markdown_adapter.rbs +0 -6
- data/sig/generated/riffer/skills/xml_adapter.rbs +0 -3
- data/sig/generated/riffer/stream_events/base.rbs +0 -3
- data/sig/generated/riffer/stream_events/finish_reason_done.rbs +1 -6
- data/sig/generated/riffer/stream_events/guardrail_modification.rbs +0 -10
- data/sig/generated/riffer/stream_events/guardrail_tripwire.rbs +0 -10
- data/sig/generated/riffer/stream_events/interrupt.rbs +2 -10
- data/sig/generated/riffer/stream_events/reasoning_delta.rbs +0 -3
- data/sig/generated/riffer/stream_events/reasoning_done.rbs +4 -6
- data/sig/generated/riffer/stream_events/skill_activation.rbs +0 -3
- data/sig/generated/riffer/stream_events/text_delta.rbs +0 -2
- data/sig/generated/riffer/stream_events/text_done.rbs +0 -2
- data/sig/generated/riffer/stream_events/token_usage_done.rbs +0 -2
- data/sig/generated/riffer/stream_events/tool_call_delta.rbs +1 -5
- data/sig/generated/riffer/stream_events/tool_call_done.rbs +0 -5
- data/sig/generated/riffer/stream_events/web_search_done.rbs +0 -3
- data/sig/generated/riffer/stream_events/web_search_status.rbs +1 -5
- data/sig/generated/riffer/testing.rbs +0 -38
- data/sig/generated/riffer/tool.rbs +0 -27
- data/sig/generated/riffer/tools/response.rbs +3 -29
- data/sig/generated/riffer/tools/runtime/fibers.rbs +0 -5
- data/sig/generated/riffer/tools/runtime/inline.rbs +0 -1
- data/sig/generated/riffer/tools/runtime/threaded.rbs +0 -5
- data/sig/generated/riffer/tools/runtime.rbs +4 -26
- data/sig/generated/riffer/tools/toolable.rbs +0 -37
- data/sig/generated/riffer/tracing/capture.rbs +6 -9
- data/sig/generated/riffer/tracing/no_op.rbs +0 -7
- data/sig/generated/riffer/tracing/otel.rbs +7 -16
- data/sig/generated/riffer/tracing/stream_recorder.rbs +0 -2
- data/sig/generated/riffer/tracing.rbs +4 -23
- data/sig/generated/riffer.rbs +2 -29
- data/sig/manual/riffer/helpers/deep_dup.rbs +5 -0
- data/sig/manual/riffer/helpers/validate.rbs +5 -0
- metadata +39 -3
- data/sig/generated/riffer/messages/file_part.rbs +0 -101
|
@@ -1,351 +1,46 @@
|
|
|
1
1
|
# Generated from lib/riffer/config.rb with RBS::Inline
|
|
2
2
|
|
|
3
|
-
# Configuration for the Riffer framework.
|
|
4
3
|
class Riffer::Config
|
|
5
|
-
class AmazonBedrock < Struct[untyped]
|
|
6
|
-
attr_accessor api_token(): untyped
|
|
7
|
-
|
|
8
|
-
attr_accessor region(): untyped
|
|
9
|
-
|
|
10
|
-
attr_accessor client(): untyped
|
|
11
|
-
|
|
12
|
-
def self.new: (?untyped api_token, ?untyped region, ?untyped client) -> instance
|
|
13
|
-
| (?api_token: untyped, ?region: untyped, ?client: untyped) -> instance
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
class Anthropic < Struct[untyped]
|
|
17
|
-
attr_accessor api_key(): untyped
|
|
18
|
-
|
|
19
|
-
attr_accessor client(): untyped
|
|
20
|
-
|
|
21
|
-
def self.new: (?untyped api_key, ?untyped client) -> instance
|
|
22
|
-
| (?api_key: untyped, ?client: untyped) -> instance
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
class AzureOpenAI < Struct[untyped]
|
|
26
|
-
attr_accessor api_key(): untyped
|
|
27
|
-
|
|
28
|
-
attr_accessor endpoint(): untyped
|
|
29
|
-
|
|
30
|
-
attr_accessor client(): untyped
|
|
31
|
-
|
|
32
|
-
def self.new: (?untyped api_key, ?untyped endpoint, ?untyped client) -> instance
|
|
33
|
-
| (?api_key: untyped, ?endpoint: untyped, ?client: untyped) -> instance
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
class Gemini < Struct[untyped]
|
|
37
|
-
attr_accessor api_key(): untyped
|
|
38
|
-
|
|
39
|
-
attr_accessor client(): untyped
|
|
40
|
-
|
|
41
|
-
def self.new: (?untyped api_key, ?untyped client) -> instance
|
|
42
|
-
| (?api_key: untyped, ?client: untyped) -> instance
|
|
43
|
-
end
|
|
44
|
-
|
|
45
|
-
class OpenAI < Struct[untyped]
|
|
46
|
-
attr_accessor api_key(): untyped
|
|
47
|
-
|
|
48
|
-
attr_accessor base_url(): untyped
|
|
49
|
-
|
|
50
|
-
attr_accessor client(): untyped
|
|
51
|
-
|
|
52
|
-
def self.new: (?untyped api_key, ?untyped base_url, ?untyped client) -> instance
|
|
53
|
-
| (?api_key: untyped, ?base_url: untyped, ?client: untyped) -> instance
|
|
54
|
-
end
|
|
55
|
-
|
|
56
|
-
class OpenRouter < Struct[untyped]
|
|
57
|
-
attr_accessor api_key(): untyped
|
|
58
|
-
|
|
59
|
-
attr_accessor client(): untyped
|
|
60
|
-
|
|
61
|
-
def self.new: (?untyped api_key, ?untyped client) -> instance
|
|
62
|
-
| (?api_key: untyped, ?client: untyped) -> instance
|
|
63
|
-
end
|
|
64
|
-
|
|
65
|
-
class Evals < Struct[untyped]
|
|
66
|
-
attr_accessor judge_model(): untyped
|
|
67
|
-
|
|
68
|
-
def self.new: (?untyped judge_model) -> instance
|
|
69
|
-
| (?judge_model: untyped) -> instance
|
|
70
|
-
end
|
|
71
|
-
|
|
72
|
-
class Mcp < Struct[untyped]
|
|
73
|
-
attr_accessor credentials(): untyped
|
|
74
|
-
|
|
75
|
-
attr_accessor discovery_runner(): untyped
|
|
76
|
-
|
|
77
|
-
def self.new: (?untyped credentials, ?untyped discovery_runner) -> instance
|
|
78
|
-
| (?credentials: untyped, ?discovery_runner: untyped) -> instance
|
|
79
|
-
end
|
|
80
|
-
|
|
81
|
-
# Skills-related global configuration.
|
|
82
|
-
class Skills
|
|
83
|
-
# The tool class the LLM calls to activate a skill; defaults to
|
|
84
|
-
# <tt>Riffer::Skills::ActivateTool</tt>.
|
|
85
|
-
attr_reader default_activate_tool: singleton(Riffer::Tool)
|
|
86
|
-
|
|
87
|
-
# Default skills backend for agents that declare a +skills+ block without
|
|
88
|
-
# one; defaults to +nil+.
|
|
89
|
-
attr_reader default_backend: (Riffer::Skills::Backend | Proc)?
|
|
90
|
-
|
|
91
|
-
# --
|
|
92
|
-
# : () -> void
|
|
93
|
-
def initialize: () -> void
|
|
94
|
-
|
|
95
|
-
# Sets the default skill activation tool class. Raises Riffer::ArgumentError
|
|
96
|
-
# on an invalid value.
|
|
97
|
-
# --
|
|
98
|
-
# : (singleton(Riffer::Tool)) -> void
|
|
99
|
-
def default_activate_tool=: (singleton(Riffer::Tool)) -> void
|
|
100
|
-
|
|
101
|
-
# Sets the default skills backend. Raises Riffer::ArgumentError on an
|
|
102
|
-
# invalid value.
|
|
103
|
-
# --
|
|
104
|
-
# : ((Riffer::Skills::Backend | Proc)?) -> void
|
|
105
|
-
def default_backend=: ((Riffer::Skills::Backend | Proc)?) -> void
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
# Tracing-related global configuration.
|
|
109
|
-
class Tracing
|
|
110
|
-
# Whether riffer emits OTEL spans; defaults to +true+, a no-op until a
|
|
111
|
-
# host wires an OTEL SDK.
|
|
112
|
-
attr_reader enabled: bool
|
|
113
|
-
|
|
114
|
-
# Whether LLM-call spans capture full message content
|
|
115
|
-
# (<tt>gen_ai.input.messages</tt>, <tt>gen_ai.output.messages</tt>,
|
|
116
|
-
# <tt>gen_ai.system_instructions</tt>); defaults to +false+ — message
|
|
117
|
-
# content routinely carries sensitive data.
|
|
118
|
-
attr_reader capture_messages: bool
|
|
119
|
-
|
|
120
|
-
# The backend riffer routes spans through; defaults to +nil+, a no-op.
|
|
121
|
-
# Riffer auto-detects no backend; assigning one is opt-in.
|
|
122
|
-
attr_reader backend: untyped
|
|
123
|
-
|
|
124
|
-
# --
|
|
125
|
-
# : () -> void
|
|
126
|
-
def initialize: () -> void
|
|
127
|
-
|
|
128
|
-
# Sets the enabled flag, coercing boolean-ish values so an env-var
|
|
129
|
-
# +"false"+ (truthy in Ruby) doesn't silently keep tracing on. Raises
|
|
130
|
-
# Riffer::ArgumentError on an unrecognized value.
|
|
131
|
-
# --
|
|
132
|
-
# : (untyped) -> void
|
|
133
|
-
def enabled=: (untyped) -> void
|
|
134
|
-
|
|
135
|
-
# Sets the capture_messages flag, coercing boolean-ish values so an
|
|
136
|
-
# env-var +"false"+ (truthy in Ruby) doesn't silently enable content
|
|
137
|
-
# capture. Raises Riffer::ArgumentError on an unrecognized value.
|
|
138
|
-
# --
|
|
139
|
-
# : (untyped) -> void
|
|
140
|
-
def capture_messages=: (untyped) -> void
|
|
141
|
-
|
|
142
|
-
# Sets the tracing backend riffer routes spans through. Raises
|
|
143
|
-
# Riffer::ArgumentError unless the value is +nil+ or responds to the full
|
|
144
|
-
# delegated contract: +in_span+, +current_context+, and +with_context+.
|
|
145
|
-
# --
|
|
146
|
-
# : (untyped) -> void
|
|
147
|
-
def backend=: (untyped) -> void
|
|
148
|
-
end
|
|
149
|
-
|
|
150
|
-
# File-attachment-download policy for +Riffer::Messages::FilePart+ URL sources
|
|
151
|
-
class Files
|
|
152
|
-
# Allow file attachments to be downloaded to send to providers.
|
|
153
|
-
attr_reader allow_downloads: bool
|
|
154
|
-
|
|
155
|
-
# Maximum file size to download before failing.
|
|
156
|
-
attr_reader max_bytes: Integer
|
|
157
|
-
|
|
158
|
-
# Maximum amount of time to spend downloading a file before failing.
|
|
159
|
-
attr_reader timeout: Integer
|
|
160
|
-
|
|
161
|
-
# Maximum number of files to include in an individual message.
|
|
162
|
-
attr_reader max_per_message: Integer?
|
|
163
|
-
|
|
164
|
-
# Execution pattern for downloading files.
|
|
165
|
-
attr_reader runner: Riffer::Runner
|
|
166
|
-
|
|
167
|
-
# The object used to fetch a URL source's bytes
|
|
168
|
-
attr_reader downloader: untyped
|
|
169
|
-
|
|
170
|
-
# --
|
|
171
|
-
# : () -> void
|
|
172
|
-
def initialize: () -> void
|
|
173
|
-
|
|
174
|
-
# Sets the allow_downloads flag, coercing boolean-ish values so an env-var
|
|
175
|
-
# +"false"+ (truthy in Ruby) doesn't silently enable downloads. Raises
|
|
176
|
-
# Riffer::ArgumentError on an unrecognized value.
|
|
177
|
-
# --
|
|
178
|
-
# : (untyped) -> void
|
|
179
|
-
def allow_downloads=: (untyped) -> void
|
|
180
|
-
|
|
181
|
-
# Sets max_bytes, provided value is a positive integer.
|
|
182
|
-
# Raises Riffer::ArgumentError if value is not an Integer or less than or equal to 0.
|
|
183
|
-
# --
|
|
184
|
-
# : (untyped) -> void
|
|
185
|
-
def max_bytes=: (untyped) -> void
|
|
186
|
-
|
|
187
|
-
# Sets timeout, provided value is a positive integer.
|
|
188
|
-
# Raises Riffer::ArgumentError if value is not an Integer or is less than or equal to 0.
|
|
189
|
-
# --
|
|
190
|
-
# : (untyped) -> void
|
|
191
|
-
def timeout=: (untyped) -> void
|
|
192
|
-
|
|
193
|
-
# Sets max_per_message, provided value is either nil or a positive integer.
|
|
194
|
-
# Raises Riffer::ArgumentError if value is not an Integer or nil, or is less than or equal to 0.
|
|
195
|
-
# --
|
|
196
|
-
# : (untyped) -> void
|
|
197
|
-
def max_per_message=: (untyped) -> void
|
|
198
|
-
|
|
199
|
-
# Sets the runner used to process file downloads, provided value is a Riffer::Runner.
|
|
200
|
-
# Raises Riffer::ArgumentError if value is not a Riffer::Runner.
|
|
201
|
-
# --
|
|
202
|
-
# : (untyped) -> void
|
|
203
|
-
def runner=: (untyped) -> void
|
|
204
|
-
|
|
205
|
-
# Sets the object used to download bytes from a URL.
|
|
206
|
-
# Raises Riffer::ArgumentError if value does not respond to +#call+.
|
|
207
|
-
# --
|
|
208
|
-
# : (untyped) -> void
|
|
209
|
-
def downloader=: (untyped) -> void
|
|
210
|
-
end
|
|
211
|
-
|
|
212
|
-
# Consumer-configured token pricing, keyed by +provider/model+ id. Riffer
|
|
213
|
-
# ships no price table, so an unconfigured model carries no cost.
|
|
214
|
-
class Pricing
|
|
215
|
-
@rates: Hash[String, Riffer::Config::Pricing::Rates]
|
|
216
|
-
|
|
217
|
-
# Per-million-token rates for one model's four token buckets. +cache_read+
|
|
218
|
-
# and +cache_write+ fall back to the +input+ rate when unset.
|
|
219
|
-
class Rates
|
|
220
|
-
# Input rate per million tokens.
|
|
221
|
-
attr_reader input: Float
|
|
222
|
-
|
|
223
|
-
# Output rate per million tokens.
|
|
224
|
-
attr_reader output: Float
|
|
225
|
-
|
|
226
|
-
# Cache-read rate per million tokens.
|
|
227
|
-
attr_reader cache_read: Float?
|
|
228
|
-
|
|
229
|
-
# Cache-write rate per million tokens.
|
|
230
|
-
attr_reader cache_write: Float?
|
|
231
|
-
|
|
232
|
-
# --
|
|
233
|
-
# : (input: Float, output: Float, ?cache_read: Float?, ?cache_write: Float?) -> void
|
|
234
|
-
def initialize: (input: Float, output: Float, ?cache_read: Float?, ?cache_write: Float?) -> void
|
|
235
|
-
|
|
236
|
-
# Returns the cost for the given token counts.
|
|
237
|
-
# --
|
|
238
|
-
# : (input_tokens: Integer, output_tokens: Integer, ?cache_read_tokens: Integer?, ?cache_write_tokens: Integer?) -> Float
|
|
239
|
-
def cost_for: (input_tokens: Integer, output_tokens: Integer, ?cache_read_tokens: Integer?, ?cache_write_tokens: Integer?) -> Float
|
|
240
|
-
end
|
|
241
|
-
|
|
242
|
-
# --
|
|
243
|
-
# : () -> void
|
|
244
|
-
def initialize: () -> void
|
|
245
|
-
|
|
246
|
-
# Registers per-million-token rates for a +provider/model+ id/s. Raises
|
|
247
|
-
# Riffer::ArgumentError on a malformed id or a negative/non-numeric rate.
|
|
248
|
-
# --
|
|
249
|
-
# : ((String | Array[String]), input: Numeric, output: Numeric, ?cache_read: Numeric?, ?cache_write: Numeric?) -> void
|
|
250
|
-
def set: (String | Array[String], input: Numeric, output: Numeric, ?cache_read: Numeric?, ?cache_write: Numeric?) -> void
|
|
251
|
-
|
|
252
|
-
# Returns the rates registered for a +provider/model+ id.
|
|
253
|
-
# --
|
|
254
|
-
# : (String) -> Riffer::Config::Pricing::Rates?
|
|
255
|
-
def rates_for: (String) -> Riffer::Config::Pricing::Rates?
|
|
256
|
-
|
|
257
|
-
# Returns true when no rates are registered.
|
|
258
|
-
# --
|
|
259
|
-
# : () -> bool
|
|
260
|
-
def empty?: () -> bool
|
|
261
|
-
|
|
262
|
-
private
|
|
263
|
-
|
|
264
|
-
# --
|
|
265
|
-
# : (String) -> void
|
|
266
|
-
def validate_model!: (String) -> void
|
|
267
|
-
|
|
268
|
-
# --
|
|
269
|
-
# : (untyped, String) -> Float
|
|
270
|
-
def coerce_rate: (untyped, String) -> Float
|
|
271
|
-
|
|
272
|
-
# --
|
|
273
|
-
# : (untyped, String) -> Float?
|
|
274
|
-
def coerce_optional_rate: (untyped, String) -> Float?
|
|
275
|
-
end
|
|
276
|
-
|
|
277
4
|
VALID_MESSAGE_ID_STRATEGIES: untyped
|
|
278
5
|
|
|
279
|
-
# Amazon Bedrock configuration.
|
|
280
6
|
attr_reader amazon_bedrock: Riffer::Config::AmazonBedrock
|
|
281
7
|
|
|
282
|
-
# Anthropic configuration.
|
|
283
8
|
attr_reader anthropic: Riffer::Config::Anthropic
|
|
284
9
|
|
|
285
|
-
# Azure OpenAI configuration.
|
|
286
10
|
attr_reader azure_openai: Riffer::Config::AzureOpenAI
|
|
287
11
|
|
|
288
|
-
# Google Gemini configuration.
|
|
289
12
|
attr_reader gemini: Riffer::Config::Gemini
|
|
290
13
|
|
|
291
|
-
# OpenAI configuration.
|
|
292
14
|
attr_reader openai: Riffer::Config::OpenAI
|
|
293
15
|
|
|
294
|
-
# OpenRouter configuration.
|
|
295
16
|
attr_reader openrouter: Riffer::Config::OpenRouter
|
|
296
17
|
|
|
297
|
-
# Evals configuration.
|
|
298
18
|
attr_reader evals: Riffer::Config::Evals
|
|
299
19
|
|
|
300
|
-
# MCP configuration. +credentials+ is an optional Proc returning per-run
|
|
301
|
-
# +tools/call+ headers (or +nil+ to deny); +discovery_runner+ runs tool
|
|
302
|
-
# discovery.
|
|
303
20
|
attr_reader mcp: Riffer::Config::Mcp
|
|
304
21
|
|
|
305
|
-
# Global tool runtime configuration (experimental); defaults to
|
|
306
|
-
# <tt>Riffer::Tools::Runtime::Inline.new</tt>.
|
|
307
22
|
attr_reader tool_runtime: singleton(Riffer::Tools::Runtime) | Riffer::Tools::Runtime | Proc
|
|
308
23
|
|
|
309
|
-
# Sets the global tool runtime. Raises Riffer::ArgumentError on an invalid
|
|
310
|
-
# value.
|
|
311
24
|
# --
|
|
312
25
|
# : ((singleton(Riffer::Tools::Runtime) | Riffer::Tools::Runtime | Proc)) -> void
|
|
313
26
|
def tool_runtime=: (singleton(Riffer::Tools::Runtime) | Riffer::Tools::Runtime | Proc) -> void
|
|
314
27
|
|
|
315
|
-
# Skills-related global configuration.
|
|
316
28
|
attr_reader skills: Riffer::Config::Skills
|
|
317
29
|
|
|
318
|
-
# Tracing-related global configuration.
|
|
319
30
|
attr_reader tracing: Riffer::Config::Tracing
|
|
320
31
|
|
|
321
32
|
attr_reader files: Riffer::Config::Files
|
|
322
33
|
|
|
323
|
-
# Consumer-configured per-model token pricing.
|
|
324
34
|
attr_reader pricing: Riffer::Config::Pricing
|
|
325
35
|
|
|
326
|
-
#
|
|
327
|
-
#
|
|
328
|
-
# seeded messages passed to +Riffer::Agent#generate+ must carry their own.
|
|
36
|
+
# When not +:none+, seeded messages passed to +Riffer::Agent#generate+ must
|
|
37
|
+
# carry their own +id+.
|
|
329
38
|
attr_reader message_id_strategy: Symbol
|
|
330
39
|
|
|
331
|
-
# Sets the message id strategy. Raises Riffer::ArgumentError unless the value
|
|
332
|
-
# is +:none+, +:uuid+, or +:uuidv7+.
|
|
333
40
|
# --
|
|
334
41
|
# : (Symbol) -> void
|
|
335
42
|
def message_id_strategy=: (Symbol) -> void
|
|
336
43
|
|
|
337
|
-
# Experimental: when +true+, riffer maintains the +tool_use+ ↔ +tool_result+
|
|
338
|
-
# invariant itself — stripping orphaned exchanges and filling interrupted
|
|
339
|
-
# ones. Defaults to +false+; the surface may change without notice.
|
|
340
|
-
attr_reader experimental_history_healing: bool
|
|
341
|
-
|
|
342
|
-
# Sets the +experimental_history_healing+ flag, coercing boolean-ish values so
|
|
343
|
-
# an env-var +"false"+ (truthy in Ruby) doesn't silently enable healing.
|
|
344
|
-
# Raises Riffer::ArgumentError on an unrecognized value.
|
|
345
|
-
# --
|
|
346
|
-
# : (untyped) -> void
|
|
347
|
-
def experimental_history_healing=: (untyped) -> void
|
|
348
|
-
|
|
349
44
|
# --
|
|
350
45
|
# : () -> void
|
|
351
46
|
def initialize: () -> void
|
|
@@ -1,14 +1,5 @@
|
|
|
1
1
|
# Generated from lib/riffer/evals/evaluator.rb with RBS::Inline
|
|
2
2
|
|
|
3
|
-
# Base class for all evaluators. Set +instructions+ and the base class calls
|
|
4
|
-
# the judge automatically; override +#evaluate+ for custom logic. See
|
|
5
|
-
# examples/evaluators/ for reference implementations.
|
|
6
|
-
#
|
|
7
|
-
# class MyEvaluator < Riffer::Evals::Evaluator
|
|
8
|
-
# instructions "Assess medical accuracy of the response..."
|
|
9
|
-
# higher_is_better true
|
|
10
|
-
# judge_model "anthropic/claude-opus-4-5-20251101"
|
|
11
|
-
# end
|
|
12
3
|
class Riffer::Evals::Evaluator
|
|
13
4
|
self.@instructions: String?
|
|
14
5
|
|
|
@@ -16,29 +7,32 @@ class Riffer::Evals::Evaluator
|
|
|
16
7
|
|
|
17
8
|
self.@judge_model: String?
|
|
18
9
|
|
|
10
|
+
self.@identifier: String?
|
|
11
|
+
|
|
19
12
|
@judge: Riffer::Evals::Judge?
|
|
20
13
|
|
|
21
|
-
|
|
22
|
-
|
|
14
|
+
DEFAULT_IDENTIFIER: String
|
|
15
|
+
|
|
23
16
|
# --
|
|
24
17
|
# : (?String?) -> String?
|
|
25
18
|
def self.instructions: (?String?) -> String?
|
|
26
19
|
|
|
27
|
-
# Gets or sets whether higher scores are better.
|
|
28
|
-
#
|
|
29
20
|
# --
|
|
30
21
|
# : (?bool?) -> bool
|
|
31
22
|
def self.higher_is_better: (?bool?) -> bool
|
|
32
23
|
|
|
33
|
-
# Gets or sets the judge model for LLM-as-judge evaluations.
|
|
34
|
-
#
|
|
35
24
|
# --
|
|
36
25
|
# : (?String?) -> String?
|
|
37
26
|
def self.judge_model: (?String?) -> String?
|
|
38
27
|
|
|
39
|
-
#
|
|
40
|
-
#
|
|
41
|
-
|
|
28
|
+
# --
|
|
29
|
+
# : (?String?) -> String
|
|
30
|
+
def self.identifier: (?String?) -> String
|
|
31
|
+
|
|
32
|
+
# --
|
|
33
|
+
# : () -> String
|
|
34
|
+
private def self.derived_identifier: () -> String
|
|
35
|
+
|
|
42
36
|
# --
|
|
43
37
|
# : (input: String | Array[Hash[Symbol, untyped] | Riffer::Messages::Base], output: String, ?ground_truth: String?, ?messages: Array[Riffer::Messages::Base]) -> Riffer::Evals::Result
|
|
44
38
|
def evaluate: (input: String | Array[Hash[Symbol, untyped] | Riffer::Messages::Base], output: String, ?ground_truth: String?, ?messages: Array[Riffer::Messages::Base]) -> Riffer::Evals::Result
|
|
@@ -49,13 +43,10 @@ class Riffer::Evals::Evaluator
|
|
|
49
43
|
# : (String | Array[Hash[Symbol, untyped] | Riffer::Messages::Base]) -> String
|
|
50
44
|
def format_input: (String | Array[Hash[Symbol, untyped] | Riffer::Messages::Base]) -> String
|
|
51
45
|
|
|
52
|
-
# Returns a Judge instance configured for this evaluator.
|
|
53
|
-
#
|
|
54
46
|
# --
|
|
55
47
|
# : () -> Riffer::Evals::Judge
|
|
56
48
|
def judge: () -> Riffer::Evals::Judge
|
|
57
49
|
|
|
58
|
-
# Builds a Result for this evaluator.
|
|
59
50
|
# --
|
|
60
51
|
# : (score: Float, ?reason: String?, ?metadata: Hash[Symbol, untyped], ?token_usage: Riffer::Providers::TokenUsage?) -> Riffer::Evals::Result
|
|
61
52
|
def result: (score: Float, ?reason: String?, ?metadata: Hash[Symbol, untyped], ?token_usage: Riffer::Providers::TokenUsage?) -> Riffer::Evals::Result
|
|
@@ -1,20 +1,6 @@
|
|
|
1
1
|
# Generated from lib/riffer/evals/evaluator_runner.rb with RBS::Inline
|
|
2
2
|
|
|
3
|
-
# Orchestrates running evaluators against an agent across multiple scenarios.
|
|
4
|
-
#
|
|
5
|
-
# result = Riffer::Evals::EvaluatorRunner.run(
|
|
6
|
-
# agent: MyAgent,
|
|
7
|
-
# scenarios: [
|
|
8
|
-
# { input: "What is Ruby?", ground_truth: "A programming language" },
|
|
9
|
-
# { input: "What is Python?" }
|
|
10
|
-
# ],
|
|
11
|
-
# evaluators: [AnswerRelevancyEvaluator]
|
|
12
|
-
# )
|
|
13
|
-
#
|
|
14
|
-
# result.scores # => { AnswerRelevancyEvaluator => 0.85 }
|
|
15
3
|
module Riffer::Evals::EvaluatorRunner
|
|
16
|
-
# Runs evaluators against an agent for the given scenarios. Raises
|
|
17
|
-
# Riffer::ArgumentError on an invalid agent or evaluator.
|
|
18
4
|
# --
|
|
19
5
|
# : (agent: singleton(Riffer::Agent), scenarios: Array[Hash[Symbol, untyped]], evaluators: Array[singleton(Riffer::Evals::Evaluator)], ?context: Hash[Symbol, untyped]?) -> Riffer::Evals::RunResult
|
|
20
6
|
def run: (agent: singleton(Riffer::Agent), scenarios: Array[Hash[Symbol, untyped]], evaluators: Array[singleton(Riffer::Evals::Evaluator)], ?context: Hash[Symbol, untyped]?) -> Riffer::Evals::RunResult
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
# Generated from lib/riffer/evals/judge.rb with RBS::Inline
|
|
2
2
|
|
|
3
|
-
# Executes LLM-as-judge evaluations, using tool calling internally to get
|
|
4
|
-
# structured output from the judge model.
|
|
5
3
|
class Riffer::Evals::Judge
|
|
6
4
|
@provider_instance: Riffer::Providers::Base?
|
|
7
5
|
|
|
@@ -9,22 +7,20 @@ class Riffer::Evals::Judge
|
|
|
9
7
|
|
|
10
8
|
@model_name: String?
|
|
11
9
|
|
|
12
|
-
# Internal tool for structured evaluation output.
|
|
13
10
|
class EvaluationTool < Riffer::Tool
|
|
14
11
|
# --
|
|
15
12
|
# : (context: Riffer::Agent::Context?, score: Float, reason: String) -> Riffer::Tools::Response
|
|
16
13
|
def call: (context: Riffer::Agent::Context?, score: Float, reason: String) -> Riffer::Tools::Response
|
|
17
14
|
end
|
|
18
15
|
|
|
19
|
-
# The model string (provider/model format).
|
|
20
16
|
attr_reader model: String
|
|
21
17
|
|
|
22
|
-
|
|
18
|
+
attr_reader tags: Hash[String, String]
|
|
19
|
+
|
|
23
20
|
# --
|
|
24
|
-
# : (model: String) -> void
|
|
25
|
-
def initialize: (model: String) -> void
|
|
21
|
+
# : (model: String, ?tags: Hash[String, String]) -> void
|
|
22
|
+
def initialize: (model: String, ?tags: Hash[String, String]) -> void
|
|
26
23
|
|
|
27
|
-
# Evaluates an input/output pair using the configured LLM.
|
|
28
24
|
# --
|
|
29
25
|
# : (instructions: String, input: String, output: String, ?ground_truth: String?) -> Hash[Symbol, untyped]
|
|
30
26
|
def evaluate: (instructions: String, input: String, output: String, ?ground_truth: String?) -> Hash[Symbol, untyped]
|
|
@@ -1,33 +1,23 @@
|
|
|
1
1
|
# Generated from lib/riffer/evals/result.rb with RBS::Inline
|
|
2
2
|
|
|
3
|
-
# Represents the result of a single evaluation.
|
|
4
3
|
class Riffer::Evals::Result
|
|
5
|
-
# The evaluator class that produced this result.
|
|
6
4
|
attr_reader evaluator: singleton(Riffer::Evals::Evaluator)
|
|
7
5
|
|
|
8
|
-
# The evaluation score (0.0 to 1.0).
|
|
9
6
|
attr_reader score: Float
|
|
10
7
|
|
|
11
|
-
# Human-readable explanation of the score.
|
|
12
8
|
attr_reader reason: String?
|
|
13
9
|
|
|
14
|
-
# Additional metadata from the evaluation.
|
|
15
10
|
attr_reader metadata: Hash[Symbol, untyped]
|
|
16
11
|
|
|
17
|
-
# Whether higher scores are better for this evaluator.
|
|
18
12
|
attr_reader higher_is_better: bool
|
|
19
13
|
|
|
20
|
-
#
|
|
21
|
-
# evaluator used an LLM. Nil for rule-based evaluators.
|
|
14
|
+
# Nil for rule-based evaluators.
|
|
22
15
|
attr_reader token_usage: Riffer::Providers::TokenUsage?
|
|
23
16
|
|
|
24
|
-
# Raises Riffer::ArgumentError if +score+ is not between 0.0 and 1.0.
|
|
25
17
|
# --
|
|
26
18
|
# : (evaluator: singleton(Riffer::Evals::Evaluator), score: Float, ?reason: String?, ?metadata: Hash[Symbol, untyped], ?higher_is_better: bool, ?token_usage: Riffer::Providers::TokenUsage?) -> void
|
|
27
19
|
def initialize: (evaluator: singleton(Riffer::Evals::Evaluator), score: Float, ?reason: String?, ?metadata: Hash[Symbol, untyped], ?higher_is_better: bool, ?token_usage: Riffer::Providers::TokenUsage?) -> void
|
|
28
20
|
|
|
29
|
-
# Returns a hash representation of the result.
|
|
30
|
-
#
|
|
31
21
|
# --
|
|
32
22
|
# : () -> Hash[Symbol, untyped]
|
|
33
23
|
def to_h: () -> Hash[Symbol, untyped]
|
|
@@ -1,36 +1,24 @@
|
|
|
1
1
|
# Generated from lib/riffer/evals/run_result.rb with RBS::Inline
|
|
2
2
|
|
|
3
|
-
# Represents the complete result of an evaluation run across multiple scenarios.
|
|
4
3
|
class Riffer::Evals::RunResult
|
|
5
|
-
# Per-scenario evaluation results.
|
|
6
4
|
attr_reader scenario_results: Array[Riffer::Evals::ScenarioResult]
|
|
7
5
|
|
|
8
6
|
# --
|
|
9
7
|
# : (scenario_results: Array[Riffer::Evals::ScenarioResult]) -> void
|
|
10
8
|
def initialize: (scenario_results: Array[Riffer::Evals::ScenarioResult]) -> void
|
|
11
9
|
|
|
12
|
-
# Returns average scores keyed by evaluator class across all scenarios.
|
|
13
|
-
#
|
|
14
10
|
# --
|
|
15
11
|
# : () -> Hash[singleton(Riffer::Evals::Evaluator), Float]
|
|
16
12
|
def scores: () -> Hash[singleton(Riffer::Evals::Evaluator), Float]
|
|
17
13
|
|
|
18
|
-
# Returns the summed token usage the agents under test spent across every
|
|
19
|
-
# scenario, or nil when none reported usage.
|
|
20
|
-
#
|
|
21
14
|
# --
|
|
22
15
|
# : () -> Riffer::Providers::TokenUsage?
|
|
23
16
|
def token_usage: () -> Riffer::Providers::TokenUsage?
|
|
24
17
|
|
|
25
|
-
# Returns the summed token usage across every scenario's LLM-as-judge
|
|
26
|
-
# evaluators, or nil when none reported usage.
|
|
27
|
-
#
|
|
28
18
|
# --
|
|
29
19
|
# : () -> Riffer::Providers::TokenUsage?
|
|
30
20
|
def evaluator_token_usage: () -> Riffer::Providers::TokenUsage?
|
|
31
21
|
|
|
32
|
-
# Returns a hash representation of the run result.
|
|
33
|
-
#
|
|
34
22
|
# --
|
|
35
23
|
# : () -> Hash[Symbol, untyped]
|
|
36
24
|
def to_h: () -> Hash[Symbol, untyped]
|
|
@@ -1,44 +1,30 @@
|
|
|
1
1
|
# Generated from lib/riffer/evals/scenario_result.rb with RBS::Inline
|
|
2
2
|
|
|
3
|
-
# Represents the result of evaluating a single scenario.
|
|
4
3
|
class Riffer::Evals::ScenarioResult
|
|
5
|
-
# The input that was evaluated.
|
|
6
4
|
attr_reader input: String
|
|
7
5
|
|
|
8
|
-
# The agent output for this scenario.
|
|
9
6
|
attr_reader output: String
|
|
10
7
|
|
|
11
|
-
# The ground truth used during evaluation.
|
|
12
8
|
attr_reader ground_truth: String?
|
|
13
9
|
|
|
14
|
-
# Individual evaluation results.
|
|
15
10
|
attr_reader results: Array[Riffer::Evals::Result]
|
|
16
11
|
|
|
17
|
-
# The full message history from the agent conversation.
|
|
18
12
|
attr_reader messages: Array[Riffer::Messages::Base]
|
|
19
13
|
|
|
20
|
-
# Token usage the agent under test spent generating this scenario's output.
|
|
21
14
|
attr_reader token_usage: Riffer::Providers::TokenUsage?
|
|
22
15
|
|
|
23
16
|
# --
|
|
24
17
|
# : (input: String, output: String, ground_truth: String?, results: Array[Riffer::Evals::Result], ?messages: Array[Riffer::Messages::Base], ?token_usage: Riffer::Providers::TokenUsage?) -> void
|
|
25
18
|
def initialize: (input: String, output: String, ground_truth: String?, results: Array[Riffer::Evals::Result], ?messages: Array[Riffer::Messages::Base], ?token_usage: Riffer::Providers::TokenUsage?) -> void
|
|
26
19
|
|
|
27
|
-
# Returns scores keyed by evaluator class.
|
|
28
|
-
#
|
|
29
20
|
# --
|
|
30
21
|
# : () -> Hash[singleton(Riffer::Evals::Evaluator), Float]
|
|
31
22
|
def scores: () -> Hash[singleton(Riffer::Evals::Evaluator), Float]
|
|
32
23
|
|
|
33
|
-
# Returns the summed token usage across this scenario's LLM-as-judge
|
|
34
|
-
# evaluators, or nil when none reported usage.
|
|
35
|
-
#
|
|
36
24
|
# --
|
|
37
25
|
# : () -> Riffer::Providers::TokenUsage?
|
|
38
26
|
def evaluator_token_usage: () -> Riffer::Providers::TokenUsage?
|
|
39
27
|
|
|
40
|
-
# Returns a hash representation of the scenario result.
|
|
41
|
-
#
|
|
42
28
|
# --
|
|
43
29
|
# : () -> Hash[Symbol, untyped]
|
|
44
30
|
def to_h: () -> Hash[Symbol, untyped]
|
|
@@ -8,10 +8,6 @@ class Riffer::Files::Resolver
|
|
|
8
8
|
# : (provider: Riffer::Providers::Base) -> void
|
|
9
9
|
def initialize: (provider: Riffer::Providers::Base) -> void
|
|
10
10
|
|
|
11
|
-
# Resolves every file in every User message in place - downloading,
|
|
12
|
-
# verifying, and caching as the provider's capability and each file's
|
|
13
|
-
# sha256 require. Raises Riffer::FileError on any file that can't be
|
|
14
|
-
# resolved
|
|
15
11
|
# : (Array[Riffer::Messages::Base]) -> void
|
|
16
12
|
def resolve!: (Array[Riffer::Messages::Base]) -> void
|
|
17
13
|
|
|
@@ -20,17 +16,14 @@ class Riffer::Files::Resolver
|
|
|
20
16
|
# : (Riffer::Messages::User) -> void
|
|
21
17
|
def check_file_count!: (Riffer::Messages::User) -> void
|
|
22
18
|
|
|
23
|
-
# : (Riffer::Messages::FilePart) -> void
|
|
24
|
-
def resolve_file!: (Riffer::Messages::FilePart) -> void
|
|
19
|
+
# : (Riffer::Messages::User::FilePart) -> void
|
|
20
|
+
def resolve_file!: (Riffer::Messages::User::FilePart) -> void
|
|
25
21
|
|
|
26
|
-
# : (Riffer::Messages::FilePart) -> void
|
|
27
|
-
def verify_inline!: (Riffer::Messages::FilePart) -> void
|
|
22
|
+
# : (Riffer::Messages::User::FilePart) -> void
|
|
23
|
+
def verify_inline!: (Riffer::Messages::User::FilePart) -> void
|
|
28
24
|
|
|
29
|
-
#
|
|
30
|
-
|
|
31
|
-
# would hold memory nothing reads and let later turns skip re-verifying.
|
|
32
|
-
# : (Riffer::Messages::FilePart, cache: (false | Symbol)) -> void
|
|
33
|
-
def download!: (Riffer::Messages::FilePart, cache: false | Symbol) -> void
|
|
25
|
+
# : (Riffer::Messages::User::FilePart, cache: (false | Symbol)) -> void
|
|
26
|
+
def download!: (Riffer::Messages::User::FilePart, cache: false | Symbol) -> void
|
|
34
27
|
|
|
35
28
|
def verify_bytes!: (untyped bytes, untyped sha256) -> untyped
|
|
36
29
|
end
|
|
@@ -1,48 +1,28 @@
|
|
|
1
1
|
# Generated from lib/riffer/guardrail.rb with RBS::Inline
|
|
2
2
|
|
|
3
|
-
# Base class for guardrails that process input and output in the agent pipeline.
|
|
4
|
-
#
|
|
5
|
-
# class MyGuardrail < Riffer::Guardrail
|
|
6
|
-
# def process_input(messages, context:)
|
|
7
|
-
# # Return pass(messages), transform(modified_messages), or block(reason)
|
|
8
|
-
# pass(messages)
|
|
9
|
-
# end
|
|
10
|
-
#
|
|
11
|
-
# def process_output(response, messages:, context:)
|
|
12
|
-
# # Return pass(response), transform(modified_response), or block(reason)
|
|
13
|
-
# pass(response)
|
|
14
|
-
# end
|
|
15
|
-
# end
|
|
16
3
|
class Riffer::Guardrail
|
|
17
|
-
# Processes input messages before they're sent to the LLM; override in
|
|
18
|
-
# subclasses.
|
|
19
4
|
# --
|
|
20
5
|
# : (Array[Riffer::Messages::Base], context: untyped) -> Riffer::Guardrails::Result
|
|
21
6
|
def process_input: (Array[Riffer::Messages::Base], context: untyped) -> Riffer::Guardrails::Result
|
|
22
7
|
|
|
23
|
-
# Processes the output response after it's received from the LLM; override in
|
|
24
|
-
# subclasses.
|
|
25
8
|
# --
|
|
26
9
|
# : (Riffer::Messages::Assistant, messages: Array[Riffer::Messages::Base], context: untyped) -> Riffer::Guardrails::Result
|
|
27
10
|
def process_output: (Riffer::Messages::Assistant, messages: Array[Riffer::Messages::Base], context: untyped) -> Riffer::Guardrails::Result
|
|
28
11
|
|
|
29
|
-
#
|
|
30
|
-
#
|
|
12
|
+
# Used as the tracing span suffix and the <tt>riffer.guardrail.name</tt>
|
|
13
|
+
# attribute.
|
|
31
14
|
# --
|
|
32
15
|
# : () -> String
|
|
33
16
|
def name: () -> String
|
|
34
17
|
|
|
35
|
-
# Creates a pass result that continues with unchanged data.
|
|
36
18
|
# --
|
|
37
19
|
# : (untyped) -> Riffer::Guardrails::Result
|
|
38
20
|
def pass: (untyped) -> Riffer::Guardrails::Result
|
|
39
21
|
|
|
40
|
-
# Creates a transform result that continues with transformed data.
|
|
41
22
|
# --
|
|
42
23
|
# : (untyped) -> Riffer::Guardrails::Result
|
|
43
24
|
def transform: (untyped) -> Riffer::Guardrails::Result
|
|
44
25
|
|
|
45
|
-
# Creates a block result that halts execution.
|
|
46
26
|
# --
|
|
47
27
|
# : (String, ?metadata: Hash[Symbol, untyped]?) -> Riffer::Guardrails::Result
|
|
48
28
|
def block: (String, ?metadata: Hash[Symbol, untyped]?) -> Riffer::Guardrails::Result
|