riffer 0.30.0 → 0.32.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (214) hide show
  1. checksums.yaml +4 -4
  2. data/.agents/code-style.md +63 -4
  3. data/.agents/rbs-inline.md +1 -6
  4. data/.release-please-manifest.json +1 -1
  5. data/AGENTS.md +1 -2
  6. data/CHANGELOG.md +25 -0
  7. data/docs/08_MESSAGES.md +1 -1
  8. data/docs/14_MCP.md +50 -5
  9. data/docs/15_SERIALIZATION.md +23 -12
  10. data/docs/providers/02_AMAZON_BEDROCK.md +14 -0
  11. data/lib/riffer/agent/config.rb +42 -47
  12. data/lib/riffer/agent/context.rb +70 -50
  13. data/lib/riffer/agent/response.rb +4 -20
  14. data/lib/riffer/agent/run.rb +28 -67
  15. data/lib/riffer/agent/serializer.rb +36 -85
  16. data/lib/riffer/agent/session/repair.rb +14 -40
  17. data/lib/riffer/agent/session.rb +25 -67
  18. data/lib/riffer/agent/structured_output/result.rb +3 -11
  19. data/lib/riffer/agent/structured_output.rb +5 -13
  20. data/lib/riffer/agent.rb +81 -199
  21. data/lib/riffer/config.rb +34 -101
  22. data/lib/riffer/evals/evaluator.rb +7 -27
  23. data/lib/riffer/evals/evaluator_runner.rb +11 -19
  24. data/lib/riffer/evals/judge.rb +4 -25
  25. data/lib/riffer/evals/result.rb +1 -18
  26. data/lib/riffer/evals/run_result.rb +0 -11
  27. data/lib/riffer/evals/scenario_result.rb +0 -14
  28. data/lib/riffer/evals.rb +0 -6
  29. data/lib/riffer/guardrail.rb +4 -27
  30. data/lib/riffer/guardrails/modification.rb +0 -10
  31. data/lib/riffer/guardrails/result.rb +3 -30
  32. data/lib/riffer/guardrails/runner.rb +5 -22
  33. data/lib/riffer/guardrails/tripwire.rb +1 -19
  34. data/lib/riffer/guardrails.rb +2 -4
  35. data/lib/riffer/helpers/call_or_value.rb +4 -3
  36. data/lib/riffer/helpers/class_name_converter.rb +3 -1
  37. data/lib/riffer/helpers/dependencies.rb +5 -7
  38. data/lib/riffer/helpers.rb +0 -5
  39. data/lib/riffer/mcp/authenticated_tool.rb +9 -9
  40. data/lib/riffer/mcp/client.rb +12 -17
  41. data/lib/riffer/mcp/manifest.rb +13 -10
  42. data/lib/riffer/mcp/registration.rb +2 -11
  43. data/lib/riffer/mcp/registry.rb +44 -52
  44. data/lib/riffer/mcp/search_tool.rb +53 -0
  45. data/lib/riffer/mcp/tool_factory.rb +13 -18
  46. data/lib/riffer/mcp.rb +12 -17
  47. data/lib/riffer/messages/assistant.rb +2 -9
  48. data/lib/riffer/messages/base.rb +46 -16
  49. data/lib/riffer/messages/file_part.rb +32 -24
  50. data/lib/riffer/messages/system.rb +0 -5
  51. data/lib/riffer/messages/tool.rb +0 -10
  52. data/lib/riffer/messages/user.rb +0 -10
  53. data/lib/riffer/messages.rb +0 -7
  54. data/lib/riffer/params/boolean.rb +2 -4
  55. data/lib/riffer/params/param.rb +28 -39
  56. data/lib/riffer/params.rb +9 -21
  57. data/lib/riffer/providers/amazon_bedrock.rb +42 -28
  58. data/lib/riffer/providers/anthropic.rb +4 -9
  59. data/lib/riffer/providers/azure_open_ai.rb +3 -19
  60. data/lib/riffer/providers/base.rb +13 -26
  61. data/lib/riffer/providers/gemini.rb +4 -4
  62. data/lib/riffer/providers/mock.rb +6 -26
  63. data/lib/riffer/providers/open_ai.rb +6 -8
  64. data/lib/riffer/providers/open_router.rb +4 -10
  65. data/lib/riffer/providers/repository.rb +4 -3
  66. data/lib/riffer/providers/token_usage.rb +9 -20
  67. data/lib/riffer/providers.rb +0 -8
  68. data/lib/riffer/runner/fibers.rb +10 -16
  69. data/lib/riffer/runner/sequential.rb +1 -4
  70. data/lib/riffer/runner/threaded.rb +3 -14
  71. data/lib/riffer/runner.rb +2 -15
  72. data/lib/riffer/skills/activate_tool.rb +2 -11
  73. data/lib/riffer/skills/adapter.rb +4 -22
  74. data/lib/riffer/skills/backend.rb +7 -21
  75. data/lib/riffer/skills/config.rb +10 -31
  76. data/lib/riffer/skills/context.rb +5 -20
  77. data/lib/riffer/skills/filesystem_backend.rb +7 -25
  78. data/lib/riffer/skills/frontmatter.rb +10 -28
  79. data/lib/riffer/skills/markdown_adapter.rb +2 -9
  80. data/lib/riffer/skills/xml_adapter.rb +2 -8
  81. data/lib/riffer/stream_events/base.rb +1 -6
  82. data/lib/riffer/stream_events/guardrail_modification.rb +1 -8
  83. data/lib/riffer/stream_events/guardrail_tripwire.rb +1 -8
  84. data/lib/riffer/stream_events/interrupt.rb +4 -7
  85. data/lib/riffer/stream_events/reasoning_delta.rb +2 -4
  86. data/lib/riffer/stream_events/reasoning_done.rb +2 -4
  87. data/lib/riffer/stream_events/skill_activation.rb +2 -4
  88. data/lib/riffer/stream_events/text_delta.rb +0 -2
  89. data/lib/riffer/stream_events/text_done.rb +1 -3
  90. data/lib/riffer/stream_events/token_usage_done.rb +1 -8
  91. data/lib/riffer/stream_events/tool_call_delta.rb +2 -3
  92. data/lib/riffer/stream_events/tool_call_done.rb +1 -3
  93. data/lib/riffer/stream_events/web_search_done.rb +1 -3
  94. data/lib/riffer/stream_events/web_search_status.rb +2 -3
  95. data/lib/riffer/stream_events.rb +0 -10
  96. data/lib/riffer/tool.rb +6 -13
  97. data/lib/riffer/tools/response.rb +8 -4
  98. data/lib/riffer/tools/runtime/fibers.rb +0 -3
  99. data/lib/riffer/tools/runtime/inline.rb +1 -4
  100. data/lib/riffer/tools/runtime/threaded.rb +0 -2
  101. data/lib/riffer/tools/runtime.rb +5 -38
  102. data/lib/riffer/tools/toolable.rb +5 -16
  103. data/lib/riffer/tools.rb +0 -4
  104. data/lib/riffer/version.rb +1 -1
  105. data/lib/riffer.rb +7 -8
  106. data/sig/generated/riffer/agent/config.rbs +29 -46
  107. data/sig/generated/riffer/agent/context.rbs +40 -48
  108. data/sig/generated/riffer/agent/response.rbs +4 -20
  109. data/sig/generated/riffer/agent/run.rbs +12 -61
  110. data/sig/generated/riffer/agent/serializer.rbs +28 -81
  111. data/sig/generated/riffer/agent/session/repair.rbs +12 -40
  112. data/sig/generated/riffer/agent/session.rbs +25 -67
  113. data/sig/generated/riffer/agent/structured_output/result.rbs +2 -10
  114. data/sig/generated/riffer/agent/structured_output.rbs +5 -12
  115. data/sig/generated/riffer/agent.rbs +62 -191
  116. data/sig/generated/riffer/config.rbs +34 -100
  117. data/sig/generated/riffer/evals/evaluator.rbs +7 -27
  118. data/sig/generated/riffer/evals/evaluator_runner.rbs +9 -19
  119. data/sig/generated/riffer/evals/judge.rbs +4 -24
  120. data/sig/generated/riffer/evals/result.rbs +1 -17
  121. data/sig/generated/riffer/evals/run_result.rbs +0 -10
  122. data/sig/generated/riffer/evals/scenario_result.rbs +0 -13
  123. data/sig/generated/riffer/evals.rbs +0 -6
  124. data/sig/generated/riffer/guardrail.rbs +4 -27
  125. data/sig/generated/riffer/guardrails/modification.rbs +0 -10
  126. data/sig/generated/riffer/guardrails/result.rbs +3 -30
  127. data/sig/generated/riffer/guardrails/runner.rbs +5 -22
  128. data/sig/generated/riffer/guardrails/tripwire.rbs +1 -19
  129. data/sig/generated/riffer/guardrails.rbs +2 -4
  130. data/sig/generated/riffer/helpers/call_or_value.rbs +4 -3
  131. data/sig/generated/riffer/helpers/class_name_converter.rbs +1 -1
  132. data/sig/generated/riffer/helpers/dependencies.rbs +3 -7
  133. data/sig/generated/riffer/helpers.rbs +0 -5
  134. data/sig/generated/riffer/mcp/authenticated_tool.rbs +5 -4
  135. data/sig/generated/riffer/mcp/client.rbs +10 -16
  136. data/sig/generated/riffer/mcp/manifest.rbs +9 -9
  137. data/sig/generated/riffer/mcp/registration.rbs +2 -10
  138. data/sig/generated/riffer/mcp/registry.rbs +11 -18
  139. data/sig/generated/riffer/mcp/search_tool.rbs +26 -0
  140. data/sig/generated/riffer/mcp/tool_factory.rbs +10 -15
  141. data/sig/generated/riffer/mcp.rbs +10 -17
  142. data/sig/generated/riffer/messages/assistant.rbs +2 -8
  143. data/sig/generated/riffer/messages/base.rbs +11 -16
  144. data/sig/generated/riffer/messages/file_part.rbs +13 -23
  145. data/sig/generated/riffer/messages/system.rbs +0 -4
  146. data/sig/generated/riffer/messages/tool.rbs +0 -9
  147. data/sig/generated/riffer/messages/user.rbs +0 -9
  148. data/sig/generated/riffer/messages.rbs +0 -7
  149. data/sig/generated/riffer/params/boolean.rbs +2 -4
  150. data/sig/generated/riffer/params/param.rbs +21 -39
  151. data/sig/generated/riffer/params.rbs +9 -21
  152. data/sig/generated/riffer/providers/amazon_bedrock.rbs +21 -25
  153. data/sig/generated/riffer/providers/anthropic.rbs +2 -7
  154. data/sig/generated/riffer/providers/azure_open_ai.rbs +3 -18
  155. data/sig/generated/riffer/providers/base.rbs +9 -25
  156. data/sig/generated/riffer/providers/gemini.rbs +0 -2
  157. data/sig/generated/riffer/providers/mock.rbs +6 -26
  158. data/sig/generated/riffer/providers/open_ai.rbs +1 -5
  159. data/sig/generated/riffer/providers/open_router.rbs +4 -10
  160. data/sig/generated/riffer/providers/repository.rbs +2 -3
  161. data/sig/generated/riffer/providers/token_usage.rbs +6 -16
  162. data/sig/generated/riffer/providers.rbs +0 -8
  163. data/sig/generated/riffer/runner/fibers.rbs +8 -15
  164. data/sig/generated/riffer/runner/sequential.rbs +1 -3
  165. data/sig/generated/riffer/runner/threaded.rbs +3 -13
  166. data/sig/generated/riffer/runner.rbs +2 -14
  167. data/sig/generated/riffer/skills/activate_tool.rbs +2 -11
  168. data/sig/generated/riffer/skills/adapter.rbs +4 -22
  169. data/sig/generated/riffer/skills/backend.rbs +7 -21
  170. data/sig/generated/riffer/skills/config.rbs +10 -31
  171. data/sig/generated/riffer/skills/context.rbs +5 -20
  172. data/sig/generated/riffer/skills/filesystem_backend.rbs +7 -24
  173. data/sig/generated/riffer/skills/frontmatter.rbs +10 -27
  174. data/sig/generated/riffer/skills/markdown_adapter.rbs +2 -9
  175. data/sig/generated/riffer/skills/xml_adapter.rbs +2 -8
  176. data/sig/generated/riffer/stream_events/base.rbs +1 -6
  177. data/sig/generated/riffer/stream_events/guardrail_modification.rbs +1 -8
  178. data/sig/generated/riffer/stream_events/guardrail_tripwire.rbs +1 -8
  179. data/sig/generated/riffer/stream_events/interrupt.rbs +4 -7
  180. data/sig/generated/riffer/stream_events/reasoning_delta.rbs +2 -4
  181. data/sig/generated/riffer/stream_events/reasoning_done.rbs +2 -4
  182. data/sig/generated/riffer/stream_events/skill_activation.rbs +2 -4
  183. data/sig/generated/riffer/stream_events/text_delta.rbs +0 -2
  184. data/sig/generated/riffer/stream_events/text_done.rbs +1 -3
  185. data/sig/generated/riffer/stream_events/token_usage_done.rbs +1 -7
  186. data/sig/generated/riffer/stream_events/tool_call_delta.rbs +2 -3
  187. data/sig/generated/riffer/stream_events/tool_call_done.rbs +1 -3
  188. data/sig/generated/riffer/stream_events/web_search_done.rbs +1 -3
  189. data/sig/generated/riffer/stream_events/web_search_status.rbs +2 -3
  190. data/sig/generated/riffer/stream_events.rbs +0 -10
  191. data/sig/generated/riffer/tool.rbs +5 -12
  192. data/sig/generated/riffer/tools/response.rbs +6 -4
  193. data/sig/generated/riffer/tools/runtime/fibers.rbs +0 -3
  194. data/sig/generated/riffer/tools/runtime/inline.rbs +1 -3
  195. data/sig/generated/riffer/tools/runtime/threaded.rbs +0 -2
  196. data/sig/generated/riffer/tools/runtime.rbs +5 -37
  197. data/sig/generated/riffer/tools/toolable.rbs +4 -14
  198. data/sig/generated/riffer/tools.rbs +0 -4
  199. data/sig/generated/riffer.rbs +5 -4
  200. data/sig/manual/riffer/agent/session/repair.rbs +5 -0
  201. data/sig/manual/riffer/evals/evaluator_runner.rbs +5 -0
  202. data/sig/manual/riffer/helpers/class_name_converter.rbs +5 -0
  203. data/sig/manual/riffer/helpers/dependencies.rbs +5 -0
  204. data/sig/manual/riffer/mcp/authenticated_tool.rbs +5 -0
  205. data/sig/manual/riffer/mcp/registry.rbs +5 -0
  206. data/sig/manual/riffer/mcp/tool_factory.rbs +5 -0
  207. data/sig/manual/riffer/mcp.rbs +5 -0
  208. data/sig/manual/riffer/providers/repository.rbs +5 -0
  209. data/sig/manual/riffer.rbs +5 -0
  210. metadata +17 -9
  211. data/.agents/rdoc.md +0 -69
  212. data/lib/riffer/messages/converter.rb +0 -90
  213. data/sig/generated/riffer/messages/converter.rbs +0 -33
  214. data/sig/manual/riffer/tools/toolable.rbs +0 -6
@@ -1,12 +1,8 @@
1
1
  # Generated from lib/riffer/evals/evaluator.rb with RBS::Inline
2
2
 
3
- # Base class for all evaluators in the Riffer framework.
4
- #
5
- # Provides a DSL for defining evaluator metadata and the evaluate method.
6
- # Simple evaluators only need to set +instructions+ — the base class
7
- # handles calling the judge automatically.
8
- #
9
- # See examples/evaluators/ for reference implementations.
3
+ # Base class for all evaluators. Set +instructions+ and the base class calls
4
+ # the judge automatically; override +#evaluate+ for custom logic. See
5
+ # examples/evaluators/ for reference implementations.
10
6
  #
11
7
  # class MyEvaluator < Riffer::Evals::Evaluator
12
8
  # instructions "Assess medical accuracy of the response..."
@@ -40,30 +36,15 @@ class Riffer::Evals::Evaluator
40
36
  # : (?String?) -> String?
41
37
  def self.judge_model: (?String?) -> String?
42
38
 
43
- # Evaluates an input/output pair.
44
- #
45
- # The default implementation calls the judge with the class-level +instructions+.
46
- # Override this method for custom evaluation logic (e.g. rule-based evaluators).
47
- #
48
- # [input] the input to evaluate; String or Array of message hashes/Message objects.
49
- # [output] the agent's response to evaluate.
50
- # [ground_truth] optional reference answer for comparison.
51
- # [messages] the full message history from the agent conversation.
52
- #
53
- # Raises NotImplementedError if neither +instructions+ is set nor +evaluate+ is overridden.
54
- #
39
+ # Evaluates an input/output pair. The default calls the judge with the
40
+ # class-level +instructions+; override for custom logic (e.g. rule-based
41
+ # evaluators).
55
42
  # --
56
43
  # : (input: String | Array[Hash[Symbol, untyped] | Riffer::Messages::Base], output: String, ?ground_truth: String?, ?messages: Array[Riffer::Messages::Base]) -> Riffer::Evals::Result
57
44
  def evaluate: (input: String | Array[Hash[Symbol, untyped] | Riffer::Messages::Base], output: String, ?ground_truth: String?, ?messages: Array[Riffer::Messages::Base]) -> Riffer::Evals::Result
58
45
 
59
46
  private
60
47
 
61
- # Formats the input for the judge.
62
- #
63
- # String inputs are passed through as-is.
64
- # Array inputs (message hashes or Message objects) are formatted
65
- # as labeled role/content pairs separated by blank lines.
66
- #
67
48
  # --
68
49
  # : (String | Array[Hash[Symbol, untyped] | Riffer::Messages::Base]) -> String
69
50
  def format_input: (String | Array[Hash[Symbol, untyped] | Riffer::Messages::Base]) -> String
@@ -74,8 +55,7 @@ class Riffer::Evals::Evaluator
74
55
  # : () -> Riffer::Evals::Judge
75
56
  def judge: () -> Riffer::Evals::Judge
76
57
 
77
- # Helper to build a Result object.
78
- #
58
+ # Builds a Result for this evaluator.
79
59
  # --
80
60
  # : (score: Float, ?reason: String?, ?metadata: Hash[Symbol, untyped]) -> Riffer::Evals::Result
81
61
  def result: (score: Float, ?reason: String?, ?metadata: Hash[Symbol, untyped]) -> Riffer::Evals::Result
@@ -2,10 +2,6 @@
2
2
 
3
3
  # Orchestrates running evaluators against an agent across multiple scenarios.
4
4
  #
5
- # Accepts an agent class, a list of scenarios, and evaluator classes.
6
- # Generates agent output for each scenario and runs all evaluators,
7
- # returning a RunResult with per-scenario details and aggregate scores.
8
- #
9
5
  # result = Riffer::Evals::EvaluatorRunner.run(
10
6
  # agent: MyAgent,
11
7
  # scenarios: [
@@ -16,30 +12,24 @@
16
12
  # )
17
13
  #
18
14
  # result.scores # => { AnswerRelevancyEvaluator => 0.85 }
19
- class Riffer::Evals::EvaluatorRunner
20
- # Runs evaluators against an agent for the given scenarios.
21
- #
22
- # [agent] an Agent subclass (not an instance).
23
- # [scenarios] array of hashes with +:input+, optional +:ground_truth+, and optional +:context+.
24
- # [evaluators] array of Evaluator subclasses to run against each scenario.
25
- # [context] optional hash passed to +agent.generate+. Per-scenario +:context+ takes precedence.
26
- #
27
- # Raises Riffer::ArgumentError if agent is not a Riffer::Agent subclass
28
- # or any eval is not a Riffer::Evals::Evaluator subclass.
29
- #
15
+ module Riffer::Evals::EvaluatorRunner
16
+ # Runs evaluators against an agent for the given scenarios. Raises
17
+ # Riffer::ArgumentError on an invalid agent or evaluator.
30
18
  # --
31
19
  # : (agent: singleton(Riffer::Agent), scenarios: Array[Hash[Symbol, untyped]], evaluators: Array[singleton(Riffer::Evals::Evaluator)], ?context: Hash[Symbol, untyped]?) -> Riffer::Evals::RunResult
32
- def self.run: (agent: singleton(Riffer::Agent), scenarios: Array[Hash[Symbol, untyped]], evaluators: Array[singleton(Riffer::Evals::Evaluator)], ?context: Hash[Symbol, untyped]?) -> Riffer::Evals::RunResult
20
+ def run: (agent: singleton(Riffer::Agent), scenarios: Array[Hash[Symbol, untyped]], evaluators: Array[singleton(Riffer::Evals::Evaluator)], ?context: Hash[Symbol, untyped]?) -> Riffer::Evals::RunResult
21
+
22
+ private
33
23
 
34
24
  # --
35
25
  # : (singleton(Riffer::Agent)) -> void
36
- def self.validate_agent!: (singleton(Riffer::Agent)) -> void
26
+ def validate_agent!: (singleton(Riffer::Agent)) -> void
37
27
 
38
28
  # --
39
29
  # : (Array[singleton(Riffer::Evals::Evaluator)]) -> void
40
- def self.validate_evaluators!: (Array[singleton(Riffer::Evals::Evaluator)]) -> void
30
+ def validate_evaluators!: (Array[singleton(Riffer::Evals::Evaluator)]) -> void
41
31
 
42
32
  # --
43
33
  # : (agent: singleton(Riffer::Agent), scenario: Hash[Symbol, untyped], evaluators: Array[singleton(Riffer::Evals::Evaluator)], ?context: Hash[Symbol, untyped]?) -> Riffer::Evals::ScenarioResult
44
- def self.run_scenario: (agent: singleton(Riffer::Agent), scenario: Hash[Symbol, untyped], evaluators: Array[singleton(Riffer::Evals::Evaluator)], ?context: Hash[Symbol, untyped]?) -> Riffer::Evals::ScenarioResult
34
+ def run_scenario: (agent: singleton(Riffer::Agent), scenario: Hash[Symbol, untyped], evaluators: Array[singleton(Riffer::Evals::Evaluator)], ?context: Hash[Symbol, untyped]?) -> Riffer::Evals::ScenarioResult
45
35
  end
@@ -1,19 +1,7 @@
1
1
  # Generated from lib/riffer/evals/judge.rb with RBS::Inline
2
2
 
3
- # Executes LLM-as-judge evaluations using the provider infrastructure.
4
- #
5
- # The Judge class handles calling an LLM to evaluate agent outputs
6
- # and parsing the structured response. It uses tool calling internally
7
- # to get guaranteed structured output from the judge model.
8
- #
9
- # judge = Riffer::Evals::Judge.new(model: "anthropic/claude-opus-4-5-20251101")
10
- # result = judge.evaluate(
11
- # instructions: "Assess answer relevancy...",
12
- # input: "What is Ruby?",
13
- # output: "Ruby is a programming language."
14
- # )
15
- # result[:score] # => 0.85
16
- # result[:reason] # => "The response is relevant..."
3
+ # Executes LLM-as-judge evaluations, using tool calling internally to get
4
+ # structured output from the judge model.
17
5
  class Riffer::Evals::Judge
18
6
  @provider_options: Hash[Symbol, untyped]
19
7
 
@@ -33,20 +21,12 @@ class Riffer::Evals::Judge
33
21
  # The model string (provider/model format).
34
22
  attr_reader model: String
35
23
 
36
- # Initializes a new judge.
37
- #
24
+ # Raises Riffer::ArgumentError unless +model+ is "provider/model" format.
38
25
  # --
39
26
  # : (model: String, ?provider_options: Hash[Symbol, untyped]) -> void
40
27
  def initialize: (model: String, ?provider_options: Hash[Symbol, untyped]) -> void
41
28
 
42
- # Evaluates using the configured LLM.
43
- #
44
- # Composes system and user messages from the semantic fields:
45
- # [instructions] evaluation criteria and scoring rubric.
46
- # [input] the original input/question.
47
- # [output] the agent's response to evaluate.
48
- # [ground_truth] optional reference answer for comparison.
49
- #
29
+ # Evaluates an input/output pair using the configured LLM.
50
30
  # --
51
31
  # : (instructions: String, input: String, output: String, ?ground_truth: String?) -> Hash[Symbol, untyped]
52
32
  def evaluate: (instructions: String, input: String, output: String, ?ground_truth: String?) -> Hash[Symbol, untyped]
@@ -1,19 +1,6 @@
1
1
  # Generated from lib/riffer/evals/result.rb with RBS::Inline
2
2
 
3
3
  # Represents the result of a single evaluation.
4
- #
5
- # Contains the score, reason, and metadata from running an evaluator.
6
- #
7
- # result = Riffer::Evals::Result.new(
8
- # evaluator: AnswerRelevancyEvaluator,
9
- # score: 0.85,
10
- # reason: "The response addresses the question directly.",
11
- # higher_is_better: true
12
- # )
13
- #
14
- # result.score # => 0.85
15
- # result.evaluator # => AnswerRelevancyEvaluator
16
- # result.higher_is_better # => true
17
4
  class Riffer::Evals::Result
18
5
  # The evaluator class that produced this result.
19
6
  attr_reader evaluator: singleton(Riffer::Evals::Evaluator)
@@ -30,10 +17,7 @@ class Riffer::Evals::Result
30
17
  # Whether higher scores are better for this evaluator.
31
18
  attr_reader higher_is_better: bool
32
19
 
33
- # Initializes a new evaluation result.
34
- #
35
- # Raises Riffer::ArgumentError if score is not between 0.0 and 1.0.
36
- #
20
+ # Raises Riffer::ArgumentError if +score+ is not between 0.0 and 1.0.
37
21
  # --
38
22
  # : (evaluator: singleton(Riffer::Evals::Evaluator), score: Float, ?reason: String?, ?metadata: Hash[Symbol, untyped], ?higher_is_better: bool) -> void
39
23
  def initialize: (evaluator: singleton(Riffer::Evals::Evaluator), score: Float, ?reason: String?, ?metadata: Hash[Symbol, untyped], ?higher_is_better: bool) -> void
@@ -1,20 +1,10 @@
1
1
  # Generated from lib/riffer/evals/run_result.rb with RBS::Inline
2
2
 
3
3
  # Represents the complete result of an evaluation run across multiple scenarios.
4
- #
5
- # Contains per-scenario results and provides aggregate scores.
6
- #
7
- # run_result = Riffer::Evals::RunResult.new(
8
- # scenario_results: [scenario_result1, scenario_result2]
9
- # )
10
- #
11
- # run_result.scores # => { MyEvaluator => 0.85 }
12
4
  class Riffer::Evals::RunResult
13
5
  # Per-scenario evaluation results.
14
6
  attr_reader scenario_results: Array[Riffer::Evals::ScenarioResult]
15
7
 
16
- # Initializes a new run result.
17
- #
18
8
  # --
19
9
  # : (scenario_results: Array[Riffer::Evals::ScenarioResult]) -> void
20
10
  def initialize: (scenario_results: Array[Riffer::Evals::ScenarioResult]) -> void
@@ -1,17 +1,6 @@
1
1
  # Generated from lib/riffer/evals/scenario_result.rb with RBS::Inline
2
2
 
3
3
  # Represents the result of evaluating a single scenario.
4
- #
5
- # Contains the input, output, ground truth, and individual evaluator results.
6
- #
7
- # scenario_result = Riffer::Evals::ScenarioResult.new(
8
- # input: "What is Ruby?",
9
- # output: "A programming language.",
10
- # ground_truth: "A programming language",
11
- # results: [result1, result2]
12
- # )
13
- #
14
- # scenario_result.scores # => { MyEvaluator => 0.85 }
15
4
  class Riffer::Evals::ScenarioResult
16
5
  # The input that was evaluated.
17
6
  attr_reader input: String
@@ -28,8 +17,6 @@ class Riffer::Evals::ScenarioResult
28
17
  # The full message history from the agent conversation.
29
18
  attr_reader messages: Array[Riffer::Messages::Base]
30
19
 
31
- # Initializes a new scenario result.
32
- #
33
20
  # --
34
21
  # : (input: String, output: String, ground_truth: String?, results: Array[Riffer::Evals::Result], ?messages: Array[Riffer::Messages::Base]) -> void
35
22
  def initialize: (input: String, output: String, ground_truth: String?, results: Array[Riffer::Evals::Result], ?messages: Array[Riffer::Messages::Base]) -> void
@@ -1,10 +1,4 @@
1
1
  # Generated from lib/riffer/evals.rb with RBS::Inline
2
2
 
3
- # Riffer::Evals provides LLM-as-judge evaluation capabilities.
4
- #
5
- # Evals allow you to measure the quality of agent outputs using
6
- # configurable evaluators and scenarios.
7
- #
8
- # See Riffer::Evals::Evaluator, Riffer::Evals::EvaluatorRunner, and Riffer::Evals::RunResult.
9
3
  module Riffer::Evals
10
4
  end
@@ -2,8 +2,6 @@
2
2
 
3
3
  # Base class for guardrails that process input and output in the agent pipeline.
4
4
  #
5
- # Subclass this to create custom guardrails:
6
- #
7
5
  # class MyGuardrail < Riffer::Guardrail
8
6
  # def process_input(messages, context:)
9
7
  # # Return pass(messages), transform(modified_messages), or block(reason)
@@ -16,50 +14,29 @@
16
14
  # end
17
15
  # end
18
16
  class Riffer::Guardrail
19
- # Processes input messages before they are sent to the LLM.
20
- #
21
- # Override this method in subclasses to implement input processing.
22
- #
23
- # [messages] the input messages.
24
- # [context] optional context passed to the agent.
25
- #
17
+ # Processes input messages before they're sent to the LLM; override in
18
+ # subclasses.
26
19
  # --
27
20
  # : (Array[Riffer::Messages::Base], context: untyped) -> Riffer::Guardrails::Result
28
21
  def process_input: (Array[Riffer::Messages::Base], context: untyped) -> Riffer::Guardrails::Result
29
22
 
30
- # Processes output response after it is received from the LLM.
31
- #
32
- # Override this method in subclasses to implement output processing.
33
- #
34
- # [response] the LLM response.
35
- # [messages] the conversation messages.
36
- # [context] optional context passed to the agent.
37
- #
23
+ # Processes the output response after it's received from the LLM; override in
24
+ # subclasses.
38
25
  # --
39
26
  # : (Riffer::Messages::Assistant, messages: Array[Riffer::Messages::Base], context: untyped) -> Riffer::Guardrails::Result
40
27
  def process_output: (Riffer::Messages::Assistant, messages: Array[Riffer::Messages::Base], context: untyped) -> Riffer::Guardrails::Result
41
28
 
42
29
  # Creates a pass result that continues with unchanged data.
43
- #
44
- # [data] the original data to pass through.
45
- #
46
30
  # --
47
31
  # : (untyped) -> Riffer::Guardrails::Result
48
32
  def pass: (untyped) -> Riffer::Guardrails::Result
49
33
 
50
34
  # Creates a transform result that continues with transformed data.
51
- #
52
- # [data] the transformed data.
53
- #
54
35
  # --
55
36
  # : (untyped) -> Riffer::Guardrails::Result
56
37
  def transform: (untyped) -> Riffer::Guardrails::Result
57
38
 
58
39
  # Creates a block result that halts execution.
59
- #
60
- # [reason] the reason for blocking.
61
- # [metadata] optional additional information.
62
- #
63
40
  # --
64
41
  # : (String, ?metadata: Hash[Symbol, untyped]?) -> Riffer::Guardrails::Result
65
42
  def block: (String, ?metadata: Hash[Symbol, untyped]?) -> Riffer::Guardrails::Result
@@ -1,10 +1,6 @@
1
1
  # Generated from lib/riffer/guardrails/modification.rb with RBS::Inline
2
2
 
3
3
  # Records a guardrail transformation event.
4
- #
5
- # When a guardrail transforms data (via +transform+), a Modification is
6
- # created to record which guardrail made the change, in which phase, and
7
- # which message indices were affected.
8
4
  class Riffer::Guardrails::Modification
9
5
  # The guardrail class that transformed data.
10
6
  attr_reader guardrail: singleton(Riffer::Guardrail)
@@ -15,12 +11,6 @@ class Riffer::Guardrails::Modification
15
11
  # The indices of messages that were changed.
16
12
  attr_reader message_indices: Array[Integer]
17
13
 
18
- # Creates a new modification record.
19
- #
20
- # [guardrail] the guardrail class that transformed.
21
- # [phase] :before or :after.
22
- # [message_indices] indices of changed messages.
23
- #
24
14
  # --
25
15
  # : (guardrail: singleton(Riffer::Guardrail), phase: Symbol, message_indices: Array[Integer]) -> void
26
16
  def initialize: (guardrail: singleton(Riffer::Guardrail), phase: Symbol, message_indices: Array[Integer]) -> void
@@ -1,17 +1,7 @@
1
1
  # Generated from lib/riffer/guardrails/result.rb with RBS::Inline
2
2
 
3
- # Represents the result of a guardrail execution.
4
- #
5
- # Results can be one of three types:
6
- # - pass: Continue with the original data unchanged
7
- # - transform: Continue with transformed data
8
- # - block: Halt execution with a reason
9
- #
10
- # Use the factory methods to create results:
11
- #
12
- # Result.pass(data)
13
- # Result.transform(data)
14
- # Result.block(reason, metadata: nil)
3
+ # Represents the result of a guardrail execution: +pass+ (continue unchanged),
4
+ # +transform+ (continue with changed data), or +block+ (halt with a reason).
15
5
  class Riffer::Guardrails::Result
16
6
  TYPES: Array[Symbol]
17
7
 
@@ -25,38 +15,21 @@ class Riffer::Guardrails::Result
25
15
  attr_reader metadata: Hash[Symbol, untyped]?
26
16
 
27
17
  # Creates a pass result that continues with unchanged data.
28
- #
29
- # [data] the original data to pass through.
30
- #
31
18
  # --
32
19
  # : (untyped) -> Riffer::Guardrails::Result
33
20
  def self.pass: (untyped) -> Riffer::Guardrails::Result
34
21
 
35
22
  # Creates a transform result that continues with transformed data.
36
- #
37
- # [data] the transformed data.
38
- #
39
23
  # --
40
24
  # : (untyped) -> Riffer::Guardrails::Result
41
25
  def self.transform: (untyped) -> Riffer::Guardrails::Result
42
26
 
43
27
  # Creates a block result that halts execution.
44
- #
45
- # [reason] the reason for blocking.
46
- # [metadata] optional additional information.
47
- #
48
28
  # --
49
29
  # : (String, ?metadata: Hash[Symbol, untyped]?) -> Riffer::Guardrails::Result
50
30
  def self.block: (String, ?metadata: Hash[Symbol, untyped]?) -> Riffer::Guardrails::Result
51
31
 
52
- # Creates a new result.
53
- #
54
- # [type] the result type (:pass, :transform, or :block).
55
- # [data] the data or reason.
56
- # [metadata] optional metadata for block results.
57
- #
58
- # Raises Riffer::ArgumentError if the result type is invalid.
59
- #
32
+ # Raises Riffer::ArgumentError if +type+ is not :pass, :transform, or :block.
60
33
  # --
61
34
  # : (Symbol, untyped, ?metadata: Hash[Symbol, untyped]?) -> void
62
35
  def initialize: (Symbol, untyped, ?metadata: Hash[Symbol, untyped]?) -> void
@@ -1,13 +1,7 @@
1
1
  # Generated from lib/riffer/guardrails/runner.rb with RBS::Inline
2
2
 
3
- # Executes guardrails sequentially and manages the processing pipeline.
4
- #
5
- # The runner processes guardrails in order, passing the output of each
6
- # to the next. If any guardrail blocks, execution stops and a tripwire
7
- # is returned.
8
- #
9
- # runner = Runner.new(guardrail_configs, phase: :before, context: context)
10
- # data, tripwire, modifications = runner.run(messages)
3
+ # Executes guardrails sequentially, passing each one's output to the next; if
4
+ # any blocks, execution stops and a tripwire is returned.
11
5
  class Riffer::Guardrails::Runner
12
6
  # The guardrail configs to execute.
13
7
  attr_reader guardrail_configs: Array[Hash[Symbol, untyped]]
@@ -18,24 +12,13 @@ class Riffer::Guardrails::Runner
18
12
  # The context passed to guardrails.
19
13
  attr_reader context: untyped
20
14
 
21
- # Creates a new runner.
22
- #
23
- # [guardrail_configs] configs with :class and :options keys.
24
- # [phase] :before or :after.
25
- # [context] optional context to pass to guardrails.
26
- #
27
15
  # --
28
16
  # : (Array[Hash[Symbol, untyped]], phase: Symbol, ?context: untyped) -> void
29
17
  def initialize: (Array[Hash[Symbol, untyped]], phase: Symbol, ?context: untyped) -> void
30
18
 
31
- # Runs the guardrails sequentially.
32
- #
33
- # For before phase, data should be an array of messages.
34
- # For after phase, data should be a response and messages must be provided.
35
- #
36
- # [data] the data to process (messages for before, response for after).
37
- # [messages] the conversation messages (required for after phase).
38
- #
19
+ # Runs the guardrails sequentially. For the +:before+ phase +data+ is the
20
+ # messages array; for +:after+ it's the response (and +messages+ must be
21
+ # provided).
39
22
  # --
40
23
  # : (untyped, ?messages: Array[Riffer::Messages::Base]?) -> [untyped, Riffer::Guardrails::Tripwire?, Array[Riffer::Guardrails::Modification]]
41
24
  def run: (untyped, ?messages: Array[Riffer::Messages::Base]?) -> [ untyped, Riffer::Guardrails::Tripwire?, Array[Riffer::Guardrails::Modification] ]
@@ -1,16 +1,6 @@
1
1
  # Generated from lib/riffer/guardrails/tripwire.rb with RBS::Inline
2
2
 
3
3
  # Captures information about a blocked guardrail execution.
4
- #
5
- # When a guardrail blocks execution, a Tripwire is created to record
6
- # the reason, which guardrail triggered it, and which phase it occurred in.
7
- #
8
- # tripwire = Tripwire.new(
9
- # reason: "PII detected in input",
10
- # guardrail: PiiRedactor,
11
- # phase: :before,
12
- # metadata: { detected_types: [:email, :phone] }
13
- # )
14
4
  class Riffer::Guardrails::Tripwire
15
5
  PHASES: Array[Symbol]
16
6
 
@@ -26,15 +16,7 @@ class Riffer::Guardrails::Tripwire
26
16
  # Optional metadata about the block.
27
17
  attr_reader metadata: Hash[Symbol, untyped]?
28
18
 
29
- # Creates a new tripwire.
30
- #
31
- # [reason] the reason for blocking.
32
- # [guardrail] the guardrail class that blocked.
33
- # [phase] :before or :after.
34
- # [metadata] optional additional information.
35
- #
36
- # Raises Riffer::ArgumentError if the phase is invalid.
37
- #
19
+ # Raises Riffer::ArgumentError if +phase+ is invalid.
38
20
  # --
39
21
  # : (reason: String, guardrail: singleton(Riffer::Guardrail), phase: Symbol, ?metadata: Hash[Symbol, untyped]?) -> void
40
22
  def initialize: (reason: String, guardrail: singleton(Riffer::Guardrail), phase: Symbol, ?metadata: Hash[Symbol, untyped]?) -> void
@@ -1,9 +1,7 @@
1
1
  # Generated from lib/riffer/guardrails.rb with RBS::Inline
2
2
 
3
- # Namespace module for guardrail components.
4
- #
5
- # Guardrails provide pre-processing of input messages and post-processing
6
- # of output responses in the agent pipeline.
3
+ # Namespace for guardrail components that pre-process input and post-process
4
+ # output in the agent pipeline.
7
5
  module Riffer::Guardrails
8
6
  PHASES: Array[Symbol]
9
7
  end
@@ -1,9 +1,10 @@
1
1
  # Generated from lib/riffer/helpers/call_or_value.rb with RBS::Inline
2
2
 
3
- # Resolves the "Proc-or-value" idiom: if +thing+ is a Proc, calls it
4
- # (passing +context+ when its arity is non-zero); otherwise returns
5
- # +thing+ unchanged. When +thing+ is +nil+, returns +default+.
3
+ # Resolves the Proc-or-value idiom.
6
4
  module Riffer::Helpers::CallOrValue
5
+ # Calls +thing+ when it's a Proc (passing +context+ if its arity is non-zero),
6
+ # returns it unchanged otherwise, or +default+ when +nil+.
7
+ # --
7
8
  # : (untyped, ?context: untyped, ?default: untyped) -> untyped
8
9
  def resolve: (untyped, ?context: untyped, ?default: untyped) -> untyped
9
10
  end
@@ -8,5 +8,5 @@ module Riffer::Helpers::ClassNameConverter
8
8
  #
9
9
  # --
10
10
  # : (String, ?separator: String) -> String
11
- def class_name_to_path: (String, ?separator: String) -> String
11
+ def convert: (String, ?separator: String) -> String
12
12
  end
@@ -1,17 +1,13 @@
1
1
  # Generated from lib/riffer/helpers/dependencies.rb with RBS::Inline
2
2
 
3
- # Helper module for lazy loading gem dependencies.
4
- #
5
- # Used by providers to load their required gems only when needed.
3
+ # Lazy-loads gem dependencies used by providers to load required gems only
4
+ # when needed.
6
5
  module Riffer::Helpers::Dependencies
7
6
  # Raised when a required gem cannot be loaded.
8
7
  class LoadError < ::LoadError
9
8
  end
10
9
 
11
- # Requires a gem by name, raising a helpful error if it is not installed.
12
- #
13
- # Raises LoadError if the gem cannot be required.
14
- #
10
+ # Requires a gem by name; raises LoadError if it isn't installed.
15
11
  # --
16
12
  # : (String) -> true
17
13
  def depends_on: (String) -> true
@@ -1,9 +1,4 @@
1
1
  # Generated from lib/riffer/helpers.rb with RBS::Inline
2
2
 
3
- # Namespace for shared helper modules in the Riffer framework.
4
- #
5
- # Helpers provide reusable functionality across the library:
6
- # - Riffer::Helpers::ClassNameConverter - Class name to path conversion
7
- # - Riffer::Helpers::Dependencies - Lazy gem dependency loading
8
3
  module Riffer::Helpers
9
4
  end
@@ -1,17 +1,18 @@
1
1
  # Generated from lib/riffer/mcp/authenticated_tool.rb with RBS::Inline
2
2
 
3
- # Wraps MCP-generated tool classes so +tools/call+ uses +Riffer.config.mcp.credentials+
4
- # per invocation while delegating metadata to the inner class.
3
+ # Wraps MCP-generated tool classes so +tools/call+ resolves
4
+ # +Riffer.config.mcp.credentials+ per invocation, delegating metadata to the
5
+ # inner class.
5
6
  module Riffer::Mcp::AuthenticatedTool
6
7
  # Returns one wrapper class per inner tool, sharing +manifest+ and +matched_tags+.
7
8
  #
8
9
  # --
9
10
  # : (Array[singleton(Riffer::Tool)], Riffer::Mcp::Manifest, Array[Symbol]) -> Array[singleton(Riffer::Tool)]
10
- def self.wrap_all: (Array[singleton(Riffer::Tool)], Riffer::Mcp::Manifest, Array[Symbol]) -> Array[singleton(Riffer::Tool)]
11
+ def wrap_all: (Array[singleton(Riffer::Tool)], Riffer::Mcp::Manifest, Array[Symbol]) -> Array[singleton(Riffer::Tool)]
11
12
 
12
13
  # --
13
14
  # : (singleton(Riffer::Tool), Riffer::Mcp::Manifest, Array[Symbol]) -> singleton(Riffer::Tool)
14
15
  # Class.new(Riffer::Tool) is typed as ::Class by steep — it cannot verify the subtype
15
16
  # relationship for dynamically created anonymous classes, so the ignore is required.
16
- def self.wrap_one: (singleton(Riffer::Tool), Riffer::Mcp::Manifest, Array[Symbol]) -> singleton(Riffer::Tool)
17
+ def wrap_one: (singleton(Riffer::Tool), Riffer::Mcp::Manifest, Array[Symbol]) -> singleton(Riffer::Tool)
17
18
  end
@@ -1,28 +1,17 @@
1
1
  # Generated from lib/riffer/mcp/client.rb with RBS::Inline
2
2
 
3
- # Thin wrapper around the MCP Ruby SDK client (mcp gem v0.8+).
4
- #
5
- # Resolves headers (if a Proc) once at initialization, then provides
6
- # +tools_list+ and +tools_call+. Used for discovery (+Manifest#discovery_headers+)
7
- # and for +tools/call+ when no +credentials+ proc is configured.
8
- #
9
- # MCP gem API used:
10
- # MCP::Client::HTTP.new(url:, headers:) — HTTP transport (requires faraday)
11
- # MCP::Client.new(transport:) — client
12
- # client.tools — Array<MCP::Client::Tool>
13
- # client.call_tool(tool:, arguments:) — raw JSON-RPC response Hash
3
+ # Thin wrapper around the MCP Ruby SDK client (mcp gem v0.8+). Resolves headers
4
+ # (if a Proc) once at init, then provides +tools_list+ / +tools_call+ — used for
5
+ # discovery and for +tools/call+ when no +credentials+ proc is configured.
14
6
  class Riffer::Mcp::Client
15
- include Riffer::Helpers::Dependencies
16
-
17
7
  @client: untyped
18
8
 
19
9
  # --
20
10
  # : (endpoint: String, ?headers: (Hash[String, String] | Proc), ?client: untyped?) -> void
21
11
  def initialize: (endpoint: String, ?headers: Hash[String, String] | Proc, ?client: untyped?) -> void
22
12
 
23
- # Returns an array of tool definition hashes, each with +:name+, +:description+,
24
- # and +:input_schema+ keys.
25
- #
13
+ # Returns tool definition hashes with +:name+, +:description+, and
14
+ # +:input_schema+ keys.
26
15
  # --
27
16
  # : () -> Array[Hash[Symbol, untyped]]
28
17
  def tools_list: () -> Array[Hash[Symbol, untyped]]
@@ -32,4 +21,9 @@ class Riffer::Mcp::Client
32
21
  # --
33
22
  # : (String, ?Hash[untyped, untyped]) -> String
34
23
  def tools_call: (String, ?Hash[untyped, untyped]) -> String
24
+
25
+ private
26
+
27
+ # : (String) -> true
28
+ def depends_on: (String) -> true
35
29
  end
@@ -1,25 +1,25 @@
1
1
  # Generated from lib/riffer/mcp/manifest.rb with RBS::Inline
2
2
 
3
- # Riffer::Mcp::Manifest holds the configuration for a single MCP server.
4
- #
5
- # +name+ - String identifier used as the registration key and generated-agent identifier.
6
- # +tags+ - Array[Symbol]; normalized to symbols at construction time.
7
- # +endpoint+ - String HTTPS URL passed to the MCP transport.
8
- # +discovery_headers+ - Hash or Proc; resolved once when building the discovery client for +tools/list+.
9
- # +credentials_scope+ - Optional symbol hint: +:global+, +:tenant+, +:user+ — documents whether
10
- # invocation credentials are expected to depend on tenant and/or user keys in +context+ (no ids stored).
11
- # Apps may treat +:user+ as "user in tenant" and pass both keys in +context+.
3
+ # Holds the configuration for a single MCP server.
12
4
  class Riffer::Mcp::Manifest
5
+ # Identifier used as the registration key and generated-agent identifier.
13
6
  attr_reader name: String
14
7
 
8
+ # Tags for matching +use_mcp+.
15
9
  attr_reader tags: Array[Symbol]
16
10
 
11
+ # HTTPS URL passed to the MCP transport.
17
12
  attr_reader endpoint: String
18
13
 
14
+ # Headers (or a Proc) resolved once when building the discovery client.
19
15
  attr_reader discovery_headers: (Hash[String, untyped] | ::Proc)?
20
16
 
17
+ # Optional hint (+:global+/+:tenant+/+:user+) for whether invocation
18
+ # credentials depend on tenant/user keys in +context+.
21
19
  attr_reader credentials_scope: Symbol?
22
20
 
21
+ # Raises Riffer::ArgumentError unless +name+ is present and +endpoint+ is a
22
+ # valid HTTPS URL.
23
23
  # --
24
24
  # : (name: String, endpoint: String, ?tags: Array[untyped]?, ?discovery_headers: (Hash[String, untyped] | ::Proc)?, ?credentials_scope: (String | Symbol)?) -> void
25
25
  def initialize: (name: String, endpoint: String, ?tags: Array[untyped]?, ?discovery_headers: (Hash[String, untyped] | ::Proc)?, ?credentials_scope: (String | Symbol)?) -> void