raif 1.2.2 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. data/app/jobs/raif/conversation_entry_job.rb +1 -1
  3. data/app/models/raif/agents/re_act_step.rb +1 -2
  4. data/app/models/raif/concerns/has_llm.rb +1 -1
  5. data/app/models/raif/concerns/task_run_args.rb +62 -0
  6. data/app/models/raif/conversation.rb +5 -1
  7. data/app/models/raif/conversation_entry.rb +6 -8
  8. data/app/models/raif/llm.rb +1 -1
  9. data/app/models/raif/llms/open_router.rb +3 -1
  10. data/app/models/raif/task.rb +22 -9
  11. data/app/views/raif/conversation_entries/_form.html.erb +1 -1
  12. data/app/views/raif/conversations/_full_conversation.html.erb +3 -6
  13. data/app/views/raif/conversations/_initial_chat_message.html.erb +5 -0
  14. data/config/locales/en.yml +8 -0
  15. data/db/migrate/20250804013843_add_task_run_args_to_raif_tasks.rb +13 -0
  16. data/db/migrate/20250811171150_make_raif_task_creator_optional.rb +8 -0
  17. data/exe/raif +7 -0
  18. data/lib/generators/raif/agent/agent_generator.rb +22 -7
  19. data/lib/generators/raif/agent/templates/agent.rb.tt +20 -24
  20. data/lib/generators/raif/agent/templates/agent_eval_set.rb.tt +48 -0
  21. data/lib/generators/raif/agent/templates/application_agent.rb.tt +0 -2
  22. data/lib/generators/raif/base_generator.rb +19 -0
  23. data/lib/generators/raif/conversation/conversation_generator.rb +21 -2
  24. data/lib/generators/raif/conversation/templates/application_conversation.rb.tt +0 -2
  25. data/lib/generators/raif/conversation/templates/conversation.rb.tt +29 -33
  26. data/lib/generators/raif/conversation/templates/conversation_eval_set.rb.tt +70 -0
  27. data/lib/generators/raif/eval_set/eval_set_generator.rb +28 -0
  28. data/lib/generators/raif/eval_set/templates/eval_set.rb.tt +21 -0
  29. data/lib/generators/raif/evals/setup/setup_generator.rb +47 -0
  30. data/lib/generators/raif/install/install_generator.rb +15 -0
  31. data/lib/generators/raif/install/templates/initializer.rb +11 -0
  32. data/lib/generators/raif/model_tool/model_tool_generator.rb +5 -5
  33. data/lib/generators/raif/model_tool/templates/model_tool.rb.tt +78 -78
  34. data/lib/generators/raif/model_tool/templates/model_tool_invocation_partial.html.erb.tt +1 -1
  35. data/lib/generators/raif/task/task_generator.rb +22 -3
  36. data/lib/generators/raif/task/templates/application_task.rb.tt +0 -2
  37. data/lib/generators/raif/task/templates/task.rb.tt +55 -59
  38. data/lib/generators/raif/task/templates/task_eval_set.rb.tt +54 -0
  39. data/lib/raif/cli/base.rb +39 -0
  40. data/lib/raif/cli/evals.rb +47 -0
  41. data/lib/raif/cli/evals_setup.rb +27 -0
  42. data/lib/raif/cli.rb +67 -0
  43. data/lib/raif/configuration.rb +20 -6
  44. data/lib/raif/evals/eval.rb +30 -0
  45. data/lib/raif/evals/eval_set.rb +111 -0
  46. data/lib/raif/evals/eval_sets/expectations.rb +53 -0
  47. data/lib/raif/evals/eval_sets/llm_judge_expectations.rb +255 -0
  48. data/lib/raif/evals/expectation_result.rb +39 -0
  49. data/lib/raif/evals/llm_judge.rb +32 -0
  50. data/lib/raif/evals/llm_judges/binary.rb +94 -0
  51. data/lib/raif/evals/llm_judges/comparative.rb +89 -0
  52. data/lib/raif/evals/llm_judges/scored.rb +63 -0
  53. data/lib/raif/evals/llm_judges/summarization.rb +166 -0
  54. data/lib/raif/evals/run.rb +201 -0
  55. data/lib/raif/evals/scoring_rubric.rb +174 -0
  56. data/lib/raif/evals.rb +26 -0
  57. data/lib/raif/llm_registry.rb +33 -0
  58. data/lib/raif/migration_checker.rb +3 -3
  59. data/lib/raif/utils/colors.rb +23 -0
  60. data/lib/raif/utils.rb +1 -0
  61. data/lib/raif/version.rb +1 -1
  62. data/lib/raif.rb +4 -0
  63. data/spec/support/current_temperature_test_tool.rb +34 -0
  64. data/spec/support/test_conversation.rb +1 -1
  65. metadata +35 -3
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ class Raif::ModelTools::CurrentTemperatureTestTool < Raif::ModelTool
4
+ tool_arguments_schema do
5
+ string :zip_code, description: "The zip code to get the current temperature for"
6
+ end
7
+
8
+ tool_description do
9
+ "A tool to get the current temperature for a given zip code"
10
+ end
11
+
12
+ class << self
13
+ def process_invocation(tool_invocation)
14
+ tool_invocation.update!(
15
+ result: {
16
+ temperature: 72
17
+ }
18
+ )
19
+
20
+ tool_invocation.result
21
+ end
22
+
23
+ def triggers_observation_to_model?
24
+ true
25
+ end
26
+
27
+ def observation_for_invocation(tool_invocation)
28
+ zip_code = tool_invocation.tool_arguments["zip_code"]
29
+ temperature = tool_invocation.result["temperature"]
30
+
31
+ "The current temperature for zip code #{zip_code} is #{temperature} degrees Fahrenheit."
32
+ end
33
+ end
34
+ end
@@ -12,7 +12,7 @@ class Raif::TestConversation < Raif::Conversation
12
12
  end
13
13
 
14
14
  def process_model_response_message(message:, entry:)
15
- message.gsub("jerk", "[REDACTED]")
15
+ message&.gsub("jerk", "[REDACTED]")
16
16
  end
17
17
 
18
18
  end
metadata CHANGED
@@ -1,12 +1,12 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: raif
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.2
4
+ version: 1.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ben Roesch
8
8
  - Brian Leslie
9
- bindir: bin
9
+ bindir: exe
10
10
  cert_chain: []
11
11
  date: 1980-01-02 00:00:00.000000000 Z
12
12
  dependencies:
@@ -142,7 +142,8 @@ description: Raif (Ruby AI Framework) is a Rails engine that helps you add AI-po
142
142
  email:
143
143
  - ben@cultivatelabs.com
144
144
  - brian@cultivatelabs.com
145
- executables: []
145
+ executables:
146
+ - raif
146
147
  extensions: []
147
148
  extra_rdoc_files: []
148
149
  files:
@@ -199,6 +200,7 @@ files:
199
200
  - app/models/raif/concerns/llms/open_ai_completions/tool_formatting.rb
200
201
  - app/models/raif/concerns/llms/open_ai_responses/message_formatting.rb
201
202
  - app/models/raif/concerns/llms/open_ai_responses/tool_formatting.rb
203
+ - app/models/raif/concerns/task_run_args.rb
202
204
  - app/models/raif/conversation.rb
203
205
  - app/models/raif/conversation_entry.rb
204
206
  - app/models/raif/embedding_model.rb
@@ -261,6 +263,7 @@ files:
261
263
  - app/views/raif/conversation_entries/new.turbo_stream.erb
262
264
  - app/views/raif/conversations/_available_user_tools.html.erb
263
265
  - app/views/raif/conversations/_full_conversation.html.erb
266
+ - app/views/raif/conversations/_initial_chat_message.html.erb
264
267
  - app/views/raif/conversations/show.html.erb
265
268
  - config/i18n-tasks.yml
266
269
  - config/importmap.rb
@@ -277,12 +280,21 @@ files:
277
280
  - db/migrate/20250527213016_add_response_id_and_response_array_to_model_completions.rb
278
281
  - db/migrate/20250603140622_add_citations_to_raif_model_completions.rb
279
282
  - db/migrate/20250603202013_add_stream_response_to_raif_model_completions.rb
283
+ - db/migrate/20250804013843_add_task_run_args_to_raif_tasks.rb
284
+ - db/migrate/20250811171150_make_raif_task_creator_optional.rb
285
+ - exe/raif
280
286
  - lib/generators/raif/agent/agent_generator.rb
281
287
  - lib/generators/raif/agent/templates/agent.rb.tt
288
+ - lib/generators/raif/agent/templates/agent_eval_set.rb.tt
282
289
  - lib/generators/raif/agent/templates/application_agent.rb.tt
290
+ - lib/generators/raif/base_generator.rb
283
291
  - lib/generators/raif/conversation/conversation_generator.rb
284
292
  - lib/generators/raif/conversation/templates/application_conversation.rb.tt
285
293
  - lib/generators/raif/conversation/templates/conversation.rb.tt
294
+ - lib/generators/raif/conversation/templates/conversation_eval_set.rb.tt
295
+ - lib/generators/raif/eval_set/eval_set_generator.rb
296
+ - lib/generators/raif/eval_set/templates/eval_set.rb.tt
297
+ - lib/generators/raif/evals/setup/setup_generator.rb
286
298
  - lib/generators/raif/install/install_generator.rb
287
299
  - lib/generators/raif/install/templates/initializer.rb
288
300
  - lib/generators/raif/model_tool/model_tool_generator.rb
@@ -291,8 +303,13 @@ files:
291
303
  - lib/generators/raif/task/task_generator.rb
292
304
  - lib/generators/raif/task/templates/application_task.rb.tt
293
305
  - lib/generators/raif/task/templates/task.rb.tt
306
+ - lib/generators/raif/task/templates/task_eval_set.rb.tt
294
307
  - lib/generators/raif/views_generator.rb
295
308
  - lib/raif.rb
309
+ - lib/raif/cli.rb
310
+ - lib/raif/cli/base.rb
311
+ - lib/raif/cli/evals.rb
312
+ - lib/raif/cli/evals_setup.rb
296
313
  - lib/raif/configuration.rb
297
314
  - lib/raif/embedding_model_registry.rb
298
315
  - lib/raif/engine.rb
@@ -306,18 +323,33 @@ files:
306
323
  - lib/raif/errors/open_ai/json_schema_error.rb
307
324
  - lib/raif/errors/streaming_error.rb
308
325
  - lib/raif/errors/unsupported_feature_error.rb
326
+ - lib/raif/evals.rb
327
+ - lib/raif/evals/eval.rb
328
+ - lib/raif/evals/eval_set.rb
329
+ - lib/raif/evals/eval_sets/expectations.rb
330
+ - lib/raif/evals/eval_sets/llm_judge_expectations.rb
331
+ - lib/raif/evals/expectation_result.rb
332
+ - lib/raif/evals/llm_judge.rb
333
+ - lib/raif/evals/llm_judges/binary.rb
334
+ - lib/raif/evals/llm_judges/comparative.rb
335
+ - lib/raif/evals/llm_judges/scored.rb
336
+ - lib/raif/evals/llm_judges/summarization.rb
337
+ - lib/raif/evals/run.rb
338
+ - lib/raif/evals/scoring_rubric.rb
309
339
  - lib/raif/json_schema_builder.rb
310
340
  - lib/raif/languages.rb
311
341
  - lib/raif/llm_registry.rb
312
342
  - lib/raif/migration_checker.rb
313
343
  - lib/raif/rspec.rb
314
344
  - lib/raif/utils.rb
345
+ - lib/raif/utils/colors.rb
315
346
  - lib/raif/utils/html_fragment_processor.rb
316
347
  - lib/raif/utils/html_to_markdown_converter.rb
317
348
  - lib/raif/utils/readable_content_extractor.rb
318
349
  - lib/raif/version.rb
319
350
  - lib/tasks/raif_tasks.rake
320
351
  - spec/support/complex_test_tool.rb
352
+ - spec/support/current_temperature_test_tool.rb
321
353
  - spec/support/rspec_helpers.rb
322
354
  - spec/support/test_conversation.rb
323
355
  - spec/support/test_embedding_model.rb