raif 1.2.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/jobs/raif/conversation_entry_job.rb +1 -1
- data/app/models/raif/agents/re_act_step.rb +1 -2
- data/app/models/raif/concerns/has_llm.rb +1 -1
- data/app/models/raif/concerns/task_run_args.rb +62 -0
- data/app/models/raif/conversation.rb +5 -1
- data/app/models/raif/conversation_entry.rb +6 -8
- data/app/models/raif/llm.rb +1 -1
- data/app/models/raif/llms/open_router.rb +3 -1
- data/app/models/raif/task.rb +22 -9
- data/app/views/raif/conversation_entries/_form.html.erb +1 -1
- data/app/views/raif/conversations/_full_conversation.html.erb +3 -6
- data/app/views/raif/conversations/_initial_chat_message.html.erb +5 -0
- data/config/locales/en.yml +8 -0
- data/db/migrate/20250804013843_add_task_run_args_to_raif_tasks.rb +13 -0
- data/db/migrate/20250811171150_make_raif_task_creator_optional.rb +8 -0
- data/exe/raif +7 -0
- data/lib/generators/raif/agent/agent_generator.rb +22 -7
- data/lib/generators/raif/agent/templates/agent.rb.tt +20 -24
- data/lib/generators/raif/agent/templates/agent_eval_set.rb.tt +48 -0
- data/lib/generators/raif/agent/templates/application_agent.rb.tt +0 -2
- data/lib/generators/raif/base_generator.rb +19 -0
- data/lib/generators/raif/conversation/conversation_generator.rb +21 -2
- data/lib/generators/raif/conversation/templates/application_conversation.rb.tt +0 -2
- data/lib/generators/raif/conversation/templates/conversation.rb.tt +29 -33
- data/lib/generators/raif/conversation/templates/conversation_eval_set.rb.tt +70 -0
- data/lib/generators/raif/eval_set/eval_set_generator.rb +28 -0
- data/lib/generators/raif/eval_set/templates/eval_set.rb.tt +21 -0
- data/lib/generators/raif/evals/setup/setup_generator.rb +47 -0
- data/lib/generators/raif/install/install_generator.rb +15 -0
- data/lib/generators/raif/install/templates/initializer.rb +11 -0
- data/lib/generators/raif/model_tool/model_tool_generator.rb +5 -5
- data/lib/generators/raif/model_tool/templates/model_tool.rb.tt +78 -78
- data/lib/generators/raif/model_tool/templates/model_tool_invocation_partial.html.erb.tt +1 -1
- data/lib/generators/raif/task/task_generator.rb +22 -3
- data/lib/generators/raif/task/templates/application_task.rb.tt +0 -2
- data/lib/generators/raif/task/templates/task.rb.tt +55 -59
- data/lib/generators/raif/task/templates/task_eval_set.rb.tt +54 -0
- data/lib/raif/cli/base.rb +39 -0
- data/lib/raif/cli/evals.rb +47 -0
- data/lib/raif/cli/evals_setup.rb +27 -0
- data/lib/raif/cli.rb +67 -0
- data/lib/raif/configuration.rb +20 -6
- data/lib/raif/evals/eval.rb +30 -0
- data/lib/raif/evals/eval_set.rb +111 -0
- data/lib/raif/evals/eval_sets/expectations.rb +53 -0
- data/lib/raif/evals/eval_sets/llm_judge_expectations.rb +255 -0
- data/lib/raif/evals/expectation_result.rb +39 -0
- data/lib/raif/evals/llm_judge.rb +32 -0
- data/lib/raif/evals/llm_judges/binary.rb +94 -0
- data/lib/raif/evals/llm_judges/comparative.rb +89 -0
- data/lib/raif/evals/llm_judges/scored.rb +63 -0
- data/lib/raif/evals/llm_judges/summarization.rb +166 -0
- data/lib/raif/evals/run.rb +201 -0
- data/lib/raif/evals/scoring_rubric.rb +174 -0
- data/lib/raif/evals.rb +26 -0
- data/lib/raif/llm_registry.rb +33 -0
- data/lib/raif/migration_checker.rb +3 -3
- data/lib/raif/utils/colors.rb +23 -0
- data/lib/raif/utils.rb +1 -0
- data/lib/raif/version.rb +1 -1
- data/lib/raif.rb +4 -0
- data/spec/support/current_temperature_test_tool.rb +34 -0
- data/spec/support/test_conversation.rb +1 -1
- metadata +35 -3
@@ -0,0 +1,34 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class Raif::ModelTools::CurrentTemperatureTestTool < Raif::ModelTool
|
4
|
+
tool_arguments_schema do
|
5
|
+
string :zip_code, description: "The zip code to get the current temperature for"
|
6
|
+
end
|
7
|
+
|
8
|
+
tool_description do
|
9
|
+
"A tool to get the current temperature for a given zip code"
|
10
|
+
end
|
11
|
+
|
12
|
+
class << self
|
13
|
+
def process_invocation(tool_invocation)
|
14
|
+
tool_invocation.update!(
|
15
|
+
result: {
|
16
|
+
temperature: 72
|
17
|
+
}
|
18
|
+
)
|
19
|
+
|
20
|
+
tool_invocation.result
|
21
|
+
end
|
22
|
+
|
23
|
+
def triggers_observation_to_model?
|
24
|
+
true
|
25
|
+
end
|
26
|
+
|
27
|
+
def observation_for_invocation(tool_invocation)
|
28
|
+
zip_code = tool_invocation.tool_arguments["zip_code"]
|
29
|
+
temperature = tool_invocation.result["temperature"]
|
30
|
+
|
31
|
+
"The current temperature for zip code #{zip_code} is #{temperature} degrees Fahrenheit."
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
metadata
CHANGED
@@ -1,12 +1,12 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: raif
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ben Roesch
|
8
8
|
- Brian Leslie
|
9
|
-
bindir:
|
9
|
+
bindir: exe
|
10
10
|
cert_chain: []
|
11
11
|
date: 1980-01-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
@@ -142,7 +142,8 @@ description: Raif (Ruby AI Framework) is a Rails engine that helps you add AI-po
|
|
142
142
|
email:
|
143
143
|
- ben@cultivatelabs.com
|
144
144
|
- brian@cultivatelabs.com
|
145
|
-
executables:
|
145
|
+
executables:
|
146
|
+
- raif
|
146
147
|
extensions: []
|
147
148
|
extra_rdoc_files: []
|
148
149
|
files:
|
@@ -199,6 +200,7 @@ files:
|
|
199
200
|
- app/models/raif/concerns/llms/open_ai_completions/tool_formatting.rb
|
200
201
|
- app/models/raif/concerns/llms/open_ai_responses/message_formatting.rb
|
201
202
|
- app/models/raif/concerns/llms/open_ai_responses/tool_formatting.rb
|
203
|
+
- app/models/raif/concerns/task_run_args.rb
|
202
204
|
- app/models/raif/conversation.rb
|
203
205
|
- app/models/raif/conversation_entry.rb
|
204
206
|
- app/models/raif/embedding_model.rb
|
@@ -261,6 +263,7 @@ files:
|
|
261
263
|
- app/views/raif/conversation_entries/new.turbo_stream.erb
|
262
264
|
- app/views/raif/conversations/_available_user_tools.html.erb
|
263
265
|
- app/views/raif/conversations/_full_conversation.html.erb
|
266
|
+
- app/views/raif/conversations/_initial_chat_message.html.erb
|
264
267
|
- app/views/raif/conversations/show.html.erb
|
265
268
|
- config/i18n-tasks.yml
|
266
269
|
- config/importmap.rb
|
@@ -277,12 +280,21 @@ files:
|
|
277
280
|
- db/migrate/20250527213016_add_response_id_and_response_array_to_model_completions.rb
|
278
281
|
- db/migrate/20250603140622_add_citations_to_raif_model_completions.rb
|
279
282
|
- db/migrate/20250603202013_add_stream_response_to_raif_model_completions.rb
|
283
|
+
- db/migrate/20250804013843_add_task_run_args_to_raif_tasks.rb
|
284
|
+
- db/migrate/20250811171150_make_raif_task_creator_optional.rb
|
285
|
+
- exe/raif
|
280
286
|
- lib/generators/raif/agent/agent_generator.rb
|
281
287
|
- lib/generators/raif/agent/templates/agent.rb.tt
|
288
|
+
- lib/generators/raif/agent/templates/agent_eval_set.rb.tt
|
282
289
|
- lib/generators/raif/agent/templates/application_agent.rb.tt
|
290
|
+
- lib/generators/raif/base_generator.rb
|
283
291
|
- lib/generators/raif/conversation/conversation_generator.rb
|
284
292
|
- lib/generators/raif/conversation/templates/application_conversation.rb.tt
|
285
293
|
- lib/generators/raif/conversation/templates/conversation.rb.tt
|
294
|
+
- lib/generators/raif/conversation/templates/conversation_eval_set.rb.tt
|
295
|
+
- lib/generators/raif/eval_set/eval_set_generator.rb
|
296
|
+
- lib/generators/raif/eval_set/templates/eval_set.rb.tt
|
297
|
+
- lib/generators/raif/evals/setup/setup_generator.rb
|
286
298
|
- lib/generators/raif/install/install_generator.rb
|
287
299
|
- lib/generators/raif/install/templates/initializer.rb
|
288
300
|
- lib/generators/raif/model_tool/model_tool_generator.rb
|
@@ -291,8 +303,13 @@ files:
|
|
291
303
|
- lib/generators/raif/task/task_generator.rb
|
292
304
|
- lib/generators/raif/task/templates/application_task.rb.tt
|
293
305
|
- lib/generators/raif/task/templates/task.rb.tt
|
306
|
+
- lib/generators/raif/task/templates/task_eval_set.rb.tt
|
294
307
|
- lib/generators/raif/views_generator.rb
|
295
308
|
- lib/raif.rb
|
309
|
+
- lib/raif/cli.rb
|
310
|
+
- lib/raif/cli/base.rb
|
311
|
+
- lib/raif/cli/evals.rb
|
312
|
+
- lib/raif/cli/evals_setup.rb
|
296
313
|
- lib/raif/configuration.rb
|
297
314
|
- lib/raif/embedding_model_registry.rb
|
298
315
|
- lib/raif/engine.rb
|
@@ -306,18 +323,33 @@ files:
|
|
306
323
|
- lib/raif/errors/open_ai/json_schema_error.rb
|
307
324
|
- lib/raif/errors/streaming_error.rb
|
308
325
|
- lib/raif/errors/unsupported_feature_error.rb
|
326
|
+
- lib/raif/evals.rb
|
327
|
+
- lib/raif/evals/eval.rb
|
328
|
+
- lib/raif/evals/eval_set.rb
|
329
|
+
- lib/raif/evals/eval_sets/expectations.rb
|
330
|
+
- lib/raif/evals/eval_sets/llm_judge_expectations.rb
|
331
|
+
- lib/raif/evals/expectation_result.rb
|
332
|
+
- lib/raif/evals/llm_judge.rb
|
333
|
+
- lib/raif/evals/llm_judges/binary.rb
|
334
|
+
- lib/raif/evals/llm_judges/comparative.rb
|
335
|
+
- lib/raif/evals/llm_judges/scored.rb
|
336
|
+
- lib/raif/evals/llm_judges/summarization.rb
|
337
|
+
- lib/raif/evals/run.rb
|
338
|
+
- lib/raif/evals/scoring_rubric.rb
|
309
339
|
- lib/raif/json_schema_builder.rb
|
310
340
|
- lib/raif/languages.rb
|
311
341
|
- lib/raif/llm_registry.rb
|
312
342
|
- lib/raif/migration_checker.rb
|
313
343
|
- lib/raif/rspec.rb
|
314
344
|
- lib/raif/utils.rb
|
345
|
+
- lib/raif/utils/colors.rb
|
315
346
|
- lib/raif/utils/html_fragment_processor.rb
|
316
347
|
- lib/raif/utils/html_to_markdown_converter.rb
|
317
348
|
- lib/raif/utils/readable_content_extractor.rb
|
318
349
|
- lib/raif/version.rb
|
319
350
|
- lib/tasks/raif_tasks.rake
|
320
351
|
- spec/support/complex_test_tool.rb
|
352
|
+
- spec/support/current_temperature_test_tool.rb
|
321
353
|
- spec/support/rspec_helpers.rb
|
322
354
|
- spec/support/test_conversation.rb
|
323
355
|
- spec/support/test_embedding_model.rb
|