raif 1.2.2 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. data/app/jobs/raif/conversation_entry_job.rb +1 -1
  3. data/app/models/raif/agents/re_act_step.rb +1 -2
  4. data/app/models/raif/concerns/has_llm.rb +1 -1
  5. data/app/models/raif/concerns/task_run_args.rb +62 -0
  6. data/app/models/raif/conversation.rb +5 -1
  7. data/app/models/raif/conversation_entry.rb +6 -8
  8. data/app/models/raif/llm.rb +1 -1
  9. data/app/models/raif/llms/open_router.rb +3 -1
  10. data/app/models/raif/task.rb +22 -9
  11. data/app/views/raif/conversation_entries/_form.html.erb +1 -1
  12. data/app/views/raif/conversations/_full_conversation.html.erb +3 -6
  13. data/app/views/raif/conversations/_initial_chat_message.html.erb +5 -0
  14. data/config/locales/en.yml +8 -0
  15. data/db/migrate/20250804013843_add_task_run_args_to_raif_tasks.rb +13 -0
  16. data/db/migrate/20250811171150_make_raif_task_creator_optional.rb +8 -0
  17. data/exe/raif +7 -0
  18. data/lib/generators/raif/agent/agent_generator.rb +22 -7
  19. data/lib/generators/raif/agent/templates/agent.rb.tt +20 -24
  20. data/lib/generators/raif/agent/templates/agent_eval_set.rb.tt +48 -0
  21. data/lib/generators/raif/agent/templates/application_agent.rb.tt +0 -2
  22. data/lib/generators/raif/base_generator.rb +19 -0
  23. data/lib/generators/raif/conversation/conversation_generator.rb +21 -2
  24. data/lib/generators/raif/conversation/templates/application_conversation.rb.tt +0 -2
  25. data/lib/generators/raif/conversation/templates/conversation.rb.tt +29 -33
  26. data/lib/generators/raif/conversation/templates/conversation_eval_set.rb.tt +70 -0
  27. data/lib/generators/raif/eval_set/eval_set_generator.rb +28 -0
  28. data/lib/generators/raif/eval_set/templates/eval_set.rb.tt +21 -0
  29. data/lib/generators/raif/evals/setup/setup_generator.rb +47 -0
  30. data/lib/generators/raif/install/install_generator.rb +15 -0
  31. data/lib/generators/raif/install/templates/initializer.rb +11 -0
  32. data/lib/generators/raif/model_tool/model_tool_generator.rb +5 -5
  33. data/lib/generators/raif/model_tool/templates/model_tool.rb.tt +78 -78
  34. data/lib/generators/raif/model_tool/templates/model_tool_invocation_partial.html.erb.tt +1 -1
  35. data/lib/generators/raif/task/task_generator.rb +22 -3
  36. data/lib/generators/raif/task/templates/application_task.rb.tt +0 -2
  37. data/lib/generators/raif/task/templates/task.rb.tt +55 -59
  38. data/lib/generators/raif/task/templates/task_eval_set.rb.tt +54 -0
  39. data/lib/raif/cli/base.rb +39 -0
  40. data/lib/raif/cli/evals.rb +47 -0
  41. data/lib/raif/cli/evals_setup.rb +27 -0
  42. data/lib/raif/cli.rb +67 -0
  43. data/lib/raif/configuration.rb +20 -6
  44. data/lib/raif/evals/eval.rb +30 -0
  45. data/lib/raif/evals/eval_set.rb +111 -0
  46. data/lib/raif/evals/eval_sets/expectations.rb +53 -0
  47. data/lib/raif/evals/eval_sets/llm_judge_expectations.rb +255 -0
  48. data/lib/raif/evals/expectation_result.rb +39 -0
  49. data/lib/raif/evals/llm_judge.rb +32 -0
  50. data/lib/raif/evals/llm_judges/binary.rb +94 -0
  51. data/lib/raif/evals/llm_judges/comparative.rb +89 -0
  52. data/lib/raif/evals/llm_judges/scored.rb +63 -0
  53. data/lib/raif/evals/llm_judges/summarization.rb +166 -0
  54. data/lib/raif/evals/run.rb +201 -0
  55. data/lib/raif/evals/scoring_rubric.rb +174 -0
  56. data/lib/raif/evals.rb +26 -0
  57. data/lib/raif/llm_registry.rb +33 -0
  58. data/lib/raif/migration_checker.rb +3 -3
  59. data/lib/raif/utils/colors.rb +23 -0
  60. data/lib/raif/utils.rb +1 -0
  61. data/lib/raif/version.rb +1 -1
  62. data/lib/raif.rb +4 -0
  63. data/spec/support/current_temperature_test_tool.rb +34 -0
  64. data/spec/support/test_conversation.rb +1 -1
  65. metadata +35 -3
@@ -1,39 +1,35 @@
1
- # frozen_string_literal: true
1
+ <% raif_module_namespacing(["Conversations"]) do -%>
2
+ class <%= class_name.demodulize %> < Raif::ApplicationConversation
3
+ # Set the response format for the conversation. Options are :html, :text, or :json.
4
+ # If you set this to something other than :text, make sure to include instructions to the model in your system prompt
5
+ llm_response_format :<%= options[:response_format] %>
2
6
 
3
- module Raif
4
- module Conversations
5
- class <%= class_name %> < Raif::ApplicationConversation
6
- # Set the response format for the task. Options are :html, :text, or :json.
7
- # If you set this to something other than :text, make sure to include instructions to the model in your system prompt
8
- llm_response_format :<%= options[:response_format] %>
7
+ # If you want to always include a certain set of model tools with this conversation type,
8
+ # uncomment this callback to populate the available_model_tools attribute with your desired model tools.
9
+ # before_create -> { self.available_model_tools = ["Raif::ModelTools::Example"] }
9
10
 
10
- # If you want to always include a certain set of model tools with this conversation type,
11
- # uncomment this callback to populate the available_model_tools attribute with your desired model tools.
12
- # before_create -> { self.available_model_tools = ["Raif::ModelTools::Example"] }
11
+ # Override the methods below to customize the system prompt for this conversation type.
12
+ # def system_prompt_intro
13
+ # Raif.config.conversation_system_prompt_intro
14
+ # end
13
15
 
14
- # Override the methods below to customize the system prompt for this conversation type.
15
- # def system_prompt_intro
16
- # Raif.config.conversation_system_prompt_intro
17
- # end
16
+ # def build_system_prompt
17
+ # <<~PROMPT
18
+ # #{system_prompt_intro}
19
+ # #{system_prompt_language_preference}
20
+ # PROMPT
21
+ # end
18
22
 
19
- # def build_system_prompt
20
- # <<~PROMPT
21
- # #{system_prompt_intro}
22
- # #{system_prompt_language_preference}
23
- # PROMPT
24
- # end
23
+ # Override this method to set the initial message shown to the user.
24
+ # def initial_chat_message
25
+ # I18n.t("#{self.class.name.underscore.gsub("/", ".")}.initial_chat_message")
26
+ # end
25
27
 
26
- # Override this method to set the initial message shown to the user.
27
- # def initial_chat_message
28
- # I18n.t("#{self.class.name.underscore.gsub("/", ".")}.initial_chat_message")
29
- # end
30
-
31
- # This method will be called when receing a model response to a Raif::ConversationEntry
32
- # By default, it just passes the model response message through, but you can override
33
- # for custom response message processing
34
- # def process_model_response_message(message:, entry:)
35
- # message
36
- # end
37
- end
28
+ # This method will be called when receing a model response to a Raif::ConversationEntry
29
+ # By default, it just passes the model response message through, but you can override
30
+ # for custom response message processing
31
+ # def process_model_response_message(message:, entry:)
32
+ # message
33
+ # end
38
34
  end
39
- end
35
+ <% end -%>
@@ -0,0 +1,70 @@
1
+ <% raif_module_namespacing(["Evals", "Conversations"]) do -%>
2
+ class <%= class_name.demodulize %>EvalSet < Raif::Evals::EvalSet
3
+ # Run this eval set with:
4
+ # bundle exec raif evals ./<%= eval_set_file_path %>
5
+
6
+ # Setup method runs before each eval
7
+ setup do
8
+ # Common setup code
9
+ # @user = User.create!(email: "test@example.com")
10
+ # @conversation = Raif::Conversations::<%= class_name %>.create!(creator: @user)
11
+ end
12
+
13
+ # Teardown runs after each eval
14
+ teardown do
15
+ # Cleanup code
16
+ end
17
+
18
+ eval "<%= class_name %> responds appropriately to user greeting" do
19
+ # entry = @conversation.entries.create!(
20
+ # user_message: "Hello, how are you?",
21
+ # creator: @user
22
+ # )
23
+
24
+ # entry.process_entry!
25
+
26
+ # expect "generates a response" do
27
+ # entry.model_response_message.present?
28
+ # end
29
+
30
+ # expect "response is friendly" do
31
+ # entry.model_response_message.match?(/hello|hi|greetings/i)
32
+ # end
33
+ end
34
+
35
+ eval "<%= class_name %> maintains conversation context" do
36
+ # First message establishes context
37
+ # first_entry = @conversation.entries.create!(
38
+ # user_message: "My name is Alice",
39
+ # creator: @user
40
+ # )
41
+ # first_entry.process_entry!
42
+
43
+ # Second message references context
44
+ # second_entry = @conversation.entries.create!(
45
+ # user_message: "What's my name?",
46
+ # creator: @user
47
+ # )
48
+ # second_entry.process_entry!
49
+
50
+ # expect "remembers the user's name" do
51
+ # second_entry.model_response_message.include?("Alice")
52
+ # end
53
+ end
54
+
55
+ eval "<%= class_name %> handles tool invocations correctly" do
56
+ # Test if your conversation uses tools
57
+ # @conversation.update!(available_model_tools: [ "Raif::ModelTools::FetchUrl" ])
58
+
59
+ # entry = @conversation.entries.create!(
60
+ # user_message: "What can you tell me about the content of https://en.wikipedia.org/wiki/Moon",
61
+ # creator: @user
62
+ # )
63
+
64
+ # entry.process_entry!
65
+
66
+ # expect_tool_invocation(entry, "Raif::ModelTools::FetchUrl", with: { url: "https://en.wikipedia.org/wiki/Moon" })
67
+ end
68
+
69
+ end
70
+ <% end -%>
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../base_generator"
4
+
5
+ module Raif
6
+ module Generators
7
+ class EvalSetGenerator < BaseGenerator
8
+ source_root File.expand_path("templates", __dir__)
9
+
10
+ def create_eval_set_file
11
+ template "eval_set.rb.tt", eval_set_file_path
12
+ end
13
+
14
+ def show_instructions
15
+ say "\nEval set created!"
16
+ say "To run this eval set: bundle exec raif evals ./#{eval_set_file_path}"
17
+ say "To run all eval sets: bundle exec raif evals"
18
+ say ""
19
+ end
20
+
21
+ private
22
+
23
+ def eval_set_file_path
24
+ File.join("raif_evals", "eval_sets", class_path, "#{file_name}_eval_set.rb")
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,21 @@
1
+ <% raif_module_namespacing(["Evals"]) do -%>
2
+ class <%= class_name.demodulize %>EvalSet < Raif::Evals::EvalSet
3
+ # Run this eval set with:
4
+ # bundle exec raif evals ./<%= eval_set_file_path %>
5
+
6
+ # Setup method runs before each eval
7
+ setup do
8
+ # Common setup code
9
+ end
10
+
11
+ # Teardown runs after each eval
12
+ teardown do
13
+ # Cleanup code
14
+ end
15
+
16
+ eval "description of your eval" do
17
+ # Your eval code here
18
+ # expect_tool_invocation, expect_no_tool_invocation, expect, etc.
19
+ end
20
+ end
21
+ <% end -%>
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "rails/generators"
4
+
5
+ module Raif
6
+ module Generators
7
+ module Evals
8
+ class SetupGenerator < Rails::Generators::Base
9
+ source_root File.expand_path("templates", __dir__)
10
+
11
+ def create_directories
12
+ empty_directory "raif_evals"
13
+ empty_directory "raif_evals/eval_sets"
14
+ empty_directory "raif_evals/files"
15
+ empty_directory "raif_evals/results"
16
+ end
17
+
18
+ def create_setup_file
19
+ create_file "raif_evals/setup.rb", <<~EOS
20
+ #
21
+ # This file is loaded at the start of a run of your evals.
22
+ #
23
+ # Add any setup code that should run before your evals.
24
+ #
25
+ EOS
26
+ end
27
+
28
+ def create_gitignore
29
+ create_file "raif_evals/results/.gitignore", <<~EOS
30
+ *
31
+ !.gitignore
32
+ EOS
33
+ end
34
+
35
+ def show_instructions
36
+ say "\nRaif evals setup complete!", :green
37
+ say "You can create evals with: rails g raif:eval_set ExampleName"
38
+ say ""
39
+ say "Run evals with:"
40
+ say " bundle exec raif evals # Run all evals"
41
+ say " bundle exec raif evals ./raif_evals/eval_sets/my_eval_set.rb # Run specific eval set"
42
+ say ""
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
@@ -24,8 +24,23 @@ module Raif
24
24
  end
25
25
 
26
26
  def add_engine_route
27
+ routes_file = "config/routes.rb"
28
+
29
+ if File.exist?(routes_file)
30
+ routes_content = File.read(routes_file)
31
+ if routes_content.include?("mount Raif::Engine")
32
+ say "Raif is already mounted in #{routes_file}, skipping route", :yellow
33
+ return
34
+ end
35
+ end
36
+
27
37
  route 'mount Raif::Engine => "/raif"'
28
38
  end
39
+
40
+ def setup_evals
41
+ say "\nSetting up Raif evals...", :green
42
+ generate "raif:evals:setup"
43
+ end
29
44
  end
30
45
  end
31
46
  end
@@ -98,6 +98,9 @@ Raif.configure do |config|
98
98
  # Or you can use a lambda to return a dynamic system prompt intro:
99
99
  # config.task_system_prompt_intro = ->(task){ "You are a helpful assistant. Today's date is #{Date.today.strftime('%B %d, %Y')}." }
100
100
 
101
+ # Whether the creator association is optional for Raif::Task. Defaults to true.
102
+ # config.task_creator_optional = true
103
+
101
104
  # The system prompt intro for Raif::Conversation instances. Defaults to "You are a helpful assistant who is collaborating with a teammate."
102
105
  # config.conversation_system_prompt_intro = "You are a helpful assistant who is collaborating with a teammate."
103
106
  # Or you can use a lambda to return a dynamic system prompt intro:
@@ -134,4 +137,12 @@ Raif.configure do |config|
134
137
  # Whether LLM API requests are enabled. Defaults to true.
135
138
  # Use this to globally disable requests to LLM APIs.
136
139
  # config.llm_api_requests_enabled = true
140
+
141
+ # The default LLM model to use for LLM-as-judge evaluations.
142
+ # If not set, falls back to the default_llm_model_key.
143
+ # config.evals_default_llm_judge_model_key = ENV["RAIF_EVALS_DEFAULT_LLM_JUDGE_MODEL_KEY"].presence
144
+
145
+ # Whether to output verbose information during evaluation runs. Defaults to false.
146
+ # When true, provides more detailed output including individual test results.
147
+ # config.evals_verbose_output = false
137
148
  end
@@ -1,17 +1,17 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative "../base_generator"
4
+
3
5
  module Raif
4
6
  module Generators
5
- class ModelToolGenerator < Rails::Generators::NamedBase
7
+ class ModelToolGenerator < BaseGenerator
6
8
  source_root File.expand_path("templates", __dir__)
7
9
 
8
10
  desc "Creates a new model tool for the LLM to invoke in app/models/raif/model_tools"
9
11
 
10
12
  def create_model_tool_file
11
- template "model_tool.rb.tt", File.join("app/models/raif/model_tools", "#{file_name}.rb")
12
-
13
- # Generate the view partial for the tool invocation
14
- template "model_tool_invocation_partial.html.erb.tt", File.join("app/views/raif/model_tool_invocations", "_#{file_name}.html.erb")
13
+ template "model_tool.rb.tt", File.join("app/models/raif/model_tools", class_path, "#{file_name}.rb")
14
+ template "model_tool_invocation_partial.html.erb.tt", File.join("app/views/raif/model_tool_invocations", class_path, "_#{file_name}.html.erb")
15
15
  end
16
16
 
17
17
  def success_message
@@ -1,89 +1,89 @@
1
- # frozen_string_literal: true
1
+ <% raif_module_namespacing(["ModelTools"]) do -%>
2
+ class <%= class_name.demodulize %> < Raif::ModelTool
3
+ # For example tool implementations, see:
4
+ # Wikipedia Search Tool: https://github.com/CultivateLabs/raif/blob/main/app/models/raif/model_tools/wikipedia_search.rb
5
+ # Fetch URL Tool: https://github.com/CultivateLabs/raif/blob/main/app/models/raif/model_tools/fetch_url.rb
2
6
 
3
- class Raif::ModelTools::<%= class_name %> < Raif::ModelTool
4
- # For example tool implementations, see:
5
- # Wikipedia Search Tool: https://github.com/CultivateLabs/raif/blob/main/app/models/raif/model_tools/wikipedia_search.rb
6
- # Fetch URL Tool: https://github.com/CultivateLabs/raif/blob/main/app/models/raif/model_tools/fetch_url.rb
7
-
8
- tool_description do
9
- "Description of your tool that will be provided to the LLM so it knows when to invoke it"
10
- end
7
+ tool_description do
8
+ "Description of your tool that will be provided to the LLM so it knows when to invoke it"
9
+ end
11
10
 
12
- # Define the schema for the arguments that the LLM should use when invoking your tool.
13
- # It should be a valid JSON schema. When the model invokes your tool,
14
- # the arguments it provides will be validated against this schema using JSON::Validator from the json-schema gem.
15
- #
16
- # All attributes will be required and additionalProperties will be set to false.
17
- #
18
- # See https://docs.raif.ai/learn_more/json_schemas for more information about defining JSON schemas.
19
- tool_arguments_schema do
20
- # string :title, description: "The title of the operation", minLength: 3
11
+ # Define the schema for the arguments that the LLM should use when invoking your tool.
12
+ # It should be a valid JSON schema. When the model invokes your tool,
13
+ # the arguments it provides will be validated against this schema using JSON::Validator from the json-schema gem.
21
14
  #
22
- # object :widget, description: "A widget's description" do
23
- # boolean :is_red, description: "Whether the widget is red"
24
- # integer :rating, description: "A rating of the widget from 1 to 10", minimum: 1, maximum: 10
25
- # array :tags, description: "Associated tags" do
26
- # items type: "string"
27
- # end
28
- # end
15
+ # All attributes will be required and additionalProperties will be set to false.
29
16
  #
30
- # array :products, description: "List of products" do
31
- # object do
32
- # integer :id, description: "Product identifier"
33
- # string :name, description: "Product name"
34
- # number :price, description: "Product price", minimum: 0
35
- # end
36
- # end
37
- end
17
+ # See https://docs.raif.ai/learn_more/json_schemas for more information about defining JSON schemas.
18
+ tool_arguments_schema do
19
+ # string :title, description: "The title of the operation", minLength: 3
20
+ #
21
+ # object :widget, description: "A widget's description" do
22
+ # boolean :is_red, description: "Whether the widget is red"
23
+ # integer :rating, description: "A rating of the widget from 1 to 10", minimum: 1, maximum: 10
24
+ # array :tags, description: "Associated tags" do
25
+ # items type: "string"
26
+ # end
27
+ # end
28
+ #
29
+ # array :products, description: "List of products" do
30
+ # object do
31
+ # integer :id, description: "Product identifier"
32
+ # string :name, description: "Product name"
33
+ # number :price, description: "Product price", minimum: 0
34
+ # end
35
+ # end
36
+ end
38
37
 
39
- # An example of how the LLM should invoke your tool. This should return a hash with name and arguments keys.
40
- # `to_json` will be called on it and provided to the LLM as an example of how to invoke your tool.
41
- example_model_invocation do
42
- {
43
- "name": tool_name,
44
- "arguments": { }
45
- }
46
- end
38
+ # An example of how the LLM should invoke your tool. This should return a hash with name and arguments keys.
39
+ # `to_json` will be called on it and provided to the LLM as an example of how to invoke your tool.
40
+ example_model_invocation do
41
+ {
42
+ "name": tool_name,
43
+ "arguments": {}
44
+ }
45
+ end
47
46
 
48
- class << self
49
- # When your tool is invoked by the LLM in a Raif::Agent loop,
50
- # the results of the tool invocation are provided back to the LLM as an observation.
51
- # This method should return whatever you want provided to the LLM.
52
- # For example, if you were implementing a GoogleSearch tool, this might return a JSON
53
- # object containing search results for the query.
54
- def observation_for_invocation(tool_invocation)
55
- return "No results found" unless tool_invocation.result.present?
47
+ class << self
48
+ # When your tool is invoked by the LLM in a Raif::Agent loop,
49
+ # the results of the tool invocation are provided back to the LLM as an observation.
50
+ # This method should return whatever you want provided to the LLM.
51
+ # For example, if you were implementing a GoogleSearch tool, this might return a JSON
52
+ # object containing search results for the query.
53
+ def observation_for_invocation(tool_invocation)
54
+ return "No results found" unless tool_invocation.result.present?
56
55
 
57
- JSON.pretty_generate(tool_invocation.result)
58
- end
56
+ JSON.pretty_generate(tool_invocation.result)
57
+ end
59
58
 
60
- # When your tool is invoked in a Raif::Conversation, should the result be automatically provided back to the model?
61
- # When true, observation_for_invocation will be used to produce the observation provided to the model
62
- def triggers_observation_to_model?
63
- false
64
- end
59
+ # When your tool is invoked in a Raif::Conversation, should the result be automatically provided back to the model?
60
+ # When true, observation_for_invocation will be used to produce the observation provided to the model
61
+ def triggers_observation_to_model?
62
+ false
63
+ end
64
+
65
+ # When the LLM invokes your tool, this method will be called with a `Raif::ModelToolInvocation` record as an argument.
66
+ # It should handle the actual execution of the tool.
67
+ # For example, if you are implementing a GoogleSearch tool, this method should run the actual search
68
+ # and store the results in the tool_invocation's result JSON column.
69
+ def process_invocation(tool_invocation)
70
+ # Extract arguments from tool_invocation.tool_arguments
71
+ # query = tool_invocation.tool_arguments["query"]
72
+ #
73
+ # Process the invocation and perform the desired action
74
+ # ...
75
+ #
76
+ # Store the results in the tool_invocation
77
+ # tool_invocation.update!(
78
+ # result: {
79
+ # # Your result data structure
80
+ # }
81
+ # )
82
+ #
83
+ # Return the result
84
+ # tool_invocation.result
85
+ end
65
86
 
66
- # When the LLM invokes your tool, this method will be called with a `Raif::ModelToolInvocation` record as an argument.
67
- # It should handle the actual execution of the tool.
68
- # For example, if you are implementing a GoogleSearch tool, this method should run the actual search
69
- # and store the results in the tool_invocation's result JSON column.
70
- def process_invocation(tool_invocation)
71
- # Extract arguments from tool_invocation.tool_arguments
72
- # query = tool_invocation.tool_arguments["query"]
73
- #
74
- # Process the invocation and perform the desired action
75
- # ...
76
- #
77
- # Store the results in the tool_invocation
78
- # tool_invocation.update!(
79
- # result: {
80
- # # Your result data structure
81
- # }
82
- # )
83
- #
84
- # Return the result
85
- # tool_invocation.result
86
87
  end
87
88
  end
88
-
89
- end
89
+ <% end -%>
@@ -1,4 +1,4 @@
1
- <%#
1
+ <%%#
2
2
  This partial is used to render a model tool invocation to the user in the conversation interface.
3
3
  If you don't want the tool invocation to be displayed to the user, you can override the `renderable?` method in your model tool class to return false
4
4
  %>
@@ -1,8 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative "../base_generator"
4
+
3
5
  module Raif
4
6
  module Generators
5
- class TaskGenerator < Rails::Generators::NamedBase
7
+ class TaskGenerator < BaseGenerator
6
8
  source_root File.expand_path("templates", __dir__)
7
9
 
8
10
  class_option :response_format,
@@ -10,6 +12,11 @@ module Raif
10
12
  default: "text",
11
13
  desc: "Response format for the task (text, html, or json)"
12
14
 
15
+ class_option :skip_eval_set,
16
+ type: :boolean,
17
+ default: false,
18
+ desc: "Skip generating the corresponding eval set"
19
+
13
20
  def create_application_task
14
21
  template "application_task.rb.tt", "app/models/raif/application_task.rb" unless File.exist?("app/models/raif/application_task.rb")
15
22
  end
@@ -18,11 +25,23 @@ module Raif
18
25
  template "task.rb.tt", File.join("app/models/raif/tasks", class_path, "#{file_name}.rb")
19
26
  end
20
27
 
28
+ def create_eval_set
29
+ return if options[:skip_eval_set]
30
+
31
+ template "task_eval_set.rb.tt", eval_set_file_path
32
+ end
33
+
34
+ def show_instructions
35
+ say "\nTask created!"
36
+ say ""
37
+ end
38
+
21
39
  private
22
40
 
23
- def task_class_name
24
- class_name
41
+ def eval_set_file_path
42
+ File.join("raif_evals", "eval_sets", "tasks", class_path, "#{file_name}_eval_set.rb")
25
43
  end
44
+
26
45
  end
27
46
  end
28
47
  end
@@ -1,5 +1,3 @@
1
- # frozen_string_literal: true
2
-
3
1
  module Raif
4
2
  class ApplicationTask < Raif::Task
5
3
  # Add any shared task behavior here