raif 1.2.2 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +6 -5
- data/app/assets/builds/raif.css +4 -1
- data/app/assets/builds/raif_admin.css +13 -1
- data/app/assets/javascript/raif/controllers/conversations_controller.js +1 -1
- data/app/assets/stylesheets/raif/admin/conversation.scss +16 -0
- data/app/assets/stylesheets/raif/conversations.scss +3 -0
- data/app/assets/stylesheets/raif.scss +2 -1
- data/app/controllers/raif/admin/application_controller.rb +16 -0
- data/app/controllers/raif/admin/configs_controller.rb +94 -0
- data/app/controllers/raif/admin/model_completions_controller.rb +18 -1
- data/app/controllers/raif/admin/model_tool_invocations_controller.rb +7 -1
- data/app/controllers/raif/admin/stats/model_tool_invocations_controller.rb +21 -0
- data/app/controllers/raif/admin/stats/tasks_controller.rb +15 -6
- data/app/controllers/raif/admin/stats_controller.rb +32 -3
- data/app/controllers/raif/conversation_entries_controller.rb +1 -0
- data/app/controllers/raif/conversations_controller.rb +10 -2
- data/app/jobs/raif/conversation_entry_job.rb +8 -6
- data/app/models/raif/admin/task_stat.rb +7 -0
- data/app/models/raif/agent.rb +63 -2
- data/app/models/raif/agents/native_tool_calling_agent.rb +101 -56
- data/app/models/raif/application_record.rb +18 -0
- data/app/models/raif/concerns/agent_inference_stats.rb +35 -0
- data/app/models/raif/concerns/has_llm.rb +1 -1
- data/app/models/raif/concerns/json_schema_definition.rb +40 -5
- data/app/models/raif/concerns/llms/anthropic/message_formatting.rb +28 -0
- data/app/models/raif/concerns/llms/anthropic/response_tool_calls.rb +24 -0
- data/app/models/raif/concerns/llms/anthropic/tool_formatting.rb +4 -0
- data/app/models/raif/concerns/llms/bedrock/message_formatting.rb +36 -0
- data/app/models/raif/concerns/llms/bedrock/response_tool_calls.rb +26 -0
- data/app/models/raif/concerns/llms/bedrock/tool_formatting.rb +4 -0
- data/app/models/raif/concerns/llms/google/message_formatting.rb +109 -0
- data/app/models/raif/concerns/llms/google/response_tool_calls.rb +32 -0
- data/app/models/raif/concerns/llms/google/tool_formatting.rb +72 -0
- data/app/models/raif/concerns/llms/message_formatting.rb +11 -5
- data/app/models/raif/concerns/llms/open_ai/json_schema_validation.rb +3 -3
- data/app/models/raif/concerns/llms/open_ai_completions/message_formatting.rb +22 -0
- data/app/models/raif/concerns/llms/open_ai_completions/response_tool_calls.rb +22 -0
- data/app/models/raif/concerns/llms/open_ai_completions/tool_formatting.rb +4 -0
- data/app/models/raif/concerns/llms/open_ai_responses/message_formatting.rb +17 -0
- data/app/models/raif/concerns/llms/open_ai_responses/response_tool_calls.rb +26 -0
- data/app/models/raif/concerns/llms/open_ai_responses/tool_formatting.rb +4 -0
- data/app/models/raif/concerns/run_with.rb +127 -0
- data/app/models/raif/conversation.rb +96 -9
- data/app/models/raif/conversation_entry.rb +37 -8
- data/app/models/raif/embedding_model.rb +2 -1
- data/app/models/raif/embedding_models/open_ai.rb +1 -1
- data/app/models/raif/llm.rb +28 -3
- data/app/models/raif/llms/anthropic.rb +7 -19
- data/app/models/raif/llms/bedrock.rb +6 -20
- data/app/models/raif/llms/google.rb +140 -0
- data/app/models/raif/llms/open_ai_base.rb +19 -5
- data/app/models/raif/llms/open_ai_completions.rb +6 -11
- data/app/models/raif/llms/open_ai_responses.rb +6 -16
- data/app/models/raif/llms/open_router.rb +10 -14
- data/app/models/raif/model_completion.rb +61 -0
- data/app/models/raif/model_tool.rb +10 -2
- data/app/models/raif/model_tool_invocation.rb +38 -6
- data/app/models/raif/model_tools/agent_final_answer.rb +2 -7
- data/app/models/raif/model_tools/provider_managed/code_execution.rb +4 -0
- data/app/models/raif/model_tools/provider_managed/image_generation.rb +4 -0
- data/app/models/raif/model_tools/provider_managed/web_search.rb +4 -0
- data/app/models/raif/streaming_responses/google.rb +71 -0
- data/app/models/raif/task.rb +74 -18
- data/app/models/raif/user_tool_invocation.rb +19 -0
- data/app/views/layouts/raif/admin.html.erb +12 -1
- data/app/views/raif/admin/agents/_agent.html.erb +8 -0
- data/app/views/raif/admin/agents/_conversation_message.html.erb +28 -6
- data/app/views/raif/admin/agents/index.html.erb +2 -0
- data/app/views/raif/admin/agents/show.html.erb +46 -1
- data/app/views/raif/admin/configs/show.html.erb +117 -0
- data/app/views/raif/admin/conversations/_conversation_entry.html.erb +29 -34
- data/app/views/raif/admin/conversations/show.html.erb +2 -0
- data/app/views/raif/admin/model_completions/_model_completion.html.erb +9 -0
- data/app/views/raif/admin/model_completions/index.html.erb +26 -0
- data/app/views/raif/admin/model_completions/show.html.erb +124 -61
- data/app/views/raif/admin/model_tool_invocations/index.html.erb +22 -1
- data/app/views/raif/admin/model_tools/_list.html.erb +16 -0
- data/app/views/raif/admin/model_tools/_model_tool.html.erb +36 -0
- data/app/views/raif/admin/stats/_stats_tile.html.erb +34 -0
- data/app/views/raif/admin/stats/index.html.erb +71 -88
- data/app/views/raif/admin/stats/model_tool_invocations/index.html.erb +43 -0
- data/app/views/raif/admin/stats/tasks/index.html.erb +20 -6
- data/app/views/raif/admin/tasks/index.html.erb +6 -1
- data/app/views/raif/admin/tasks/show.html.erb +36 -3
- data/app/views/raif/conversation_entries/_form.html.erb +4 -1
- data/app/views/raif/conversations/_conversation.html.erb +10 -0
- data/app/views/raif/conversations/_entry_processed.turbo_stream.erb +12 -0
- data/app/views/raif/conversations/_full_conversation.html.erb +3 -6
- data/app/views/raif/conversations/_initial_chat_message.html.erb +5 -0
- data/app/views/raif/conversations/index.html.erb +23 -0
- data/config/locales/admin.en.yml +33 -1
- data/config/locales/en.yml +41 -4
- data/config/routes.rb +2 -0
- data/db/migrate/20250804013843_add_task_run_args_to_raif_tasks.rb +13 -0
- data/db/migrate/20250811171150_make_raif_task_creator_optional.rb +8 -0
- data/db/migrate/20250904194456_add_generating_entry_response_to_raif_conversations.rb +7 -0
- data/db/migrate/20250911125234_add_source_to_raif_tasks.rb +7 -0
- data/db/migrate/20251020005853_add_source_to_raif_agents.rb +7 -0
- data/db/migrate/20251020011346_rename_task_run_args_to_run_with.rb +7 -0
- data/db/migrate/20251020011405_add_run_with_to_raif_agents.rb +13 -0
- data/db/migrate/20251024160119_add_llm_messages_max_length_to_raif_conversations.rb +14 -0
- data/db/migrate/20251124185033_add_provider_tool_call_id_to_raif_model_tool_invocations.rb +7 -0
- data/db/migrate/20251128202941_add_tool_choice_to_raif_model_completions.rb +7 -0
- data/db/migrate/20260118144846_add_source_to_raif_conversations.rb +7 -0
- data/db/migrate/20260119000000_add_failure_tracking_to_raif_model_completions.rb +10 -0
- data/db/migrate/20260119000001_add_completed_at_to_raif_model_completions.rb +8 -0
- data/db/migrate/20260119000002_add_started_at_to_raif_model_completions.rb +8 -0
- data/exe/raif +7 -0
- data/lib/generators/raif/agent/agent_generator.rb +22 -7
- data/lib/generators/raif/agent/templates/agent.rb.tt +20 -24
- data/lib/generators/raif/agent/templates/agent_eval_set.rb.tt +48 -0
- data/lib/generators/raif/agent/templates/application_agent.rb.tt +1 -3
- data/lib/generators/raif/base_generator.rb +19 -0
- data/lib/generators/raif/conversation/conversation_generator.rb +21 -2
- data/lib/generators/raif/conversation/templates/application_conversation.rb.tt +0 -2
- data/lib/generators/raif/conversation/templates/conversation.rb.tt +34 -32
- data/lib/generators/raif/conversation/templates/conversation_eval_set.rb.tt +70 -0
- data/lib/generators/raif/eval_set/eval_set_generator.rb +28 -0
- data/lib/generators/raif/eval_set/templates/eval_set.rb.tt +21 -0
- data/lib/generators/raif/evals/setup/setup_generator.rb +47 -0
- data/lib/generators/raif/install/install_generator.rb +15 -0
- data/lib/generators/raif/install/templates/initializer.rb +89 -10
- data/lib/generators/raif/model_tool/model_tool_generator.rb +5 -5
- data/lib/generators/raif/model_tool/templates/model_tool.rb.tt +78 -78
- data/lib/generators/raif/model_tool/templates/model_tool_invocation_partial.html.erb.tt +1 -1
- data/lib/generators/raif/task/task_generator.rb +22 -3
- data/lib/generators/raif/task/templates/application_task.rb.tt +0 -2
- data/lib/generators/raif/task/templates/task.rb.tt +55 -59
- data/lib/generators/raif/task/templates/task_eval_set.rb.tt +54 -0
- data/lib/raif/cli/base.rb +39 -0
- data/lib/raif/cli/evals.rb +47 -0
- data/lib/raif/cli/evals_setup.rb +27 -0
- data/lib/raif/cli.rb +67 -0
- data/lib/raif/configuration.rb +57 -8
- data/lib/raif/engine.rb +8 -0
- data/lib/raif/errors/instance_dependent_schema_error.rb +8 -0
- data/lib/raif/errors/streaming_error.rb +6 -3
- data/lib/raif/errors.rb +1 -0
- data/lib/raif/evals/eval.rb +30 -0
- data/lib/raif/evals/eval_set.rb +111 -0
- data/lib/raif/evals/eval_sets/expectations.rb +53 -0
- data/lib/raif/evals/eval_sets/llm_judge_expectations.rb +255 -0
- data/lib/raif/evals/expectation_result.rb +39 -0
- data/lib/raif/evals/llm_judge.rb +32 -0
- data/lib/raif/evals/llm_judges/binary.rb +94 -0
- data/lib/raif/evals/llm_judges/comparative.rb +89 -0
- data/lib/raif/evals/llm_judges/scored.rb +63 -0
- data/lib/raif/evals/llm_judges/summarization.rb +166 -0
- data/lib/raif/evals/run.rb +202 -0
- data/lib/raif/evals/scoring_rubric.rb +174 -0
- data/lib/raif/evals.rb +26 -0
- data/lib/raif/json_schema_builder.rb +14 -0
- data/lib/raif/llm_registry.rb +218 -15
- data/lib/raif/messages.rb +180 -0
- data/lib/raif/migration_checker.rb +3 -3
- data/lib/raif/utils/colors.rb +23 -0
- data/lib/raif/utils.rb +1 -0
- data/lib/raif/version.rb +1 -1
- data/lib/raif.rb +13 -0
- data/lib/tasks/annotate_rb.rake +10 -0
- data/spec/support/current_temperature_test_tool.rb +34 -0
- data/spec/support/rspec_helpers.rb +8 -8
- data/spec/support/test_conversation.rb +1 -1
- metadata +77 -10
- data/app/models/raif/agents/re_act_agent.rb +0 -127
- data/app/models/raif/agents/re_act_step.rb +0 -33
|
@@ -20,16 +20,11 @@ class Raif::ModelTools::AgentFinalAnswer < Raif::ModelTool
|
|
|
20
20
|
def observation_for_invocation(tool_invocation)
|
|
21
21
|
return "No answer provided" unless tool_invocation.result.present?
|
|
22
22
|
|
|
23
|
-
tool_invocation.result
|
|
23
|
+
tool_invocation.result
|
|
24
24
|
end
|
|
25
25
|
|
|
26
26
|
def process_invocation(tool_invocation)
|
|
27
|
-
tool_invocation.update!(
|
|
28
|
-
result: {
|
|
29
|
-
final_answer: tool_invocation.tool_arguments["final_answer"]
|
|
30
|
-
}
|
|
31
|
-
)
|
|
32
|
-
|
|
27
|
+
tool_invocation.update!(result: tool_invocation.tool_arguments["final_answer"])
|
|
33
28
|
tool_invocation.result
|
|
34
29
|
end
|
|
35
30
|
end
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class Raif::StreamingResponses::Google
|
|
4
|
+
|
|
5
|
+
def initialize
|
|
6
|
+
@response_json = { "candidates" => [{ "content" => { "parts" => [] } }], "usageMetadata" => {} }
|
|
7
|
+
@finish_reason = nil
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def process_streaming_event(event_type, event)
|
|
11
|
+
delta = nil
|
|
12
|
+
|
|
13
|
+
# Google streams complete candidate objects, so we need to extract the new text
|
|
14
|
+
candidates = event["candidates"]
|
|
15
|
+
if candidates.present?
|
|
16
|
+
candidate = candidates[0]
|
|
17
|
+
|
|
18
|
+
# Check for finish reason
|
|
19
|
+
@finish_reason = candidate["finishReason"] if candidate["finishReason"].present?
|
|
20
|
+
|
|
21
|
+
# Process content parts
|
|
22
|
+
parts = candidate.dig("content", "parts")
|
|
23
|
+
delta = process_content_parts(parts) if parts.present?
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Update usage metadata
|
|
27
|
+
usage_metadata = event["usageMetadata"]
|
|
28
|
+
@response_json["usageMetadata"] = usage_metadata if usage_metadata.present?
|
|
29
|
+
|
|
30
|
+
[delta, @finish_reason]
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def current_response_json
|
|
34
|
+
@response_json
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def process_content_parts(parts)
|
|
40
|
+
delta = nil
|
|
41
|
+
|
|
42
|
+
parts.each_with_index do |part, index|
|
|
43
|
+
if part.key?("text")
|
|
44
|
+
delta = part["text"]
|
|
45
|
+
accumulate_text_part(part, index)
|
|
46
|
+
else
|
|
47
|
+
# For non-text parts (e.g., functionCall), just store directly.
|
|
48
|
+
# Note: This works because we don't enable streamFunctionCallArguments in the API request.
|
|
49
|
+
# Without that opt-in flag, function calls arrive complete in a single chunk.
|
|
50
|
+
# If streaming function call arguments is enabled in the future (Gemini 3+ models),
|
|
51
|
+
# this would need to accumulate partialArgs similar to how we accumulate text.
|
|
52
|
+
@response_json["candidates"][0]["content"]["parts"][index] = part
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
delta
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def accumulate_text_part(part, index)
|
|
60
|
+
existing_part = @response_json.dig("candidates", 0, "content", "parts", index)
|
|
61
|
+
|
|
62
|
+
if existing_part.present? && existing_part.key?("text")
|
|
63
|
+
# Accumulate text from incremental chunks
|
|
64
|
+
existing_part["text"] += part["text"]
|
|
65
|
+
else
|
|
66
|
+
# First text chunk for this index
|
|
67
|
+
@response_json["candidates"][0]["content"]["parts"][index] = part.dup
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
end
|
data/app/models/raif/task.rb
CHANGED
|
@@ -1,5 +1,42 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# == Schema Information
|
|
4
|
+
#
|
|
5
|
+
# Table name: raif_tasks
|
|
6
|
+
#
|
|
7
|
+
# id :bigint not null, primary key
|
|
8
|
+
# available_model_tools :jsonb not null
|
|
9
|
+
# completed_at :datetime
|
|
10
|
+
# creator_type :string
|
|
11
|
+
# failed_at :datetime
|
|
12
|
+
# llm_model_key :string not null
|
|
13
|
+
# prompt :text
|
|
14
|
+
# raw_response :text
|
|
15
|
+
# requested_language_key :string
|
|
16
|
+
# response_format :integer default("text"), not null
|
|
17
|
+
# run_with :jsonb
|
|
18
|
+
# source_type :string
|
|
19
|
+
# started_at :datetime
|
|
20
|
+
# system_prompt :text
|
|
21
|
+
# type :string not null
|
|
22
|
+
# created_at :datetime not null
|
|
23
|
+
# updated_at :datetime not null
|
|
24
|
+
# creator_id :bigint
|
|
25
|
+
# source_id :bigint
|
|
26
|
+
#
|
|
27
|
+
# Indexes
|
|
28
|
+
#
|
|
29
|
+
# index_raif_tasks_on_completed_at (completed_at)
|
|
30
|
+
# index_raif_tasks_on_created_at (created_at)
|
|
31
|
+
# index_raif_tasks_on_creator (creator_type,creator_id)
|
|
32
|
+
# index_raif_tasks_on_failed_at (failed_at)
|
|
33
|
+
# index_raif_tasks_on_source (source_type,source_id)
|
|
34
|
+
# index_raif_tasks_on_started_at (started_at)
|
|
35
|
+
# index_raif_tasks_on_type (type)
|
|
36
|
+
# index_raif_tasks_on_type_and_completed_at (type,completed_at)
|
|
37
|
+
# index_raif_tasks_on_type_and_failed_at (type,failed_at)
|
|
38
|
+
# index_raif_tasks_on_type_and_started_at (type,started_at)
|
|
39
|
+
#
|
|
3
40
|
module Raif
|
|
4
41
|
class Task < Raif::ApplicationRecord
|
|
5
42
|
include Raif::Concerns::HasLlm
|
|
@@ -9,10 +46,14 @@ module Raif
|
|
|
9
46
|
include Raif::Concerns::LlmResponseParsing
|
|
10
47
|
include Raif::Concerns::LlmTemperature
|
|
11
48
|
include Raif::Concerns::JsonSchemaDefinition
|
|
49
|
+
include Raif::Concerns::RunWith
|
|
12
50
|
|
|
13
51
|
llm_temperature 0.7
|
|
14
52
|
|
|
15
|
-
belongs_to :creator, polymorphic: true
|
|
53
|
+
belongs_to :creator, polymorphic: true, optional: true
|
|
54
|
+
belongs_to :source, polymorphic: true, optional: true
|
|
55
|
+
|
|
56
|
+
validates :creator, presence: true, unless: -> { Raif.config.task_creator_optional }
|
|
16
57
|
|
|
17
58
|
has_one :raif_model_completion, as: :source, dependent: :destroy, class_name: "Raif::ModelCompletion"
|
|
18
59
|
|
|
@@ -22,8 +63,6 @@ module Raif
|
|
|
22
63
|
|
|
23
64
|
normalizes :prompt, :system_prompt, with: ->(text){ text&.strip }
|
|
24
65
|
|
|
25
|
-
delegate :json_response_schema, to: :class
|
|
26
|
-
|
|
27
66
|
scope :completed, -> { where.not(completed_at: nil) }
|
|
28
67
|
scope :failed, -> { where.not(failed_at: nil) }
|
|
29
68
|
scope :in_progress, -> { where.not(started_at: nil).where(completed_at: nil, failed_at: nil) }
|
|
@@ -32,6 +71,7 @@ module Raif
|
|
|
32
71
|
attr_accessor :files, :images
|
|
33
72
|
|
|
34
73
|
after_initialize -> { self.available_model_tools ||= [] }
|
|
74
|
+
after_initialize -> { self.run_with ||= {} }
|
|
35
75
|
|
|
36
76
|
def status
|
|
37
77
|
if completed_at?
|
|
@@ -48,15 +88,23 @@ module Raif
|
|
|
48
88
|
# The primary interface for running a task. It will hit the LLM with the task's prompt and system prompt and return a Raif::Task object.
|
|
49
89
|
# It will also create a new Raif::ModelCompletion record.
|
|
50
90
|
#
|
|
51
|
-
# @param creator [Object] The creator of the task (polymorphic association)
|
|
91
|
+
# @param creator [Object, nil] The creator of the task (polymorphic association), optional
|
|
52
92
|
# @param available_model_tools [Array<Class>] Optional array of model tool classes that will be provided to the LLM for it to invoke.
|
|
53
93
|
# @param llm_model_key [Symbol, String] Optional key for the LLM model to use. If blank, Raif.config.default_llm_model_key will be used.
|
|
54
94
|
# @param images [Array] Optional array of Raif::ModelImageInput objects to include with the prompt.
|
|
55
95
|
# @param files [Array] Optional array of Raif::ModelFileInput objects to include with the prompt.
|
|
56
96
|
# @param args [Hash] Additional arguments to pass to the instance of the task that is created.
|
|
57
97
|
# @return [Raif::Task, nil] The task instance that was created and run.
|
|
58
|
-
def self.run(creator
|
|
59
|
-
task = new(
|
|
98
|
+
def self.run(creator: nil, available_model_tools: [], llm_model_key: nil, images: [], files: [], **args)
|
|
99
|
+
task = new(
|
|
100
|
+
creator: creator,
|
|
101
|
+
llm_model_key: llm_model_key,
|
|
102
|
+
available_model_tools: available_model_tools,
|
|
103
|
+
started_at: Time.current,
|
|
104
|
+
images: images,
|
|
105
|
+
files: files,
|
|
106
|
+
**args
|
|
107
|
+
)
|
|
60
108
|
|
|
61
109
|
task.save!
|
|
62
110
|
task.run
|
|
@@ -82,7 +130,6 @@ module Raif
|
|
|
82
130
|
update_columns(started_at: Time.current) if started_at.nil?
|
|
83
131
|
|
|
84
132
|
populate_prompts unless skip_prompt_population
|
|
85
|
-
messages = [{ "role" => "user", "content" => message_content }]
|
|
86
133
|
|
|
87
134
|
mc = llm.chat(
|
|
88
135
|
messages: messages,
|
|
@@ -107,21 +154,25 @@ module Raif
|
|
|
107
154
|
run(skip_prompt_population: true)
|
|
108
155
|
end
|
|
109
156
|
|
|
157
|
+
def messages
|
|
158
|
+
[{ "role" => "user", "content" => message_content }]
|
|
159
|
+
end
|
|
160
|
+
|
|
110
161
|
# Returns the LLM prompt for the task.
|
|
111
162
|
#
|
|
112
|
-
# @param creator [Object] The creator of the task (polymorphic association)
|
|
163
|
+
# @param creator [Object, nil] The creator of the task (polymorphic association), optional
|
|
113
164
|
# @param args [Hash] Additional arguments to pass to the instance of the task that is created.
|
|
114
165
|
# @return [String] The LLM prompt for the task.
|
|
115
|
-
def self.prompt(creator
|
|
166
|
+
def self.prompt(creator: nil, **args)
|
|
116
167
|
new(creator:, **args).build_prompt
|
|
117
168
|
end
|
|
118
169
|
|
|
119
170
|
# Returns the LLM system prompt for the task.
|
|
120
171
|
#
|
|
121
|
-
# @param creator [Object] The creator of the task (polymorphic association)
|
|
172
|
+
# @param creator [Object, nil] The creator of the task (polymorphic association), optional
|
|
122
173
|
# @param args [Hash] Additional arguments to pass to the instance of the task that is created.
|
|
123
174
|
# @return [String] The LLM system prompt for the task.
|
|
124
|
-
def self.system_prompt(creator
|
|
175
|
+
def self.system_prompt(creator: nil, **args)
|
|
125
176
|
new(creator:, **args).build_system_prompt
|
|
126
177
|
end
|
|
127
178
|
|
|
@@ -133,6 +184,13 @@ module Raif
|
|
|
133
184
|
end
|
|
134
185
|
end
|
|
135
186
|
|
|
187
|
+
# Instance method to get the JSON response schema
|
|
188
|
+
# For instance-dependent schemas, builds the schema with this instance as context
|
|
189
|
+
# For class-level schemas, returns the class-level schema
|
|
190
|
+
def json_response_schema
|
|
191
|
+
schema_for_instance(:json_response)
|
|
192
|
+
end
|
|
193
|
+
|
|
136
194
|
def build_prompt
|
|
137
195
|
raise NotImplementedError, "Raif::Task subclasses must implement #build_prompt"
|
|
138
196
|
end
|
|
@@ -170,21 +228,19 @@ module Raif
|
|
|
170
228
|
end
|
|
171
229
|
|
|
172
230
|
def populate_prompts
|
|
173
|
-
self.requested_language_key ||= creator
|
|
231
|
+
self.requested_language_key ||= creator&.preferred_language_key if creator&.respond_to?(:preferred_language_key)
|
|
174
232
|
self.prompt = build_prompt
|
|
175
233
|
self.system_prompt = build_system_prompt
|
|
176
234
|
end
|
|
177
235
|
|
|
178
236
|
def process_model_tool_invocations
|
|
179
|
-
return unless
|
|
180
|
-
return unless parsed_response.is_a?(Hash)
|
|
181
|
-
return unless parsed_response["tools"].present? && parsed_response["tools"].is_a?(Array)
|
|
237
|
+
return unless raif_model_completion&.response_tool_calls.present?
|
|
182
238
|
|
|
183
|
-
|
|
184
|
-
tool_klass = available_model_tools_map[
|
|
239
|
+
raif_model_completion.response_tool_calls.each do |tool_call|
|
|
240
|
+
tool_klass = available_model_tools_map[tool_call["name"]]
|
|
185
241
|
next unless tool_klass
|
|
186
242
|
|
|
187
|
-
tool_klass.invoke_tool(tool_arguments:
|
|
243
|
+
tool_klass.invoke_tool(provider_tool_call_id: tool_call["provider_tool_call_id"], tool_arguments: tool_call["arguments"], source: self)
|
|
188
244
|
end
|
|
189
245
|
end
|
|
190
246
|
|
|
@@ -1,5 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# == Schema Information
|
|
4
|
+
#
|
|
5
|
+
# Table name: raif_user_tool_invocations
|
|
6
|
+
#
|
|
7
|
+
# id :bigint not null, primary key
|
|
8
|
+
# tool_settings :jsonb not null
|
|
9
|
+
# type :string not null
|
|
10
|
+
# created_at :datetime not null
|
|
11
|
+
# updated_at :datetime not null
|
|
12
|
+
# raif_conversation_entry_id :bigint not null
|
|
13
|
+
#
|
|
14
|
+
# Indexes
|
|
15
|
+
#
|
|
16
|
+
# index_raif_user_tool_invocations_on_raif_conversation_entry_id (raif_conversation_entry_id)
|
|
17
|
+
#
|
|
18
|
+
# Foreign Keys
|
|
19
|
+
#
|
|
20
|
+
# fk_rails_... (raif_conversation_entry_id => raif_conversation_entries.id)
|
|
21
|
+
#
|
|
3
22
|
class Raif::UserToolInvocation < Raif::ApplicationRecord
|
|
4
23
|
belongs_to :raif_conversation_entry, class_name: "Raif::ConversationEntry"
|
|
5
24
|
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
<div class="position-sticky pt-3">
|
|
36
36
|
<ul class="nav flex-column">
|
|
37
37
|
<li class="nav-item">
|
|
38
|
-
<a class="nav-link <%=
|
|
38
|
+
<a class="nav-link <%= params[:controller].start_with?("raif/admin/stats") ? "active" : "" %>" href="<%= raif.admin_stats_path %>">
|
|
39
39
|
<span class="d-inline-block me-2">
|
|
40
40
|
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-bar-chart" viewBox="0 0 16 16">
|
|
41
41
|
<path d="M4 11H2v3h2v-3zm5-4H7v7h2V7zm5-5v12h-2V2h2zm-2-1a1 1 0 0 0-1 1v12a1 1 0 0 0 1 1h2a1 1 0 0 0 1-1V2a1 1 0 0 0-1-1h-2zM6 7a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1v7a1 1 0 0 1-1 1H7a1 1 0 0 1-1-1V7zm-5 4a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1v3a1 1 0 0 1-1 1H2a1 1 0 0 1-1-1v-3z" />
|
|
@@ -96,6 +96,17 @@
|
|
|
96
96
|
<%= t("raif.admin.common.agents") %>
|
|
97
97
|
</a>
|
|
98
98
|
</li>
|
|
99
|
+
<li class="nav-item">
|
|
100
|
+
<a class="nav-link <%= current_page?(raif.admin_config_path) ? "active" : "" %>" href="<%= raif.admin_config_path %>">
|
|
101
|
+
<span class="d-inline-block me-2">
|
|
102
|
+
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-gear" viewBox="0 0 16 16">
|
|
103
|
+
<path d="M8 4.754a3.246 3.246 0 1 0 0 6.492 3.246 3.246 0 0 0 0-6.492zM5.754 8a2.246 2.246 0 1 1 4.492 0 2.246 2.246 0 0 1-4.492 0z" />
|
|
104
|
+
<path d="M9.796 1.343c-.527-1.79-3.065-1.79-3.592 0l-.094.319a.873.873 0 0 1-1.255.52l-.292-.16c-1.64-.892-3.433.902-2.54 2.541l.159.292a.873.873 0 0 1-.52 1.255l-.319.094c-1.79.527-1.79 3.065 0 3.592l.319.094a.873.873 0 0 1 .52 1.255l-.16.292c-.892 1.64.901 3.434 2.541 2.54l.292-.159a.873.873 0 0 1 1.255.52l.094.319c.527 1.79 3.065 1.79 3.592 0l.094-.319a.873.873 0 0 1 1.255-.52l.292.16c1.64.893 3.434-.902 2.54-2.541l-.159-.292a.873.873 0 0 1 .52-1.255l.319-.094c1.79-.527 1.79-3.065 0-3.592l-.319-.094a.873.873 0 0 1-.52-1.255l.16-.292c.893-1.64-.902-3.433-2.541-2.54l-.292.159a.873.873 0 0 1-1.255-.52l-.094-.319z" />
|
|
105
|
+
</svg>
|
|
106
|
+
</span>
|
|
107
|
+
<%= t("raif.admin.common.config") %>
|
|
108
|
+
</a>
|
|
109
|
+
</li>
|
|
99
110
|
</ul>
|
|
100
111
|
</div>
|
|
101
112
|
</div>
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
<tr id="<%= dom_id(agent) %>" class="raif-agent">
|
|
2
2
|
<td><%= link_to "##{agent.id}", raif.admin_agent_path(agent) %></td>
|
|
3
|
+
<td><small class="text-muted"><%= agent.type.demodulize %></small></td>
|
|
3
4
|
<td><small class="text-muted"><%= agent.created_at.rfc822 %></small></td>
|
|
4
5
|
<td><small class="text-muted"><%= truncate(agent.task, length: 100) %></small></td>
|
|
5
6
|
<td>
|
|
@@ -14,5 +15,12 @@
|
|
|
14
15
|
<% end %>
|
|
15
16
|
</td>
|
|
16
17
|
<td><%= agent.iteration_count %> / <%= agent.max_iterations %></td>
|
|
18
|
+
<td>
|
|
19
|
+
<% if agent.total_cost && agent.total_cost > 0 %>
|
|
20
|
+
<small><%= "$" %><%= number_with_precision(agent.total_cost, precision: 6) %></small>
|
|
21
|
+
<% else %>
|
|
22
|
+
-
|
|
23
|
+
<% end %>
|
|
24
|
+
</td>
|
|
17
25
|
<td><small class="text-muted"><%= truncate(agent.final_answer, length: 100) if agent.final_answer %></small></td>
|
|
18
26
|
</tr>
|
|
@@ -1,15 +1,37 @@
|
|
|
1
|
-
|
|
1
|
+
<% message_type = message["type"] || message["role"] %>
|
|
2
|
+
<div class="message <%= message_type %> mb-3">
|
|
2
3
|
<div class="message-header d-flex justify-content-between align-items-center mb-2 px-2">
|
|
3
|
-
<strong class="<%= message
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
4
|
+
<strong class="<%= conversation_message_header_class(message) %>">
|
|
5
|
+
<% if message_type == "tool_call" %>
|
|
6
|
+
<%= t("raif.admin.common.tool_call") %>: <%= message["name"] %>
|
|
7
|
+
<% elsif message_type == "tool_call_result" %>
|
|
8
|
+
<%= t("raif.admin.common.tool_result") %>
|
|
9
|
+
<% else %>
|
|
10
|
+
<%= message["role"]&.capitalize %>
|
|
11
|
+
<% if message[:counter] == 0 && message["role"] == 'user' %>
|
|
12
|
+
(<%= t("raif.admin.common.initial_task") %>)
|
|
13
|
+
<% end %>
|
|
14
|
+
<% end %>
|
|
15
|
+
<% if is_final_answer %>
|
|
16
|
+
<span class="badge bg-info ms-2"><%= t("raif.admin.common.final_answer") %></span>
|
|
7
17
|
<% end %>
|
|
8
18
|
</strong>
|
|
9
19
|
<span class="badge bg-secondary">#<%= message_count %></span>
|
|
10
20
|
</div>
|
|
11
21
|
|
|
12
22
|
<div class="message-content px-3 py-2">
|
|
13
|
-
|
|
23
|
+
<% if message_type == "tool_call" %>
|
|
24
|
+
<% if message["assistant_message"].present? %>
|
|
25
|
+
<div class="mb-2"><code><%= message["assistant_message"] %></code></div>
|
|
26
|
+
<% end %>
|
|
27
|
+
<div>
|
|
28
|
+
<strong><%= t("raif.admin.common.arguments") %>:</strong>
|
|
29
|
+
<code><%= message["arguments"].to_json %></code>
|
|
30
|
+
</div>
|
|
31
|
+
<% elsif message_type == "tool_call_result" %>
|
|
32
|
+
<code><%= message["result"].is_a?(String) ? message["result"] : JSON.pretty_generate(message["result"]) %></code>
|
|
33
|
+
<% else %>
|
|
34
|
+
<code><%= message["content"] %></code>
|
|
35
|
+
<% end %>
|
|
14
36
|
</div>
|
|
15
37
|
</div>
|
|
@@ -8,10 +8,12 @@
|
|
|
8
8
|
<thead class="table-light">
|
|
9
9
|
<tr>
|
|
10
10
|
<th><%= t("raif.admin.common.id") %></th>
|
|
11
|
+
<th><%= t("raif.admin.common.type") %></th>
|
|
11
12
|
<th><%= t("raif.admin.common.created_at") %></th>
|
|
12
13
|
<th><%= t("raif.admin.common.task") %></th>
|
|
13
14
|
<th><%= t("raif.admin.common.status") %></th>
|
|
14
15
|
<th><%= t("raif.admin.common.iterations") %></th>
|
|
16
|
+
<th><%= t("raif.admin.common.total_cost") %></th>
|
|
15
17
|
<th><%= t("raif.admin.common.final_answer") %></th>
|
|
16
18
|
</tr>
|
|
17
19
|
</thead>
|
|
@@ -50,9 +50,50 @@
|
|
|
50
50
|
<div class="col-md-9"><%= @agent.requested_language_key %></div>
|
|
51
51
|
</div>
|
|
52
52
|
<% end %>
|
|
53
|
+
<div class="row mb-3">
|
|
54
|
+
<div class="col-md-3"><strong><%= t("raif.admin.common.prompt_tokens") %>:</strong></div>
|
|
55
|
+
<div class="col-md-9">
|
|
56
|
+
<% if @agent.total_prompt_tokens > 0 %>
|
|
57
|
+
<%= number_with_delimiter(@agent.total_prompt_tokens) %>
|
|
58
|
+
<% if @agent.total_prompt_token_cost > 0 %>
|
|
59
|
+
<small>(<%= t("raif.admin.common.est_cost") %>: <%= "$" %><%= number_with_precision(@agent.total_prompt_token_cost, precision: 6) %>)</small>
|
|
60
|
+
<% end %>
|
|
61
|
+
<% else %>
|
|
62
|
+
-
|
|
63
|
+
<% end %>
|
|
64
|
+
</div>
|
|
65
|
+
</div>
|
|
66
|
+
<div class="row mb-3">
|
|
67
|
+
<div class="col-md-3"><strong><%= t("raif.admin.common.completion_tokens") %>:</strong></div>
|
|
68
|
+
<div class="col-md-9">
|
|
69
|
+
<% if @agent.total_completion_tokens > 0 %>
|
|
70
|
+
<%= number_with_delimiter(@agent.total_completion_tokens) %>
|
|
71
|
+
<% if @agent.total_output_token_cost > 0 %>
|
|
72
|
+
<small>(<%= t("raif.admin.common.est_cost") %>: <%= "$" %><%= number_with_precision(@agent.total_output_token_cost, precision: 6) %>)</small>
|
|
73
|
+
<% end %>
|
|
74
|
+
<% else %>
|
|
75
|
+
-
|
|
76
|
+
<% end %>
|
|
77
|
+
</div>
|
|
78
|
+
</div>
|
|
79
|
+
<div class="row mb-3">
|
|
80
|
+
<div class="col-md-3"><strong><%= t("raif.admin.common.total_tokens") %>:</strong></div>
|
|
81
|
+
<div class="col-md-9">
|
|
82
|
+
<% if @agent.total_tokens_sum > 0 %>
|
|
83
|
+
<%= number_with_delimiter(@agent.total_tokens_sum) %>
|
|
84
|
+
<% if @agent.total_cost > 0 %>
|
|
85
|
+
<small>(<%= t("raif.admin.common.est_cost") %>: <%= "$" %><%= number_with_precision(@agent.total_cost, precision: 6) %>)</small>
|
|
86
|
+
<% end %>
|
|
87
|
+
<% else %>
|
|
88
|
+
-
|
|
89
|
+
<% end %>
|
|
90
|
+
</div>
|
|
91
|
+
</div>
|
|
53
92
|
</div>
|
|
54
93
|
</div>
|
|
55
94
|
|
|
95
|
+
<%= render "raif/admin/model_tools/list", model_tools: @agent.available_model_tools_map %>
|
|
96
|
+
|
|
56
97
|
<div class="card mb-4">
|
|
57
98
|
<div class="card-header">
|
|
58
99
|
<h5 class="mb-0"><%= t("raif.admin.common.task") %></h5>
|
|
@@ -91,7 +132,11 @@
|
|
|
91
132
|
<div class="conversation-history p-3">
|
|
92
133
|
<% @agent.conversation_history.each_with_index do |message, index| %>
|
|
93
134
|
<%= render partial: "raif/admin/agents/conversation_message",
|
|
94
|
-
locals: {
|
|
135
|
+
locals: {
|
|
136
|
+
message: message,
|
|
137
|
+
message_count: index + 1,
|
|
138
|
+
is_final_answer: message["type"] == "tool_call" && message["name"] == "agent_final_answer"
|
|
139
|
+
} %>
|
|
95
140
|
<% end %>
|
|
96
141
|
</div>
|
|
97
142
|
</div>
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
<h1 class="my-4"><%= t("raif.admin.config.show.title") %></h1>
|
|
2
|
+
|
|
3
|
+
<div class="row">
|
|
4
|
+
<div class="col-12">
|
|
5
|
+
<div class="card mb-4">
|
|
6
|
+
<div class="card-header bg-light">
|
|
7
|
+
<h5 class="mb-0"><%= t("raif.admin.config.show.configuration_settings") %></h5>
|
|
8
|
+
</div>
|
|
9
|
+
<div class="card-body">
|
|
10
|
+
<table class="table table-sm table-hover mb-0">
|
|
11
|
+
<thead>
|
|
12
|
+
<tr>
|
|
13
|
+
<th style="width: 40%"><%= t("raif.admin.config.show.setting") %></th>
|
|
14
|
+
<th style="width: 60%"><%= t("raif.admin.config.show.value") %></th>
|
|
15
|
+
</tr>
|
|
16
|
+
</thead>
|
|
17
|
+
<tbody>
|
|
18
|
+
<% @config_settings.each do |item| %>
|
|
19
|
+
<tr>
|
|
20
|
+
<td class="font-monospace"><%= item[:key] %></td>
|
|
21
|
+
<td>
|
|
22
|
+
<% if item[:value].is_a?(TrueClass) || item[:value].is_a?(FalseClass) %>
|
|
23
|
+
<span class="badge <%= item[:value] ? "bg-success" : "bg-secondary" %>">
|
|
24
|
+
<%= item[:value] %>
|
|
25
|
+
</span>
|
|
26
|
+
<% elsif item[:value].to_s.include?("*") %>
|
|
27
|
+
<code class="text-muted"><%= item[:value] %></code>
|
|
28
|
+
<% elsif item[:value].is_a?(String) && item[:value].length > 80 %>
|
|
29
|
+
<small class="text-muted"><%= item[:value] %></small>
|
|
30
|
+
<% elsif item[:value].present? %>
|
|
31
|
+
<code><%= item[:value] %></code>
|
|
32
|
+
<% else %>
|
|
33
|
+
<span class="text-muted fst-italic"><%= t("raif.admin.config.show.not_set") %></span>
|
|
34
|
+
<% end %>
|
|
35
|
+
</td>
|
|
36
|
+
</tr>
|
|
37
|
+
<% end %>
|
|
38
|
+
</tbody>
|
|
39
|
+
</table>
|
|
40
|
+
</div>
|
|
41
|
+
</div>
|
|
42
|
+
</div>
|
|
43
|
+
</div>
|
|
44
|
+
|
|
45
|
+
<div class="row mt-4">
|
|
46
|
+
<div class="col-12">
|
|
47
|
+
<div class="card">
|
|
48
|
+
<div class="card-header bg-light">
|
|
49
|
+
<h5 class="mb-0"><%= t("raif.admin.config.show.registered_llms") %></h5>
|
|
50
|
+
</div>
|
|
51
|
+
<div class="card-body">
|
|
52
|
+
<% if Raif.llm_registry.any? %>
|
|
53
|
+
<table class="table table-sm table-hover mb-0">
|
|
54
|
+
<thead>
|
|
55
|
+
<tr>
|
|
56
|
+
<th><%= t("raif.admin.config.show.key") %></th>
|
|
57
|
+
</tr>
|
|
58
|
+
</thead>
|
|
59
|
+
<tbody>
|
|
60
|
+
<% Raif.llm_registry.each do |key, _llm_config| %>
|
|
61
|
+
<tr>
|
|
62
|
+
<td>
|
|
63
|
+
<code><%= key %></code>
|
|
64
|
+
<% if key.to_s == @config.default_llm_model_key.to_s %>
|
|
65
|
+
<span class="badge bg-primary ms-2"><%= t("raif.admin.config.show.default") %></span>
|
|
66
|
+
<% end %>
|
|
67
|
+
</td>
|
|
68
|
+
</tr>
|
|
69
|
+
<% end %>
|
|
70
|
+
</tbody>
|
|
71
|
+
</table>
|
|
72
|
+
<% else %>
|
|
73
|
+
<div class="alert alert-warning mb-0">
|
|
74
|
+
<%= t("raif.admin.config.show.no_llms_registered") %>
|
|
75
|
+
</div>
|
|
76
|
+
<% end %>
|
|
77
|
+
</div>
|
|
78
|
+
</div>
|
|
79
|
+
</div>
|
|
80
|
+
</div>
|
|
81
|
+
|
|
82
|
+
<div class="row mt-4">
|
|
83
|
+
<div class="col-12">
|
|
84
|
+
<div class="card">
|
|
85
|
+
<div class="card-header bg-light">
|
|
86
|
+
<h5 class="mb-0"><%= t("raif.admin.config.show.registered_embedding_models") %></h5>
|
|
87
|
+
</div>
|
|
88
|
+
<div class="card-body">
|
|
89
|
+
<% if Raif.embedding_model_registry.present? %>
|
|
90
|
+
<table class="table table-sm table-hover mb-0">
|
|
91
|
+
<thead>
|
|
92
|
+
<tr>
|
|
93
|
+
<th><%= t("raif.admin.config.show.key") %></th>
|
|
94
|
+
</tr>
|
|
95
|
+
</thead>
|
|
96
|
+
<tbody>
|
|
97
|
+
<% Raif.embedding_model_registry.each do |key, _model_config| %>
|
|
98
|
+
<tr>
|
|
99
|
+
<td>
|
|
100
|
+
<code><%= key %></code>
|
|
101
|
+
<% if key.to_s == @config.default_embedding_model_key.to_s %>
|
|
102
|
+
<span class="badge bg-primary ms-2"><%= t("raif.admin.config.show.default") %></span>
|
|
103
|
+
<% end %>
|
|
104
|
+
</td>
|
|
105
|
+
</tr>
|
|
106
|
+
<% end %>
|
|
107
|
+
</tbody>
|
|
108
|
+
</table>
|
|
109
|
+
<% else %>
|
|
110
|
+
<div class="alert alert-info mb-0">
|
|
111
|
+
<%= t("raif.admin.config.show.no_embedding_models_registered") %>
|
|
112
|
+
</div>
|
|
113
|
+
<% end %>
|
|
114
|
+
</div>
|
|
115
|
+
</div>
|
|
116
|
+
</div>
|
|
117
|
+
</div>
|