active_genie 0.0.24 → 0.0.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +35 -50
- data/VERSION +1 -1
- data/lib/active_genie/battle/README.md +5 -5
- data/lib/active_genie/battle/generalist.rb +132 -0
- data/lib/active_genie/battle.rb +6 -5
- data/lib/active_genie/clients/providers/anthropic_client.rb +77 -0
- data/lib/active_genie/clients/{base_client.rb → providers/base_client.rb} +74 -100
- data/lib/active_genie/clients/providers/deepseek_client.rb +91 -0
- data/lib/active_genie/clients/providers/google_client.rb +132 -0
- data/lib/active_genie/clients/providers/openai_client.rb +96 -0
- data/lib/active_genie/clients/unified_client.rb +42 -12
- data/lib/active_genie/concerns/loggable.rb +11 -23
- data/lib/active_genie/config/battle_config.rb +8 -0
- data/lib/active_genie/config/data_extractor_config.rb +23 -0
- data/lib/active_genie/config/llm_config.rb +36 -0
- data/lib/active_genie/config/log_config.rb +44 -0
- data/lib/active_genie/config/providers/anthropic_config.rb +57 -0
- data/lib/active_genie/config/providers/deepseek_config.rb +50 -0
- data/lib/active_genie/config/providers/google_config.rb +52 -0
- data/lib/active_genie/config/providers/openai_config.rb +50 -0
- data/lib/active_genie/config/providers/provider_base.rb +89 -0
- data/lib/active_genie/config/providers_config.rb +62 -0
- data/lib/active_genie/config/ranking_config.rb +21 -0
- data/lib/active_genie/config/scoring_config.rb +8 -0
- data/lib/active_genie/configuration.rb +51 -28
- data/lib/active_genie/data_extractor/README.md +13 -13
- data/lib/active_genie/data_extractor/from_informal.rb +54 -48
- data/lib/active_genie/data_extractor/generalist.md +12 -0
- data/lib/active_genie/data_extractor/generalist.rb +125 -0
- data/lib/active_genie/data_extractor.rb +7 -5
- data/lib/active_genie/errors/invalid_provider_error.rb +41 -0
- data/lib/active_genie/logger.rb +17 -66
- data/lib/active_genie/ranking/README.md +31 -1
- data/lib/active_genie/ranking/elo_round.rb +107 -104
- data/lib/active_genie/ranking/free_for_all.rb +78 -74
- data/lib/active_genie/ranking/player.rb +79 -71
- data/lib/active_genie/ranking/players_collection.rb +83 -71
- data/lib/active_genie/ranking/ranking.rb +71 -94
- data/lib/active_genie/ranking/ranking_scoring.rb +71 -50
- data/lib/active_genie/ranking.rb +2 -0
- data/lib/active_genie/scoring/README.md +4 -4
- data/lib/active_genie/scoring/generalist.rb +171 -0
- data/lib/active_genie/scoring/recommended_reviewers.rb +70 -71
- data/lib/active_genie/scoring.rb +8 -5
- data/lib/active_genie.rb +23 -1
- data/lib/tasks/benchmark.rake +10 -9
- data/lib/tasks/install.rake +3 -1
- data/lib/tasks/templates/active_genie.rb +11 -6
- metadata +31 -22
- data/lib/active_genie/battle/basic.rb +0 -129
- data/lib/active_genie/clients/anthropic_client.rb +0 -84
- data/lib/active_genie/clients/google_client.rb +0 -135
- data/lib/active_genie/clients/helpers/retry.rb +0 -29
- data/lib/active_genie/clients/openai_client.rb +0 -98
- data/lib/active_genie/configuration/log_config.rb +0 -14
- data/lib/active_genie/configuration/providers/anthropic_config.rb +0 -54
- data/lib/active_genie/configuration/providers/base_config.rb +0 -85
- data/lib/active_genie/configuration/providers/deepseek_config.rb +0 -54
- data/lib/active_genie/configuration/providers/google_config.rb +0 -56
- data/lib/active_genie/configuration/providers/internal_company_api_config.rb +0 -54
- data/lib/active_genie/configuration/providers/openai_config.rb +0 -54
- data/lib/active_genie/configuration/providers_config.rb +0 -40
- data/lib/active_genie/configuration/runtime_config.rb +0 -35
- data/lib/active_genie/data_extractor/basic.rb +0 -101
- data/lib/active_genie/scoring/basic.rb +0 -170
@@ -1,56 +0,0 @@
|
|
1
|
-
require_relative '../../clients/google_client'
|
2
|
-
require_relative './base_config'
|
3
|
-
|
4
|
-
module ActiveGenie
|
5
|
-
module Configuration::Providers
|
6
|
-
# Configuration class for the Google Generative Language API client.
|
7
|
-
# Manages API keys, URLs, model selections, and client instantiation.
|
8
|
-
class GoogleConfig < BaseConfig
|
9
|
-
NAME = :google
|
10
|
-
|
11
|
-
# Retrieves the API key.
|
12
|
-
# Falls back to the GENERATIVE_LANGUAGE_GOOGLE_API_KEY environment variable if not set.
|
13
|
-
# @return [String, nil] The API key.
|
14
|
-
def api_key
|
15
|
-
@api_key || ENV['GENERATIVE_LANGUAGE_GOOGLE_API_KEY'] || ENV['GEMINI_API_KEY']
|
16
|
-
end
|
17
|
-
|
18
|
-
# Retrieves the base API URL for Google Generative Language API.
|
19
|
-
# Defaults to 'https://generativelanguage.googleapis.com'.
|
20
|
-
# @return [String] The API base URL.
|
21
|
-
def api_url
|
22
|
-
# Note: Google Generative Language API uses a specific path structure like /v1beta/models/{model}:generateContent
|
23
|
-
# The base URL here should be just the domain part.
|
24
|
-
@api_url || 'https://generativelanguage.googleapis.com'
|
25
|
-
end
|
26
|
-
|
27
|
-
# Lazily initializes and returns an instance of the GoogleClient.
|
28
|
-
# Passes itself (the config object) to the client's constructor.
|
29
|
-
# @return [ActiveGenie::Clients::GoogleClient] The client instance.
|
30
|
-
def client
|
31
|
-
@client ||= ::ActiveGenie::Clients::GoogleClient.new(self)
|
32
|
-
end
|
33
|
-
|
34
|
-
# Retrieves the model name designated for the lower tier (e.g., cost-effective, faster).
|
35
|
-
# Defaults to 'gemini-2.0-flash-lite'.
|
36
|
-
# @return [String] The lower tier model name.
|
37
|
-
def lower_tier_model
|
38
|
-
@lower_tier_model || 'gemini-2.0-flash-lite'
|
39
|
-
end
|
40
|
-
|
41
|
-
# Retrieves the model name designated for the middle tier (e.g., balanced performance).
|
42
|
-
# Defaults to 'gemini-2.0-flash'.
|
43
|
-
# @return [String] The middle tier model name.
|
44
|
-
def middle_tier_model
|
45
|
-
@middle_tier_model || 'gemini-2.0-flash'
|
46
|
-
end
|
47
|
-
|
48
|
-
# Retrieves the model name designated for the upper tier (e.g., most capable).
|
49
|
-
# Defaults to 'gemini-2.5-pro-experimental'.
|
50
|
-
# @return [String] The upper tier model name.
|
51
|
-
def upper_tier_model
|
52
|
-
@upper_tier_model || 'gemini-2.5-pro-experimental'
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
@@ -1,54 +0,0 @@
|
|
1
|
-
require_relative '../../clients/internal_company_api_client'
|
2
|
-
require_relative './base_config'
|
3
|
-
|
4
|
-
module ActiveGenie
|
5
|
-
module Configuration::Providers
|
6
|
-
# Configuration class for the Internal Company API client.
|
7
|
-
# Manages API keys, URLs, model selections, and client instantiation.
|
8
|
-
class InternalCompanyApiConfig < BaseConfig
|
9
|
-
NAME = :internal_company_api
|
10
|
-
|
11
|
-
# Retrieves the API key.
|
12
|
-
# Falls back to the INTERNAL_COMPANY_API_KEY environment variable if not set.
|
13
|
-
# @return [String, nil] The API key.
|
14
|
-
def api_key
|
15
|
-
@api_key || ENV['INTERNAL_COMPANY_API_KEY']
|
16
|
-
end
|
17
|
-
|
18
|
-
# Retrieves the base API URL for Internal Company API.
|
19
|
-
# Defaults to 'https://api.internal-company.com/v1'.
|
20
|
-
# @return [String] The API base URL.
|
21
|
-
def api_url
|
22
|
-
@api_url || 'https://api.internal-company.com/v1'
|
23
|
-
end
|
24
|
-
|
25
|
-
# Lazily initializes and returns an instance of the InternalCompanyApiClient.
|
26
|
-
# Passes itself (the config object) to the client's constructor.
|
27
|
-
# @return [ActiveGenie::Clients::InternalCompanyApiClient] The client instance.
|
28
|
-
def client
|
29
|
-
@client ||= ::ActiveGenie::Clients::InternalCompanyApiClient.new(self)
|
30
|
-
end
|
31
|
-
|
32
|
-
# Retrieves the model name designated for the lower tier (e.g., cost-effective, faster).
|
33
|
-
# Defaults to 'internal-basic'.
|
34
|
-
# @return [String] The lower tier model name.
|
35
|
-
def lower_tier_model
|
36
|
-
@lower_tier_model || 'internal-basic'
|
37
|
-
end
|
38
|
-
|
39
|
-
# Retrieves the model name designated for the middle tier (e.g., balanced performance).
|
40
|
-
# Defaults to 'internal-standard'.
|
41
|
-
# @return [String] The middle tier model name.
|
42
|
-
def middle_tier_model
|
43
|
-
@middle_tier_model || 'internal-standard'
|
44
|
-
end
|
45
|
-
|
46
|
-
# Retrieves the model name designated for the upper tier (e.g., most capable).
|
47
|
-
# Defaults to 'internal-premium'.
|
48
|
-
# @return [String] The upper tier model name.
|
49
|
-
def upper_tier_model
|
50
|
-
@upper_tier_model || 'internal-premium'
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
@@ -1,54 +0,0 @@
|
|
1
|
-
require_relative '../../clients/openai_client'
|
2
|
-
require_relative './base_config'
|
3
|
-
|
4
|
-
module ActiveGenie
|
5
|
-
module Configuration::Providers
|
6
|
-
# Configuration class for the OpenAI API client.
|
7
|
-
# Manages API keys, organization IDs, URLs, model selections, and client instantiation.
|
8
|
-
class OpenaiConfig < BaseConfig
|
9
|
-
NAME = :openai
|
10
|
-
|
11
|
-
# Retrieves the API key.
|
12
|
-
# Falls back to the OPENAI_API_KEY environment variable if not set.
|
13
|
-
# @return [String, nil] The API key.
|
14
|
-
def api_key
|
15
|
-
@api_key || ENV['OPENAI_API_KEY']
|
16
|
-
end
|
17
|
-
|
18
|
-
# Retrieves the base API URL for OpenAI API.
|
19
|
-
# Defaults to 'https://api.openai.com/v1'.
|
20
|
-
# @return [String] The API base URL.
|
21
|
-
def api_url
|
22
|
-
@api_url || 'https://api.openai.com/v1'
|
23
|
-
end
|
24
|
-
|
25
|
-
# Lazily initializes and returns an instance of the OpenaiClient.
|
26
|
-
# Passes itself (the config object) to the client's constructor.
|
27
|
-
# @return [ActiveGenie::Clients::OpenaiClient] The client instance.
|
28
|
-
def client
|
29
|
-
@client ||= ::ActiveGenie::Clients::OpenaiClient.new(self)
|
30
|
-
end
|
31
|
-
|
32
|
-
# Retrieves the model name designated for the lower tier (e.g., cost-effective, faster).
|
33
|
-
# Defaults to 'gpt-4o-mini'.
|
34
|
-
# @return [String] The lower tier model name.
|
35
|
-
def lower_tier_model
|
36
|
-
@lower_tier_model || 'gpt-4o-mini'
|
37
|
-
end
|
38
|
-
|
39
|
-
# Retrieves the model name designated for the middle tier (e.g., balanced performance).
|
40
|
-
# Defaults to 'gpt-4o'.
|
41
|
-
# @return [String] The middle tier model name.
|
42
|
-
def middle_tier_model
|
43
|
-
@middle_tier_model || 'gpt-4o'
|
44
|
-
end
|
45
|
-
|
46
|
-
# Retrieves the model name designated for the upper tier (e.g., most capable).
|
47
|
-
# Defaults to 'o1-preview'.
|
48
|
-
# @return [String] The upper tier model name.
|
49
|
-
def upper_tier_model
|
50
|
-
@upper_tier_model || 'o1-preview'
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
@@ -1,40 +0,0 @@
|
|
1
|
-
module ActiveGenie::Configuration
|
2
|
-
class ProvidersConfig
|
3
|
-
def initialize
|
4
|
-
@all = {}
|
5
|
-
@default = nil
|
6
|
-
end
|
7
|
-
|
8
|
-
def register(provider_class)
|
9
|
-
@all ||= {}
|
10
|
-
name = provider_class::NAME
|
11
|
-
@all[name] = provider_class.new
|
12
|
-
define_singleton_method(name) do
|
13
|
-
instance_variable_get("@#{name}") || instance_variable_set("@#{name}", @all[name])
|
14
|
-
end
|
15
|
-
|
16
|
-
self
|
17
|
-
end
|
18
|
-
|
19
|
-
def default
|
20
|
-
@default || @all.values.find { |p| p.api_key }.class::NAME
|
21
|
-
end
|
22
|
-
|
23
|
-
def valid
|
24
|
-
valid_provider_keys = @all.keys.select { |k| @all[k].valid? }
|
25
|
-
@all.slice(*valid_provider_keys)
|
26
|
-
end
|
27
|
-
|
28
|
-
def to_h(config = {})
|
29
|
-
hash_all = {}
|
30
|
-
@all.each do |name, provider|
|
31
|
-
hash_all[name] = provider.to_h(config.dig(name) || {})
|
32
|
-
end
|
33
|
-
hash_all
|
34
|
-
end
|
35
|
-
|
36
|
-
private
|
37
|
-
|
38
|
-
attr_writer :default
|
39
|
-
end
|
40
|
-
end
|
@@ -1,35 +0,0 @@
|
|
1
|
-
module ActiveGenie::Configuration
|
2
|
-
class RuntimeConfig
|
3
|
-
attr_writer :max_tokens, :temperature, :model, :provider, :api_key, :max_retries
|
4
|
-
|
5
|
-
def max_tokens
|
6
|
-
@max_tokens ||= 4096
|
7
|
-
end
|
8
|
-
|
9
|
-
def temperature
|
10
|
-
@temperature ||= 0.1
|
11
|
-
end
|
12
|
-
|
13
|
-
def model
|
14
|
-
@model
|
15
|
-
end
|
16
|
-
|
17
|
-
def provider
|
18
|
-
@provider ||= ActiveGenie.configuration.providers.default
|
19
|
-
end
|
20
|
-
|
21
|
-
def api_key
|
22
|
-
@api_key
|
23
|
-
end
|
24
|
-
|
25
|
-
def max_retries
|
26
|
-
@max_retries ||= 3
|
27
|
-
end
|
28
|
-
|
29
|
-
def to_h(config = {})
|
30
|
-
{
|
31
|
-
max_tokens:, temperature:, model:, provider:, api_key:, max_retries:,
|
32
|
-
}.merge(config)
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
@@ -1,101 +0,0 @@
|
|
1
|
-
|
2
|
-
require_relative '../clients/unified_client'
|
3
|
-
|
4
|
-
module ActiveGenie::DataExtractor
|
5
|
-
class Basic
|
6
|
-
def self.call(...)
|
7
|
-
new(...).call
|
8
|
-
end
|
9
|
-
|
10
|
-
# Extracts structured data from text based on a predefined schema.
|
11
|
-
#
|
12
|
-
# @param text [String] The input text to analyze and extract data from
|
13
|
-
# @param data_to_extract [Hash] Schema defining the data structure to extract.
|
14
|
-
# Each key in the hash represents a field to extract, and its value defines the expected type and constraints.
|
15
|
-
# @param config [Hash] Additional config for the extraction process
|
16
|
-
#
|
17
|
-
# @return [Hash] The extracted data matching the schema structure. Each field will include
|
18
|
-
# both the extracted value and an explanation of how it was derived.
|
19
|
-
#
|
20
|
-
# @example Extract a person's details
|
21
|
-
# schema = {
|
22
|
-
# name: { type: 'string', description: 'Full name of the person' },
|
23
|
-
# age: { type: 'integer', description: 'Age in years' }
|
24
|
-
# }
|
25
|
-
# text = "John Doe is 25 years old"
|
26
|
-
# DataExtractor.call(text, schema)
|
27
|
-
# # => { name: "John Doe", name_explanation: "Found directly in text",
|
28
|
-
# # age: 25, age_explanation: "Explicitly stated as 25 years old" }
|
29
|
-
def initialize(text, data_to_extract, config: {})
|
30
|
-
@text = text
|
31
|
-
@data_to_extract = data_to_extract
|
32
|
-
@config = ActiveGenie::Configuration.to_h(config)
|
33
|
-
end
|
34
|
-
|
35
|
-
def call
|
36
|
-
messages = [
|
37
|
-
{ role: 'system', content: PROMPT },
|
38
|
-
{ role: 'user', content: @text }
|
39
|
-
]
|
40
|
-
|
41
|
-
properties = data_to_extract_with_explaination
|
42
|
-
|
43
|
-
function = {
|
44
|
-
name: 'data_extractor',
|
45
|
-
description: 'Extract structured and typed data from user messages.',
|
46
|
-
parameters: {
|
47
|
-
type: "object",
|
48
|
-
properties:,
|
49
|
-
required: properties.keys
|
50
|
-
}
|
51
|
-
}
|
52
|
-
|
53
|
-
result = ::ActiveGenie::Clients::UnifiedClient.function_calling(
|
54
|
-
messages,
|
55
|
-
function,
|
56
|
-
model_tier: 'lower_tier',
|
57
|
-
config: @config
|
58
|
-
)
|
59
|
-
|
60
|
-
ActiveGenie::Logger.debug({
|
61
|
-
code: :data_extractor,
|
62
|
-
text: @text[0..30],
|
63
|
-
data_to_extract: @data_to_extract,
|
64
|
-
extracted_data: result
|
65
|
-
})
|
66
|
-
|
67
|
-
result
|
68
|
-
end
|
69
|
-
|
70
|
-
private
|
71
|
-
|
72
|
-
PROMPT = <<~PROMPT
|
73
|
-
Extract structured and typed data from user messages.
|
74
|
-
Identify relevant information within user messages and categorize it into predefined data fields with specific data types.
|
75
|
-
|
76
|
-
# Steps
|
77
|
-
1. **Identify Data Types**: Determine the types of data to collect, such as names, dates, email addresses, phone numbers, etc.
|
78
|
-
2. **Extract Information**: Use pattern recognition and language understanding to identify and extract the relevant pieces of data from the user message.
|
79
|
-
3. **Categorize Data**: Assign the extracted data to the appropriate predefined fields.
|
80
|
-
|
81
|
-
# Notes
|
82
|
-
- Handle missing or partial information gracefully.
|
83
|
-
- Manage multiple occurrences of similar data points by prioritizing the first one unless specified otherwise.
|
84
|
-
- Be flexible to handle variations in data format and language clues.
|
85
|
-
PROMPT
|
86
|
-
|
87
|
-
def data_to_extract_with_explaination
|
88
|
-
with_explaination = {}
|
89
|
-
|
90
|
-
@data_to_extract.each do |key, value|
|
91
|
-
with_explaination[key] = value
|
92
|
-
with_explaination["#{key}_explanation"] = {
|
93
|
-
type: 'string',
|
94
|
-
description: "The chain of thought that led to the conclusion about: #{key}. Can be blank if the user didn't provide any context",
|
95
|
-
}
|
96
|
-
end
|
97
|
-
|
98
|
-
with_explaination
|
99
|
-
end
|
100
|
-
end
|
101
|
-
end
|
@@ -1,170 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require_relative '../clients/unified_client'
|
4
|
-
|
5
|
-
module ActiveGenie::Scoring
|
6
|
-
# The Basic class provides a foundation for scoring text content against specified criteria
|
7
|
-
# using AI-powered evaluation. It supports both single and multiple reviewer scenarios,
|
8
|
-
# with the ability to automatically recommend reviewers when none are specified.
|
9
|
-
#
|
10
|
-
# The scoring process evaluates text based on given criteria and returns detailed feedback
|
11
|
-
# including individual reviewer scores, reasoning, and a final aggregated score.
|
12
|
-
#
|
13
|
-
# @example Basic usage with a single reviewer
|
14
|
-
# Basic.call("Sample text", "Evaluate grammar and clarity", ["Grammar Expert"])
|
15
|
-
#
|
16
|
-
# @example Usage with automatic reviewer recommendation
|
17
|
-
# Basic.call("Sample text", "Evaluate technical accuracy")
|
18
|
-
#
|
19
|
-
class Basic
|
20
|
-
# @param text [String] The text content to be evaluated
|
21
|
-
# @param criteria [String] The evaluation criteria or rubric to assess against
|
22
|
-
# @param reviewers [Array<String>] Optional list of specific reviewers. If empty,
|
23
|
-
# reviewers will be automatically recommended based on the content and criteria
|
24
|
-
# @param config [Hash] Additional configuration config that modify the scoring behavior
|
25
|
-
# @return [Hash] The evaluation result containing the scores and reasoning
|
26
|
-
# @return [Number] :final_score The final score of the text based on the criteria and reviewers
|
27
|
-
# @return [String] :final_reasoning Detailed explanation of why the final score was reached
|
28
|
-
def self.call(...)
|
29
|
-
new(...).call
|
30
|
-
end
|
31
|
-
|
32
|
-
def initialize(text, criteria, reviewers = [], config: {})
|
33
|
-
@text = text
|
34
|
-
@criteria = criteria
|
35
|
-
@reviewers = Array(reviewers).compact.uniq
|
36
|
-
@config = ActiveGenie::Configuration.to_h(config)
|
37
|
-
end
|
38
|
-
|
39
|
-
def call
|
40
|
-
messages = [
|
41
|
-
{ role: 'system', content: PROMPT },
|
42
|
-
{ role: 'user', content: "Scoring criteria: #{@criteria}" },
|
43
|
-
{ role: 'user', content: "Text to score: #{@text}" },
|
44
|
-
]
|
45
|
-
|
46
|
-
properties = build_properties
|
47
|
-
|
48
|
-
function = {
|
49
|
-
name: 'scoring',
|
50
|
-
description: 'Score the text based on the given criteria.',
|
51
|
-
parameters: {
|
52
|
-
type: "object",
|
53
|
-
properties:,
|
54
|
-
required: properties.keys
|
55
|
-
}
|
56
|
-
}
|
57
|
-
|
58
|
-
result = ::ActiveGenie::Clients::UnifiedClient.function_calling(
|
59
|
-
messages,
|
60
|
-
function,
|
61
|
-
model_tier: 'lower_tier',
|
62
|
-
config: @config
|
63
|
-
)
|
64
|
-
|
65
|
-
result['final_score'] = 0 if result['final_score'].nil?
|
66
|
-
|
67
|
-
ActiveGenie::Logger.debug({
|
68
|
-
code: :scoring,
|
69
|
-
text: @text[0..30],
|
70
|
-
criteria: @criteria[0..30],
|
71
|
-
reviewers: get_or_recommend_reviewers,
|
72
|
-
score: result['final_score'],
|
73
|
-
reasoning: result['final_reasoning']
|
74
|
-
})
|
75
|
-
|
76
|
-
result
|
77
|
-
end
|
78
|
-
|
79
|
-
private
|
80
|
-
|
81
|
-
def build_properties
|
82
|
-
properties = {}
|
83
|
-
get_or_recommend_reviewers.each do |reviewer|
|
84
|
-
properties["#{reviewer}_reasoning"] = {
|
85
|
-
type: 'string',
|
86
|
-
description: "The reasoning of the scoring process by #{reviewer}.",
|
87
|
-
}
|
88
|
-
properties["#{reviewer}_score"] = {
|
89
|
-
type: 'number',
|
90
|
-
description: "The score given by #{reviewer}.",
|
91
|
-
min: 0,
|
92
|
-
max: 100
|
93
|
-
}
|
94
|
-
end
|
95
|
-
|
96
|
-
properties[:final_score] = {
|
97
|
-
type: 'number',
|
98
|
-
description: 'The final score based on the previous reviewers',
|
99
|
-
}
|
100
|
-
properties[:final_reasoning] = {
|
101
|
-
type: 'string',
|
102
|
-
description: 'The final reasoning based on the previous reviewers',
|
103
|
-
}
|
104
|
-
|
105
|
-
properties
|
106
|
-
end
|
107
|
-
|
108
|
-
def get_or_recommend_reviewers
|
109
|
-
@get_or_recommend_reviewers ||= if @reviewers.count > 0
|
110
|
-
@reviewers
|
111
|
-
else
|
112
|
-
result = RecommendedReviewers.call(@text, @criteria, config: @config)
|
113
|
-
|
114
|
-
[result['reviewer1'], result['reviewer2'], result['reviewer3']]
|
115
|
-
end
|
116
|
-
end
|
117
|
-
|
118
|
-
PROMPT = <<~PROMPT
|
119
|
-
Evaluate and score the provided text based on predefined criteria, using a scoring range of 0 to 100 with 100 representing the highest possible score.
|
120
|
-
|
121
|
-
Follow the instructions below to ensure a comprehensive and objective assessment.
|
122
|
-
|
123
|
-
# Evaluation Process
|
124
|
-
|
125
|
-
1. **Analysis**:
|
126
|
-
- Thoroughly compare the text against each criterion for a comprehensive evaluation.
|
127
|
-
|
128
|
-
2. **Document Deviations**:
|
129
|
-
- Identify and document areas where the content does not align with the specified criteria.
|
130
|
-
|
131
|
-
3. **Highlight Strengths**:
|
132
|
-
- Note notable features or elements that enhance the quality or effectiveness of the content.
|
133
|
-
|
134
|
-
4. **Identify Weaknesses**:
|
135
|
-
- Specify areas where the content fails to meet the criteria or where improvements could be made.
|
136
|
-
|
137
|
-
# Scoring Fairness
|
138
|
-
|
139
|
-
- Ensure the assigned score reflects both the alignment with the criteria and the content's effectiveness.
|
140
|
-
- Consider if the fulfillment of other criteria compensates for areas lacking extreme details.
|
141
|
-
|
142
|
-
# Scoring Range
|
143
|
-
|
144
|
-
Segment scores into five parts before assigning a final score:
|
145
|
-
- **Terrible**: 0-20 - Content does not meet the criteria.
|
146
|
-
- **Bad**: 21-40 - Content is substandard but meets some criteria.
|
147
|
-
- **Average**: 41-60 - Content meets criteria with room for improvement.
|
148
|
-
- **Good**: 61-80 - Content exceeds criteria and is above average.
|
149
|
-
- **Great**: 81-100 - Content exceeds all expectations.
|
150
|
-
|
151
|
-
# Guidelines
|
152
|
-
|
153
|
-
- Maintain objectivity and avoid biases.
|
154
|
-
- Deconstruct each criterion into actionable components for systematic evaluation.
|
155
|
-
- Apply reasonable judgment in assigning a score, justifying your rationale clearly.
|
156
|
-
|
157
|
-
# Output Format
|
158
|
-
|
159
|
-
- Provide a detailed review including:
|
160
|
-
- A final score (0-100)
|
161
|
-
- Specific reasoning for the assigned score, detailing all evaluated criteria
|
162
|
-
- Include both positive aspects and suggested improvements
|
163
|
-
|
164
|
-
# Notes
|
165
|
-
|
166
|
-
- Consider edge cases where the text may partially align with criteria.
|
167
|
-
- If lacking information, reasonably judge and explain your scoring approach.
|
168
|
-
PROMPT
|
169
|
-
end
|
170
|
-
end
|