active_genie 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/lib/{requester → active_genie/clients}/openai.rb +2 -3
- data/lib/{requester/requester.rb → active_genie/clients/router.rb} +8 -8
- data/lib/active_genie/data_extractor/README.md +132 -0
- data/lib/active_genie/data_extractor/basic.rb +88 -0
- data/lib/active_genie/data_extractor/from_informal.rb +58 -0
- data/lib/active_genie/data_extractor.rb +17 -0
- data/lib/active_genie/scoring/README.md +80 -0
- data/lib/active_genie/scoring/basic.rb +116 -0
- data/lib/active_genie/scoring/recommended_reviews.rb +78 -0
- data/lib/active_genie/scoring.rb +17 -0
- data/lib/active_genie.rb +3 -2
- metadata +12 -7
- data/lib/data_extractor/README.md +0 -103
- data/lib/data_extractor/data_extractor.rb +0 -88
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e7f834cd4da9f695c2a3f69b1d7d5516831afaa2d6830f89527b46e7e19bc6f9
|
4
|
+
data.tar.gz: 8e14a0011c2551b975105d2120506461fb5f0e9e33206a6960a9de544148016b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ed3662a287028dacf2f60bda458d0de38ad16d4666421e3b836780d9f2dbb469be935e0b6c3d1fff851f82282c4c4564bebc725354f2f213d0cf21e30ce5ce93
|
7
|
+
data.tar.gz: 50b0d666a061361322ec2bfb38ef0e49110d6b31be3c3b55e3873f6ca5389003d739f209c7a9e1b695e3901de703effe02344863fed80495c104e886b1d1fa3f
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.3
|
@@ -1,10 +1,10 @@
|
|
1
1
|
require 'json'
|
2
2
|
require 'net/http'
|
3
3
|
|
4
|
-
module ActiveGenie
|
4
|
+
module ActiveGenie::Clients
|
5
5
|
class Openai
|
6
6
|
class << self
|
7
|
-
def function_calling(messages, function, options)
|
7
|
+
def function_calling(messages, function, options: {})
|
8
8
|
app_config = ActiveGenie.config_by_model(options[:model])
|
9
9
|
|
10
10
|
model = options[:model] || app_config[:model]
|
@@ -48,7 +48,6 @@ module ActiveGenie
|
|
48
48
|
|
49
49
|
end
|
50
50
|
|
51
|
-
|
52
51
|
API_URL = 'https://api.openai.com/v1/chat/completions'.freeze
|
53
52
|
DEFAULT_HEADERS = {
|
54
53
|
'Content-Type': 'application/json',
|
@@ -1,23 +1,23 @@
|
|
1
1
|
require_relative './openai'
|
2
2
|
|
3
|
-
module ActiveGenie
|
4
|
-
class
|
3
|
+
module ActiveGenie::Clients
|
4
|
+
class Router
|
5
5
|
class << self
|
6
|
-
def function_calling(messages, function, options
|
6
|
+
def function_calling(messages, function, options: {})
|
7
7
|
app_config = ActiveGenie.config_by_model(options[:model])
|
8
|
-
|
8
|
+
|
9
9
|
provider = options[:provider] || app_config[:provider]
|
10
|
-
|
11
|
-
raise "Provider #{provider} not supported" unless
|
10
|
+
client = PROVIDER_TO_CLIENT[provider&.downcase&.strip&.to_sym]
|
11
|
+
raise "Provider \"#{provider}\" not supported" unless client
|
12
12
|
|
13
|
-
response =
|
13
|
+
response = client.function_calling(messages, function, options:)
|
14
14
|
|
15
15
|
clear_invalid_values(response)
|
16
16
|
end
|
17
17
|
|
18
18
|
private
|
19
19
|
|
20
|
-
|
20
|
+
PROVIDER_TO_CLIENT = {
|
21
21
|
openai: Openai,
|
22
22
|
}
|
23
23
|
|
@@ -0,0 +1,132 @@
|
|
1
|
+
# Data Extractor
|
2
|
+
Extract structured data from text using AI-powered analysis, handling informal language and complex expressions.
|
3
|
+
|
4
|
+
## ✨ Features
|
5
|
+
- Structured data extraction - Extract typed data from unstructured text using predefined schemas
|
6
|
+
- Informal text analysis - Identifies and handles informal language patterns, including litotes
|
7
|
+
- Explanation tracking - Provides reasoning for each extracted data point
|
8
|
+
|
9
|
+
## Basic Usage
|
10
|
+
|
11
|
+
Extract structured data from text using predefined schemas:
|
12
|
+
|
13
|
+
```ruby
|
14
|
+
text = "John Doe is 25 years old"
|
15
|
+
schema = {
|
16
|
+
name: { type: 'string', description: 'Full name of the person' },
|
17
|
+
age: { type: 'integer', description: 'Age in years' }
|
18
|
+
}
|
19
|
+
result = ActiveGenie::DataExtractor.call(text, schema)
|
20
|
+
# => {
|
21
|
+
# name: "John Doe",
|
22
|
+
# name_explanation: "Found directly in text",
|
23
|
+
# age: 25,
|
24
|
+
# age_explanation: "Explicitly stated as 25 years old"
|
25
|
+
# }
|
26
|
+
|
27
|
+
product = "Nike Air Max 90 - Size 42 - $199.99"
|
28
|
+
schema = {
|
29
|
+
brand: {
|
30
|
+
type: 'string',
|
31
|
+
enum: ["Nike", "Adidas", "Puma"]
|
32
|
+
},
|
33
|
+
price: {
|
34
|
+
type: 'number',
|
35
|
+
minimum: 0
|
36
|
+
},
|
37
|
+
currency: {
|
38
|
+
type: 'string',
|
39
|
+
enum: ["USD", "EUR"]
|
40
|
+
},
|
41
|
+
size: {
|
42
|
+
type: 'integer',
|
43
|
+
minimum: 35,
|
44
|
+
maximum: 46
|
45
|
+
}
|
46
|
+
}
|
47
|
+
|
48
|
+
result = ActiveGenie::DataExtractor.call(product, schema)
|
49
|
+
# => {
|
50
|
+
# brand: "Nike",
|
51
|
+
# brand_explanation: "Brand name found at start of text",
|
52
|
+
# price: 199.99,
|
53
|
+
# price_explanation: "Price found in USD format at end",
|
54
|
+
# size: 42,
|
55
|
+
# size_explanation: "Size explicitly stated in the middle",
|
56
|
+
# currency: "USD",
|
57
|
+
# currency_explanation: "Derived from $ symbol"
|
58
|
+
# }
|
59
|
+
```
|
60
|
+
|
61
|
+
## Informal Text Processing
|
62
|
+
|
63
|
+
The `from_informal` method extends the basic extraction by analyzing rhetorical devices and informal language patterns like:
|
64
|
+
|
65
|
+
- Litotes ("not bad", "isn't terrible")
|
66
|
+
- Affirmative expressions ("sure", "no problem")
|
67
|
+
- Negative expressions ("nah", "not really")
|
68
|
+
- Hedging ("maybe", "I guess")
|
69
|
+
|
70
|
+
### Example
|
71
|
+
|
72
|
+
```ruby
|
73
|
+
text = "The weather isn't bad today"
|
74
|
+
schema = {
|
75
|
+
mood: { type: 'string', description: 'The mood of the message' }
|
76
|
+
}
|
77
|
+
|
78
|
+
result = ActiveGenie::DataExtractor.from_informal(text, schema)
|
79
|
+
# => {
|
80
|
+
# mood: "positive",
|
81
|
+
# mood_explanation: "Speaker views weather favorably",
|
82
|
+
# message_litote: true,
|
83
|
+
# litote_rephrased: "The weather is good today"
|
84
|
+
# }
|
85
|
+
```
|
86
|
+
|
87
|
+
### Usage Notes
|
88
|
+
- Best suited for processing conversational user inputs
|
89
|
+
- Automatically detects and interprets litotes
|
90
|
+
- Provides rephrased positive statements for litotes
|
91
|
+
- May require more processing time due to rhetorical analysis
|
92
|
+
- Accuracy depends on context clarity
|
93
|
+
|
94
|
+
⚠️ Performance Impact: This method performs additional rhetorical analysis, which can increase processing time.
|
95
|
+
|
96
|
+
## Interface
|
97
|
+
|
98
|
+
### `.call(text, data_to_extract, options = {})`
|
99
|
+
Extracts structured data from text based on a predefined schema.
|
100
|
+
|
101
|
+
#### Parameters
|
102
|
+
| Name | Type | Description | Required | Example |
|
103
|
+
| --- | --- | --- | --- | --- |
|
104
|
+
| `text` | `String` | The text to analyze and extract data from | Yes | "John Doe is 25 years old" |
|
105
|
+
| `data_to_extract` | `Hash` | Schema defining the data structure to extract | Yes | `{ name: { type: 'string' } }` |
|
106
|
+
| `options` | `Hash` | Additional extraction configuration | No | `{ model: "gpt-4" }` |
|
107
|
+
|
108
|
+
#### Options
|
109
|
+
| Name | Type | Description |
|
110
|
+
| --- | --- | --- |
|
111
|
+
| `model` | `String` | The model to use for the extraction |
|
112
|
+
| `api_key` | `String` | The API key to use for the extraction |
|
113
|
+
|
114
|
+
#### Returns
|
115
|
+
`Hash` containing:
|
116
|
+
- Extracted values matching the schema structure
|
117
|
+
- Explanation field for each extracted value
|
118
|
+
- Additional analysis fields when using `from_informal`
|
119
|
+
|
120
|
+
### `.from_informal(text, data_to_extract, options = {})`
|
121
|
+
Extends basic extraction with rhetorical analysis, particularly for litotes.
|
122
|
+
|
123
|
+
#### Additional Return Fields
|
124
|
+
| Name | Type | Description |
|
125
|
+
| --- | --- | --- |
|
126
|
+
| `message_litote` | `Boolean` | Whether the text contains a litote |
|
127
|
+
| `litote_rephrased` | `String` | Positive rephrasing of any detected litote |
|
128
|
+
|
129
|
+
⚠️ Performance Considerations
|
130
|
+
- Both methods may require multiple AI model calls
|
131
|
+
- Informal processing requires additional rhetorical analysis
|
132
|
+
- Consider background processing for production use
|
@@ -0,0 +1,88 @@
|
|
1
|
+
require_relative '../clients/router.rb'
|
2
|
+
|
3
|
+
module ActiveGenie::DataExtractor
|
4
|
+
class Basic
|
5
|
+
def self.call(text, data_to_extract, options: {})
|
6
|
+
new(text, data_to_extract, options:).call
|
7
|
+
end
|
8
|
+
|
9
|
+
# Extracts structured data from text based on a predefined schema.
|
10
|
+
#
|
11
|
+
# @param text [String] The input text to analyze and extract data from
|
12
|
+
# @param data_to_extract [Hash] Schema defining the data structure to extract.
|
13
|
+
# Each key in the hash represents a field to extract, and its value defines the expected type and constraints.
|
14
|
+
# @param options [Hash] Additional options for the extraction process
|
15
|
+
# @option options [String] :model The model to use for the extraction
|
16
|
+
# @option options [String] :api_key The API key to use for the extraction
|
17
|
+
#
|
18
|
+
# @return [Hash] The extracted data matching the schema structure. Each field will include
|
19
|
+
# both the extracted value and an explanation of how it was derived.
|
20
|
+
#
|
21
|
+
# @example Extract a person's details
|
22
|
+
# schema = {
|
23
|
+
# name: { type: 'string', description: 'Full name of the person' },
|
24
|
+
# age: { type: 'integer', description: 'Age in years' }
|
25
|
+
# }
|
26
|
+
# text = "John Doe is 25 years old"
|
27
|
+
# DataExtractor.call(text, schema)
|
28
|
+
# # => { name: "John Doe", name_explanation: "Found directly in text",
|
29
|
+
# # age: 25, age_explanation: "Explicitly stated as 25 years old" }
|
30
|
+
def initialize(text, data_to_extract, options: {})
|
31
|
+
@text = text
|
32
|
+
@data_to_extract = data_to_extract
|
33
|
+
@options = options
|
34
|
+
end
|
35
|
+
|
36
|
+
def call
|
37
|
+
messages = [
|
38
|
+
{ role: 'system', content: PROMPT },
|
39
|
+
{ role: 'user', content: @text }
|
40
|
+
]
|
41
|
+
function = {
|
42
|
+
name: 'data_extractor',
|
43
|
+
description: 'Extract structured and typed data from user messages.',
|
44
|
+
schema: {
|
45
|
+
type: "object",
|
46
|
+
properties: data_to_extract_with_explaination
|
47
|
+
}
|
48
|
+
}
|
49
|
+
|
50
|
+
::ActiveGenie::Clients::Router.function_calling(messages, function, options: @options)
|
51
|
+
end
|
52
|
+
|
53
|
+
private
|
54
|
+
|
55
|
+
PROMPT = <<~PROMPT
|
56
|
+
Extract structured and typed data from user messages.
|
57
|
+
Identify relevant information within user messages and categorize it into predefined data fields with specific data types.
|
58
|
+
|
59
|
+
# Steps
|
60
|
+
1. **Identify Data Types**: Determine the types of data to collect, such as names, dates, email addresses, phone numbers, etc.
|
61
|
+
2. **Extract Information**: Use pattern recognition and language understanding to identify and extract the relevant pieces of data from the user message.
|
62
|
+
3. **Categorize Data**: Assign the extracted data to the appropriate predefined fields.
|
63
|
+
4. **Structure Data**: Format the extracted and categorized data in a structured format, such as JSON.
|
64
|
+
|
65
|
+
# Output Format
|
66
|
+
The output should be a JSON object containing fields with their corresponding extracted values. If a value is not found, the field should still be included with a null value.
|
67
|
+
|
68
|
+
# Notes
|
69
|
+
- Handle missing or partial information gracefully.
|
70
|
+
- Manage multiple occurrences of similar data points by prioritizing the first one unless specified otherwise.
|
71
|
+
- Be flexible to handle variations in data format and language clues.
|
72
|
+
PROMPT
|
73
|
+
|
74
|
+
def data_to_extract_with_explaination
|
75
|
+
with_explaination = {}
|
76
|
+
|
77
|
+
@data_to_extract.each do |key, value|
|
78
|
+
with_explaination[key] = value
|
79
|
+
with_explaination["#{key}_explanation"] = {
|
80
|
+
type: 'string',
|
81
|
+
description: "The chain of thought that led to the conclusion about: #{key}. Can be blank if the user didn't provide any context",
|
82
|
+
}
|
83
|
+
end
|
84
|
+
|
85
|
+
with_explaination
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
module ActiveGenie::DataExtractor
|
2
|
+
class FromInformal
|
3
|
+
def self.call(text, data_to_extract, options: {})
|
4
|
+
new(text, data_to_extract, options:).call()
|
5
|
+
end
|
6
|
+
|
7
|
+
# Extracts data from informal text while also detecting litotes and their meanings.
|
8
|
+
# This method extends the basic extraction by analyzing rhetorical devices.
|
9
|
+
#
|
10
|
+
# @param text [String] The informal text to analyze
|
11
|
+
# @param data_to_extract [Hash] Schema defining the data structure to extract
|
12
|
+
# @param options [Hash] Additional options for the extraction process
|
13
|
+
#
|
14
|
+
# @return [Hash] The extracted data including litote analysis. In addition to the
|
15
|
+
# schema-defined fields, includes:
|
16
|
+
# - message_litote: Whether the text contains a litote
|
17
|
+
# - litote_rephrased: The positive rephrasing of any detected litote
|
18
|
+
#
|
19
|
+
# @example Analyze text with litote
|
20
|
+
# text = "The weather isn't bad today"
|
21
|
+
# schema = { mood: { type: 'string', description: 'The mood of the message' } }
|
22
|
+
# DataExtractor.from_informal(text, schema)
|
23
|
+
# # => { mood: "positive", mood_explanation: "Speaker views weather favorably",
|
24
|
+
# # message_litote: true,
|
25
|
+
# # litote_rephrased: "The weather is good today" }
|
26
|
+
def initialize(text, data_to_extract, options: {})
|
27
|
+
@text = text
|
28
|
+
@data_to_extract = data_to_extract
|
29
|
+
@options = options
|
30
|
+
end
|
31
|
+
|
32
|
+
def call
|
33
|
+
response = Basic.call(@text, data_to_extract_with_litote, options: @options)
|
34
|
+
|
35
|
+
if response['message_litote']
|
36
|
+
response = Basic.call(response['litote_rephrased'], @data_to_extract, options: @options)
|
37
|
+
end
|
38
|
+
|
39
|
+
response
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
def data_to_extract_with_litote
|
45
|
+
{
|
46
|
+
**@data_to_extract,
|
47
|
+
message_litote: {
|
48
|
+
type: 'boolean',
|
49
|
+
description: 'Return true if the message is a litote. A litote is a figure of speech that uses understatement to emphasize a point by stating a negative to further affirm a positive, often incorporating double negatives for effect.'
|
50
|
+
},
|
51
|
+
litote_rephrased: {
|
52
|
+
type: 'string',
|
53
|
+
description: 'The true meaning of the litote. Rephrase the message to a positive and active statement.'
|
54
|
+
}
|
55
|
+
}
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require_relative 'data_extractor/basic'
|
2
|
+
require_relative 'data_extractor/from_informal'
|
3
|
+
|
4
|
+
module ActiveGenie
|
5
|
+
# Extract structured data from text using AI-powered analysis, handling informal language and complex expressions.
|
6
|
+
module DataExtractor
|
7
|
+
module_function
|
8
|
+
|
9
|
+
def basic(text, data_to_extract, options: {})
|
10
|
+
Basic.call(text, data_to_extract, options:)
|
11
|
+
end
|
12
|
+
|
13
|
+
def from_informal(text, data_to_extract, options: {})
|
14
|
+
FromInformal.call(text, data_to_extract, options:)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
# Scoring
|
2
|
+
Text evaluation system that provides detailed scoring and feedback using multiple expert reviewers.
|
3
|
+
|
4
|
+
## Features
|
5
|
+
- Multi-reviewer evaluation - Get scores and feedback from multiple AI-powered expert reviewers
|
6
|
+
- Automatic reviewer selection - Smart recommendation of reviewers based on content and criteria
|
7
|
+
- Detailed feedback - Comprehensive reasoning for each reviewer's score
|
8
|
+
- Customizable weights - Adjust the importance of different reviewers' scores
|
9
|
+
- Flexible criteria - Score text against any specified evaluation criteria
|
10
|
+
|
11
|
+
## Basic Usage
|
12
|
+
|
13
|
+
Score text using predefined reviewers:
|
14
|
+
|
15
|
+
```ruby
|
16
|
+
text = "The code implements a binary search algorithm with O(log n) complexity"
|
17
|
+
criteria = "Evaluate technical accuracy and clarity"
|
18
|
+
reviewers = ["Algorithm Expert", "Technical Writer"]
|
19
|
+
|
20
|
+
result = ActiveGenie::Scoring::Basic.call(text, criteria, reviewers)
|
21
|
+
# => {
|
22
|
+
# algorithm_expert_score: 95,
|
23
|
+
# algorithm_expert_reasoning: "Accurately describes binary search and its complexity",
|
24
|
+
# technical_writer_score: 90,
|
25
|
+
# technical_writer_reasoning: "Clear and concise explanation of the algorithm",
|
26
|
+
# final_score: 92.5
|
27
|
+
# }
|
28
|
+
```
|
29
|
+
|
30
|
+
## Automatic Reviewer Selection
|
31
|
+
|
32
|
+
When no reviewers are specified, the system automatically recommends appropriate reviewers:
|
33
|
+
|
34
|
+
```ruby
|
35
|
+
text = "The patient shows signs of improved cardiac function"
|
36
|
+
criteria = "Evaluate medical accuracy and clarity"
|
37
|
+
|
38
|
+
result = ActiveGenie::Scoring::Basic.call(text, criteria)
|
39
|
+
# => {
|
40
|
+
# cardiologist_score: 88,
|
41
|
+
# cardiologist_reasoning: "Accurate assessment of cardiac improvement",
|
42
|
+
# medical_writer_score: 85,
|
43
|
+
# medical_writer_reasoning: "Clear communication of medical findings",
|
44
|
+
# general_practitioner_score: 90,
|
45
|
+
# general_practitioner_reasoning: "Well-structured medical observation",
|
46
|
+
# final_score: 87.7
|
47
|
+
# }
|
48
|
+
```
|
49
|
+
|
50
|
+
## Interface
|
51
|
+
|
52
|
+
### `Basic.call(text, criteria, reviewers = [], options: {})`
|
53
|
+
Main interface for scoring text content.
|
54
|
+
|
55
|
+
#### Parameters
|
56
|
+
- `text` [String] - The text content to be evaluated
|
57
|
+
- `criteria` [String] - The evaluation criteria or rubric to assess against
|
58
|
+
- `reviewers` [Array<String>] - Optional list of specific reviewers
|
59
|
+
- `options` [Hash] - Additional configuration options
|
60
|
+
- `:detailed_feedback` [Boolean] - Request more detailed feedback (WIP)
|
61
|
+
- `:reviewer_weights` [Hash] - Custom weights for different reviewers (WIP)
|
62
|
+
|
63
|
+
### `RecommendedReviews.call(text, criteria, options: {})`
|
64
|
+
Recommends appropriate reviewers based on content and criteria.
|
65
|
+
|
66
|
+
#### Parameters
|
67
|
+
- `text` [String] - The text content to analyze
|
68
|
+
- `criteria` [String] - The evaluation criteria
|
69
|
+
- `options` [Hash] - Additional configuration options
|
70
|
+
- `:prefer_technical` [Boolean] - Favor technical expertise (WIP)
|
71
|
+
- `:prefer_domain` [Boolean] - Favor domain expertise (WIP)
|
72
|
+
|
73
|
+
### Usage Notes
|
74
|
+
- Best suited for objective evaluation of text content
|
75
|
+
- Provides balanced scoring through multiple reviewers
|
76
|
+
- Automatically handles reviewer selection when needed
|
77
|
+
- Supports custom weighting of reviewer scores
|
78
|
+
- Returns detailed reasoning for each score
|
79
|
+
|
80
|
+
Performance Impact: Using multiple reviewers or requesting detailed feedback may increase processing time.
|
@@ -0,0 +1,116 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../clients/router'
|
4
|
+
|
5
|
+
module ActiveGenie::Scoring
|
6
|
+
# The Basic class provides a foundation for scoring text content against specified criteria
|
7
|
+
# using AI-powered evaluation. It supports both single and multiple reviewer scenarios,
|
8
|
+
# with the ability to automatically recommend reviewers when none are specified.
|
9
|
+
#
|
10
|
+
# The scoring process evaluates text based on given criteria and returns detailed feedback
|
11
|
+
# including individual reviewer scores, reasoning, and a final aggregated score.
|
12
|
+
#
|
13
|
+
# @example Basic usage with a single reviewer
|
14
|
+
# Basic.call("Sample text", "Evaluate grammar and clarity", ["Grammar Expert"])
|
15
|
+
#
|
16
|
+
# @example Usage with automatic reviewer recommendation
|
17
|
+
# Basic.call("Sample text", "Evaluate technical accuracy")
|
18
|
+
#
|
19
|
+
class Basic
|
20
|
+
def self.call(text, criteria, reviewers = [], options: {})
|
21
|
+
new(text, criteria, reviewers, options:).call
|
22
|
+
end
|
23
|
+
|
24
|
+
# Initializes a new scoring evaluation instance
|
25
|
+
#
|
26
|
+
# @param text [String] The text content to be evaluated
|
27
|
+
# @param criteria [String] The evaluation criteria or rubric to assess against
|
28
|
+
# @param reviewers [Array<String>] Optional list of specific reviewers. If empty,
|
29
|
+
# reviewers will be automatically recommended based on the content and criteria
|
30
|
+
# @param options [Hash] Additional configuration options that modify the scoring behavior
|
31
|
+
# @option options [Boolean] :detailed_feedback Request more detailed feedback in the reasoning
|
32
|
+
# @option options [Hash] :reviewer_weights Custom weights for different reviewers
|
33
|
+
def initialize(text, criteria, reviewers = [], options: {})
|
34
|
+
@text = text
|
35
|
+
@criteria = criteria
|
36
|
+
@reviewers = Array(reviewers).compact.uniq
|
37
|
+
@options = options
|
38
|
+
end
|
39
|
+
|
40
|
+
def call
|
41
|
+
messages = [
|
42
|
+
{ role: 'system', content: PROMPT },
|
43
|
+
{ role: 'user', content: "Scoring criteria: #{@criteria}" },
|
44
|
+
{ role: 'user', content: "Text to score: #{@text}" },
|
45
|
+
]
|
46
|
+
|
47
|
+
properties = {}
|
48
|
+
get_or_recommend_reviewers.each do |reviewer|
|
49
|
+
properties["#{reviewer}_reasoning"] = {
|
50
|
+
type: 'string',
|
51
|
+
description: "The reasoning of the scoring process by #{reviewer}.",
|
52
|
+
}
|
53
|
+
properties["#{reviewer}_score"] = {
|
54
|
+
type: 'number',
|
55
|
+
description: "The score given by #{reviewer}.",
|
56
|
+
min: 0,
|
57
|
+
max: 100
|
58
|
+
}
|
59
|
+
end
|
60
|
+
|
61
|
+
function = {
|
62
|
+
name: 'scoring',
|
63
|
+
description: 'Score the text based on the given criteria.',
|
64
|
+
schema: {
|
65
|
+
type: "object",
|
66
|
+
properties: {
|
67
|
+
**properties,
|
68
|
+
final_score: {
|
69
|
+
type: 'number',
|
70
|
+
description: 'The final score based on the previous reviewers',
|
71
|
+
},
|
72
|
+
final_reasoning: {
|
73
|
+
type: 'string',
|
74
|
+
description: 'The final reasoning based on the previous reviewers',
|
75
|
+
}
|
76
|
+
}
|
77
|
+
}
|
78
|
+
}
|
79
|
+
|
80
|
+
::ActiveGenie::Clients::Router.function_calling(messages, function, options: @options)
|
81
|
+
end
|
82
|
+
|
83
|
+
private
|
84
|
+
|
85
|
+
def get_or_recommend_reviewers
|
86
|
+
@get_or_recommend_reviewers ||= if @reviewers.count > 0
|
87
|
+
@reviewers
|
88
|
+
else
|
89
|
+
recommended_reviews = RecommendedReviews.call(@text, @criteria, options: @options)
|
90
|
+
|
91
|
+
[recommended_reviews[:reviewer1], recommended_reviews[:reviewer2], recommended_reviews[:reviewer3]]
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
PROMPT = <<~PROMPT
|
96
|
+
Evaluate and score the provided text based on predefined criteria, which may include rules, keywords, or patterns. Use a scoring range of 0 to 100, with 100 representing the highest possible score. Follow the instructions below to ensure an accurate and objective assessment.
|
97
|
+
|
98
|
+
# Evaluation Process
|
99
|
+
1. **Analysis**: Thoroughly compare the text against each criterion to ensure comprehensive evaluation.
|
100
|
+
2. **Document Deviations**: Clearly identify and document any areas where the content does not align with the specified criteria.
|
101
|
+
3. **Highlight Strengths**: Emphasize notable features or elements that enhance the overall quality or effectiveness of the content.
|
102
|
+
4. **Identify Weaknesses**: Specify areas where the content fails to meet the criteria or where improvements could be made.
|
103
|
+
|
104
|
+
# Output Requirements
|
105
|
+
Provide a detailed review, including:
|
106
|
+
- A final score (0-100)
|
107
|
+
- Specific reasoning for the assigned score, covering all evaluated criteria.
|
108
|
+
- Ensure the reasoning includes both positive aspects and suggested improvements.
|
109
|
+
|
110
|
+
# Guidelines
|
111
|
+
- Maintain objectivity, avoiding biases or preconceived notions.
|
112
|
+
- Deconstruct each criterion into actionable components for a systematic evaluation.
|
113
|
+
- If the text lacks information, apply reasonable judgment to assign a score while clearly explaining the rationale.
|
114
|
+
PROMPT
|
115
|
+
end
|
116
|
+
end
|
@@ -0,0 +1,78 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../clients/router.rb'
|
4
|
+
|
5
|
+
module ActiveGenie::Scoring
|
6
|
+
# The RecommendedReviews class intelligently suggests appropriate reviewer roles
|
7
|
+
# for evaluating text content based on specific criteria. It uses AI to analyze
|
8
|
+
# the content and criteria to identify the most suitable subject matter experts.
|
9
|
+
#
|
10
|
+
# The class ensures a balanced and comprehensive review process by recommending
|
11
|
+
# three distinct reviewer roles with complementary expertise and perspectives.
|
12
|
+
#
|
13
|
+
# @example Getting recommended reviewers for technical content
|
14
|
+
# RecommendedReviews.call("Technical documentation about API design",
|
15
|
+
# "Evaluate technical accuracy and clarity")
|
16
|
+
# # => { reviewer1: "API Architect", reviewer2: "Technical Writer",
|
17
|
+
# # reviewer3: "Developer Advocate", reasoning: "..." }
|
18
|
+
#
|
19
|
+
class RecommendedReviews
|
20
|
+
def self.call(text, criteria, options: {})
|
21
|
+
new(text, criteria, options:).call
|
22
|
+
end
|
23
|
+
|
24
|
+
# Initializes a new reviewer recommendation instance
|
25
|
+
#
|
26
|
+
# @param text [String] The text content to analyze for reviewer recommendations
|
27
|
+
# @param criteria [String] The evaluation criteria that will guide reviewer selection
|
28
|
+
# @param options [Hash] Additional configuration options that modify the recommendation process
|
29
|
+
# @option options [Boolean] :prefer_technical Whether to favor technical expertise
|
30
|
+
# @option options [Boolean] :prefer_domain Whether to favor domain expertise
|
31
|
+
def initialize(text, criteria, options: {})
|
32
|
+
@text = text
|
33
|
+
@criteria = criteria
|
34
|
+
@options = options
|
35
|
+
end
|
36
|
+
|
37
|
+
def call
|
38
|
+
messages = [
|
39
|
+
{ role: 'system', content: PROMPT },
|
40
|
+
{ role: 'user', content: "Scoring criteria: #{@criteria}" },
|
41
|
+
{ role: 'user', content: "Text to score: #{@text}" },
|
42
|
+
]
|
43
|
+
|
44
|
+
function = {
|
45
|
+
name: 'identify_reviewers',
|
46
|
+
description: 'Discover reviewers based on the text and given criteria.',
|
47
|
+
schema: {
|
48
|
+
type: "object",
|
49
|
+
properties: {
|
50
|
+
reasoning: { type: 'string' },
|
51
|
+
reviewer1: { type: 'string' },
|
52
|
+
reviewer2: { type: 'string' },
|
53
|
+
reviewer3: { type: 'string' },
|
54
|
+
}
|
55
|
+
}
|
56
|
+
}
|
57
|
+
|
58
|
+
::ActiveGenie::Clients::Router.function_calling(messages, function, options:)
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
PROMPT = <<~PROMPT
|
64
|
+
Identify the top 3 suitable reviewer titles or roles based on the provided text and criteria. Selected reviewers must possess subject matter expertise, offer valuable insights, and ensure diverse yet aligned perspectives on the content.
|
65
|
+
|
66
|
+
# Instructions
|
67
|
+
1. **Analyze the Text and Criteria**: Examine the content and criteria to identify relevant reviewer titles or roles.
|
68
|
+
2. **Determine Subject Matter Expertise**: Select reviewers with substantial knowledge or experience in the subject area.
|
69
|
+
3. **Evaluate Insight Contribution**: Prioritize titles or roles capable of providing meaningful and actionable feedback on the content.
|
70
|
+
4. **Incorporate Perspective Diversity**: Ensure the selection includes reviewers with varied but complementary viewpoints while maintaining alignment with the criteria.
|
71
|
+
|
72
|
+
# Constraints
|
73
|
+
- Selected reviewers must align with the content’s subject matter and criteria.
|
74
|
+
- Include reasoning for how each choice supports a thorough and insightful review.
|
75
|
+
- Avoid redundant or overly similar titles/roles to maintain diversity.
|
76
|
+
PROMPT
|
77
|
+
end
|
78
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require_relative 'scoring/basic'
|
2
|
+
require_relative 'scoring/recommended_reviews'
|
3
|
+
|
4
|
+
module ActiveGenie
|
5
|
+
# Text evaluation system that provides detailed scoring and feedback using multiple expert reviewers
|
6
|
+
module Scoring
|
7
|
+
module_function
|
8
|
+
|
9
|
+
def basic(text, criteria, reviewers = [], options: {})
|
10
|
+
Basic.call(text, criteria, reviewers, options:)
|
11
|
+
end
|
12
|
+
|
13
|
+
def recommended_reviews(text, criteria, options: {})
|
14
|
+
RecommendedReviews.call(text, criteria, options:)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
data/lib/active_genie.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
module ActiveGenie
|
2
|
-
autoload :DataExtractor, File.join(__dir__, '
|
2
|
+
autoload :DataExtractor, File.join(__dir__, 'active_genie/data_extractor')
|
3
|
+
autoload :Scoring, File.join(__dir__, 'active_genie/scoring')
|
3
4
|
autoload :Configuration, File.join(__dir__, 'active_genie/configuration')
|
4
5
|
|
5
6
|
class << self
|
@@ -16,7 +17,7 @@ module ActiveGenie
|
|
16
17
|
end
|
17
18
|
|
18
19
|
def config_by_model(model)
|
19
|
-
config.values[model&.to_s] || config.values.values.first || {}
|
20
|
+
config.values[model&.to_s&.downcase&.strip] || config.values.values.first || {}
|
20
21
|
end
|
21
22
|
end
|
22
23
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: active_genie
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Radamés Roriz
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2025-01-
|
11
|
+
date: 2025-01-25 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: |2
|
14
14
|
ActiveGenie provides a robust toolkit for integrating AI capabilities into Ruby applications.
|
@@ -27,11 +27,17 @@ files:
|
|
27
27
|
- README.md
|
28
28
|
- VERSION
|
29
29
|
- lib/active_genie.rb
|
30
|
+
- lib/active_genie/clients/openai.rb
|
31
|
+
- lib/active_genie/clients/router.rb
|
30
32
|
- lib/active_genie/configuration.rb
|
31
|
-
- lib/data_extractor
|
32
|
-
- lib/data_extractor/
|
33
|
-
- lib/
|
34
|
-
- lib/
|
33
|
+
- lib/active_genie/data_extractor.rb
|
34
|
+
- lib/active_genie/data_extractor/README.md
|
35
|
+
- lib/active_genie/data_extractor/basic.rb
|
36
|
+
- lib/active_genie/data_extractor/from_informal.rb
|
37
|
+
- lib/active_genie/scoring.rb
|
38
|
+
- lib/active_genie/scoring/README.md
|
39
|
+
- lib/active_genie/scoring/basic.rb
|
40
|
+
- lib/active_genie/scoring/recommended_reviews.rb
|
35
41
|
- lib/tasks/install.rake
|
36
42
|
- lib/tasks/templates/active_ai.yml
|
37
43
|
homepage: https://github.com/Roriz/active_genie
|
@@ -41,7 +47,6 @@ metadata:
|
|
41
47
|
homepage_uri: https://github.com/Roriz/active_genie
|
42
48
|
documentation_uri: https://github.com/Roriz/active_genie/wiki
|
43
49
|
changelog_uri: https://github.com/Roriz/active_genie/blob/master/CHANGELOG.md
|
44
|
-
source_code_uri: https://github.com/Roriz/active_genie
|
45
50
|
bug_tracker_uri: https://github.com/Roriz/active_genie/issues
|
46
51
|
rubygems_mfa_required: 'true'
|
47
52
|
post_install_message:
|
@@ -1,103 +0,0 @@
|
|
1
|
-
# Data Extractor
|
2
|
-
Extract structured data from text using LLM-powered analysis, handling informal language and complex expressions.
|
3
|
-
|
4
|
-
## ✨ Features
|
5
|
-
- Simple extraction - Extract structured data from unstructured text
|
6
|
-
- Informal extraction - Identifies and handles informal language patterns
|
7
|
-
|
8
|
-
## Basic Usage
|
9
|
-
|
10
|
-
Extract structured data from text using predefined schemas:
|
11
|
-
|
12
|
-
```ruby
|
13
|
-
text = "iPhone 14 Pro Max"
|
14
|
-
schema = {
|
15
|
-
brand: { type: 'string' },
|
16
|
-
model: { type: 'string' }
|
17
|
-
}
|
18
|
-
result = ActiveGenie::DataExtractor.call(text, schema)
|
19
|
-
# => { brand: "iPhone", model: "14 Pro Max" }
|
20
|
-
|
21
|
-
product = "Nike Air Max 90 - Size 42 - $199.99"
|
22
|
-
schema = {
|
23
|
-
brand: {
|
24
|
-
type: 'string',
|
25
|
-
enum: ["Nike", "Adidas", "Puma"]
|
26
|
-
},
|
27
|
-
price: {
|
28
|
-
type: 'number',
|
29
|
-
minimum: 0
|
30
|
-
},
|
31
|
-
currency: {
|
32
|
-
type: 'string',
|
33
|
-
enum: ["USD", "EUR"]
|
34
|
-
},
|
35
|
-
size: {
|
36
|
-
type: 'integer',
|
37
|
-
minimum: 35,
|
38
|
-
maximum: 46
|
39
|
-
}
|
40
|
-
}
|
41
|
-
|
42
|
-
result = ActiveGenie::DataExtractor.call(product, schema)
|
43
|
-
# => { brand: "Nike", price: 199.99, size: 42, currency: "USD" }
|
44
|
-
```
|
45
|
-
|
46
|
-
## Informal Text Processing
|
47
|
-
|
48
|
-
The `from_informal` method helps extract structured data from casual, conversational text by interpreting common informal expressions and linguistic patterns like:
|
49
|
-
|
50
|
-
- Affirmative expressions ("sure", "no problem", "you bet")
|
51
|
-
- Negative expressions ("nah", "not really", "pass")
|
52
|
-
- Hedging ("maybe", "I guess", "probably")
|
53
|
-
- Litotes ("not bad", "not the worst")
|
54
|
-
|
55
|
-
### Example
|
56
|
-
|
57
|
-
```ruby
|
58
|
-
text = <<~TEXT
|
59
|
-
system: Would you like to proceed with the installation?
|
60
|
-
user: not bad
|
61
|
-
TEXT
|
62
|
-
|
63
|
-
data_to_extract = {
|
64
|
-
user_consent: { type: 'boolean' }
|
65
|
-
}
|
66
|
-
|
67
|
-
result = ActiveGenie::DataExtractor.from_informal(text, data_to_extract)
|
68
|
-
puts result # => { user_consent: true }
|
69
|
-
```
|
70
|
-
|
71
|
-
### Usage Notes
|
72
|
-
- Best suited for processing conversational user inputs
|
73
|
-
- Handles ambiguous or indirect responses
|
74
|
-
- Useful for chatbots and conversational interfaces
|
75
|
-
- May require more processing time and tokens
|
76
|
-
- Accuracy depends on context clarity
|
77
|
-
|
78
|
-
⚠️ Performance Impact: This method uses additional language processing, which can increase token usage and processing time.
|
79
|
-
|
80
|
-
## Interface
|
81
|
-
`.call(text, data_to_extract, options = {})`
|
82
|
-
Extracts structured data from text based on a schema.
|
83
|
-
|
84
|
-
### Parameters
|
85
|
-
| Name | Type | Description | Required | Example | Default |
|
86
|
-
| --- | --- | --- | --- | --- | --- |
|
87
|
-
| `text` | `String` | The text to extract data from. Max 1000 chars. | Yes | "These Nike shoes are red" | - |
|
88
|
-
| `data_to_extract` | `Hash` | [JSON Schema object](https://json-schema.org/understanding-json-schema/reference/object) defining data structure | Yes | `{ category: { type: 'string', enum: ["shoes"] } }` | - |
|
89
|
-
| `options` | `Hash` | Additional provider configuration options | No | `{ model: "gpt-4" }` | `{}` |
|
90
|
-
|
91
|
-
### Returns
|
92
|
-
`Hash` - Dynamic hash based on the `data_to_extract` schema.
|
93
|
-
|
94
|
-
### Options
|
95
|
-
| Name | Type | Description | Default |
|
96
|
-
| --- | --- | --- | --- |
|
97
|
-
| `model` | `String` | The model name | `YAML.load_file(config_file).first.model` |
|
98
|
-
| `api_key` | `String` | The API key to use or api_key from model on config.yml | `YAML.load_file(config_file).first.api_key` |
|
99
|
-
|
100
|
-
⚠️ Performance Considerations
|
101
|
-
- Processes may require multiple LLM calls
|
102
|
-
- Expect ~100 tokens per request + the text from input
|
103
|
-
- Consider background processing for production use
|
@@ -1,88 +0,0 @@
|
|
1
|
-
require_relative '../requester/requester.rb'
|
2
|
-
|
3
|
-
module ActiveGenie
|
4
|
-
class DataExtractor
|
5
|
-
class << self
|
6
|
-
# Extracts data from user_texts based on the schema defined in data_to_extract.
|
7
|
-
# @param text [String] The text to extract data from.
|
8
|
-
# @param data_to_extract [Hash] The schema to extract data from the text.
|
9
|
-
# @param options [Hash] The options to pass to the function.
|
10
|
-
# @return [Hash] The extracted data.
|
11
|
-
def call(text, data_to_extract, options: {})
|
12
|
-
messages = [
|
13
|
-
{ role: 'system', content: PROMPT },
|
14
|
-
{ role: 'user', content: text[0..1000] }
|
15
|
-
]
|
16
|
-
function = {
|
17
|
-
name: 'data_extractor',
|
18
|
-
description: 'Extract structured and typed data from user messages.',
|
19
|
-
schema: {
|
20
|
-
type: "object",
|
21
|
-
properties: data_to_extract_with_explaination(data_to_extract)
|
22
|
-
}
|
23
|
-
}
|
24
|
-
|
25
|
-
::ActiveGenie::Requester.function_calling(messages, function, options)
|
26
|
-
end
|
27
|
-
|
28
|
-
def from_informal(text, data_to_extract, options = {})
|
29
|
-
messages = [
|
30
|
-
{ role: 'system', content: PROMPT },
|
31
|
-
{ role: 'user', content: text }
|
32
|
-
]
|
33
|
-
properties = data_to_extract_with_explaination(data_to_extract)
|
34
|
-
properties[:message_litote] = {
|
35
|
-
type: 'boolean',
|
36
|
-
description: 'Return true if the message is a litote. A litote is a figure of speech that uses understatement to emphasize a point by stating a negative to further affirm a positive, often incorporating double negatives for effect.'
|
37
|
-
}
|
38
|
-
properties[:litote_rephrased] = {
|
39
|
-
type: 'string',
|
40
|
-
description: 'The true meaning of the litote. Rephrase the message to a positive statement.'
|
41
|
-
}
|
42
|
-
|
43
|
-
function = {
|
44
|
-
name: 'data_extractor',
|
45
|
-
description: 'Extract structured and typed data from user messages.',
|
46
|
-
schema: { type: "object", properties: }
|
47
|
-
}
|
48
|
-
|
49
|
-
::ActiveGenie::Requester.function_calling(messages, function, options)
|
50
|
-
end
|
51
|
-
|
52
|
-
private
|
53
|
-
|
54
|
-
PROMPT = <<~PROMPT
|
55
|
-
Extract structured and typed data from user messages.
|
56
|
-
Identify relevant information within user messages and categorize it into predefined data fields with specific data types.
|
57
|
-
|
58
|
-
# Steps
|
59
|
-
1. **Identify Data Types**: Determine the types of data to collect, such as names, dates, email addresses, phone numbers, etc.
|
60
|
-
2. **Extract Information**: Use pattern recognition and language understanding to identify and extract the relevant pieces of data from the user message.
|
61
|
-
3. **Categorize Data**: Assign the extracted data to the appropriate predefined fields.
|
62
|
-
4. **Structure Data**: Format the extracted and categorized data in a structured format, such as JSON.
|
63
|
-
|
64
|
-
# Output Format
|
65
|
-
The output should be a JSON object containing fields with their corresponding extracted values. If a value is not found, the field should still be included with a null value.
|
66
|
-
|
67
|
-
# Notes
|
68
|
-
- Handle missing or partial information gracefully.
|
69
|
-
- Manage multiple occurrences of similar data points by prioritizing the first one unless specified otherwise.
|
70
|
-
- Be flexible to handle variations in data format and language clues.
|
71
|
-
PROMPT
|
72
|
-
|
73
|
-
def data_to_extract_with_explaination(data_to_extract)
|
74
|
-
with_explaination = {}
|
75
|
-
|
76
|
-
data_to_extract.each do |key, value|
|
77
|
-
with_explaination[key] = value
|
78
|
-
with_explaination["#{key}_explanation"] = {
|
79
|
-
type: 'string',
|
80
|
-
description: "The chain of thought that led to the conclusion about: #{key}. Can be blank if the user didn't provide any context",
|
81
|
-
}
|
82
|
-
end
|
83
|
-
|
84
|
-
with_explaination
|
85
|
-
end
|
86
|
-
end
|
87
|
-
end
|
88
|
-
end
|