active_genie 0.30.1 → 0.30.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +7 -5
- data/VERSION +1 -1
- data/lib/active_genie/comparator/debate.rb +13 -31
- data/lib/active_genie/comparator/fight.rb +3 -3
- data/lib/active_genie/comparator.rb +0 -2
- data/lib/active_genie/configs/base_config.rb +25 -0
- data/lib/active_genie/configs/extractor_config.rb +6 -14
- data/lib/active_genie/configs/lister_config.rb +6 -10
- data/lib/active_genie/configs/llm_config.rb +12 -25
- data/lib/active_genie/configs/log_config.rb +19 -16
- data/lib/active_genie/configs/providers/anthropic_config.rb +10 -16
- data/lib/active_genie/configs/providers/deepseek_config.rb +4 -19
- data/lib/active_genie/configs/providers/google_config.rb +4 -19
- data/lib/active_genie/configs/providers/openai_config.rb +4 -19
- data/lib/active_genie/configs/providers/provider_base.rb +21 -67
- data/lib/active_genie/configs/providers_config.rb +39 -20
- data/lib/active_genie/configs/ranker_config.rb +6 -12
- data/lib/active_genie/configuration.rb +23 -60
- data/lib/active_genie/{ranker/entities → entities}/player.rb +4 -0
- data/lib/active_genie/{ranker/entities → entities}/players.rb +1 -1
- data/lib/active_genie/entities/result.rb +29 -0
- data/lib/active_genie/errors/invalid_model_error.rb +42 -0
- data/lib/active_genie/errors/invalid_provider_error.rb +1 -1
- data/lib/active_genie/errors/provider_server_error.rb +26 -0
- data/lib/active_genie/errors/without_available_provider_error.rb +39 -0
- data/lib/active_genie/extractor/data.json +9 -0
- data/lib/active_genie/extractor/data.prompt.md +12 -0
- data/lib/active_genie/extractor/data.rb +71 -0
- data/lib/active_genie/extractor/explanation.rb +22 -41
- data/lib/active_genie/extractor/litote.rb +10 -9
- data/lib/active_genie/extractor.rb +5 -0
- data/lib/active_genie/lister/feud.json +5 -1
- data/lib/active_genie/lister/feud.rb +12 -17
- data/lib/active_genie/lister/juries.rb +18 -16
- data/lib/active_genie/logger.rb +16 -28
- data/lib/active_genie/providers/anthropic_provider.rb +12 -6
- data/lib/active_genie/providers/base_provider.rb +15 -18
- data/lib/active_genie/providers/deepseek_provider.rb +18 -10
- data/lib/active_genie/providers/google_provider.rb +11 -5
- data/lib/active_genie/providers/openai_provider.rb +8 -6
- data/lib/active_genie/providers/unified_provider.rb +58 -3
- data/lib/active_genie/ranker/elo.rb +41 -36
- data/lib/active_genie/ranker/free_for_all.rb +45 -28
- data/lib/active_genie/ranker/scoring.rb +20 -11
- data/lib/active_genie/ranker/tournament.rb +23 -35
- data/lib/active_genie/scorer/jury_bench.rb +18 -23
- data/lib/active_genie/utils/base_module.rb +34 -0
- data/lib/active_genie/utils/call_wrapper.rb +20 -0
- data/lib/active_genie/utils/deep_merge.rb +12 -0
- data/lib/active_genie/utils/fiber_by_batch.rb +2 -2
- data/lib/active_genie/utils/text_case.rb +18 -0
- data/lib/active_genie.rb +16 -18
- data/lib/tasks/benchmark.rake +1 -3
- data/lib/tasks/templates/active_genie.rb +0 -3
- data/lib/tasks/test.rake +62 -1
- metadata +25 -15
- data/lib/active_genie/configs/comparator_config.rb +0 -10
- data/lib/active_genie/configs/scorer_config.rb +0 -10
data/lib/active_genie.rb
CHANGED
|
@@ -3,10 +3,9 @@
|
|
|
3
3
|
require_relative 'active_genie/logger'
|
|
4
4
|
require_relative 'active_genie/configuration'
|
|
5
5
|
|
|
6
|
-
require_relative 'active_genie/
|
|
7
|
-
require_relative 'active_genie/
|
|
8
|
-
require_relative 'active_genie/
|
|
9
|
-
require_relative 'active_genie/configs/providers/deepseek_config'
|
|
6
|
+
require_relative 'active_genie/entities/result'
|
|
7
|
+
require_relative 'active_genie/utils/base_module'
|
|
8
|
+
require_relative 'active_genie/utils/deep_merge'
|
|
10
9
|
|
|
11
10
|
module ActiveGenie
|
|
12
11
|
autoload :Extractor, File.join(__dir__, 'active_genie/extractor')
|
|
@@ -23,27 +22,26 @@ module ActiveGenie
|
|
|
23
22
|
end
|
|
24
23
|
|
|
25
24
|
def configuration
|
|
26
|
-
@configuration ||=
|
|
25
|
+
@configuration ||= Configuration.new
|
|
27
26
|
end
|
|
28
27
|
|
|
29
|
-
def
|
|
30
|
-
return
|
|
28
|
+
def new_configuration(new_config)
|
|
29
|
+
return new_config if new_config.instance_of?(Configuration)
|
|
31
30
|
|
|
32
|
-
|
|
33
|
-
|
|
31
|
+
Configuration.new(
|
|
32
|
+
DeepMerge.call(@configuration.to_h, new_config)
|
|
33
|
+
)
|
|
34
34
|
end
|
|
35
35
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
config = Configuration.new
|
|
36
|
+
def logger
|
|
37
|
+
@logger ||= Logger.new
|
|
38
|
+
end
|
|
40
39
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
config.providers.add(Config::Providers::AnthropicConfig)
|
|
44
|
-
config.providers.add(Config::Providers::DeepseekConfig)
|
|
40
|
+
def load_tasks
|
|
41
|
+
return unless defined?(Rake)
|
|
45
42
|
|
|
46
|
-
|
|
43
|
+
Rake::Task.define_task(:environment)
|
|
44
|
+
Dir.glob(File.join(__dir__, 'tasks', '*.rake')).each { |r| load r }
|
|
47
45
|
end
|
|
48
46
|
end
|
|
49
47
|
end
|
data/lib/tasks/benchmark.rake
CHANGED
|
@@ -4,11 +4,9 @@ namespace :active_genie do
|
|
|
4
4
|
desc 'Run benchmarks, optionally for a specific module (e.g., rake active_genie:benchmark[data_extractor])'
|
|
5
5
|
task :benchmark, [:module_name] do |_, args|
|
|
6
6
|
Rake::TestTask.new(:run_benchmarks) do |t|
|
|
7
|
-
t.libs << 'benchmark'
|
|
8
|
-
|
|
9
7
|
if args[:module_name]
|
|
10
8
|
module_name = args[:module_name]
|
|
11
|
-
module_path = "benchmark
|
|
9
|
+
module_path = "test/benchmark/#{module_name}/"
|
|
12
10
|
t.test_files = FileList["#{module_path}**/*_test.rb"]
|
|
13
11
|
puts "Running benchmarks for module: #{module_name}"
|
|
14
12
|
else
|
|
@@ -5,9 +5,6 @@ ActiveGenie.configure do |config|
|
|
|
5
5
|
# config.providers.openai.api_key = ENV['OPENAI_API_KEY']
|
|
6
6
|
# config.providers.openai.organization = ENV['OPENAI_ORGANIZATION']
|
|
7
7
|
# config.providers.openai.api_url = 'https://api.openai.com/v1'
|
|
8
|
-
# config.providers.openai.lower_tier_model = 'gpt-4.1-mini'
|
|
9
|
-
# config.providers.openai.middle_tier_model = 'gpt-4.1'
|
|
10
|
-
# config.providers.openai.higher_tier_model = 'o3-mini'
|
|
11
8
|
# config.providers.openai.client = ActiveGenie::Providers::Openai::Client.new(config)
|
|
12
9
|
|
|
13
10
|
# example how add a new provider
|
data/lib/tasks/test.rake
CHANGED
|
@@ -6,10 +6,71 @@ namespace :test do
|
|
|
6
6
|
Rake::TestTask.new(:unit) do |t|
|
|
7
7
|
t.pattern = 'test/unit/**/*_test.rb'
|
|
8
8
|
end
|
|
9
|
+
desc 'Run unit tests'
|
|
10
|
+
task unit: :reset_env
|
|
9
11
|
|
|
10
12
|
Rake::TestTask.new(:integration) do |t|
|
|
11
|
-
t.pattern = 'test/integration/**/*_test.rb'
|
|
13
|
+
t.pattern = 'test/integration/**/*_test.rb'
|
|
12
14
|
end
|
|
15
|
+
desc 'Run integration tests like install rails'
|
|
16
|
+
task integration: :reset_env
|
|
13
17
|
|
|
18
|
+
desc 'Run all tests (unit and integration)'
|
|
14
19
|
task default: %i[unit integration]
|
|
20
|
+
|
|
21
|
+
desc 'Reset environment variables for API keys'
|
|
22
|
+
task :reset_env do
|
|
23
|
+
reset_environment_variables
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
desc 'Run all tests (unit and integration) or a specific test file'
|
|
28
|
+
task :test, [:file_path] do |_t, args|
|
|
29
|
+
reset_environment_variables
|
|
30
|
+
|
|
31
|
+
if args[:file_path]
|
|
32
|
+
file_path, line_number = args[:file_path].split(':')
|
|
33
|
+
|
|
34
|
+
if line_number
|
|
35
|
+
test_name = find_test_name_at_line(file_path, line_number.to_i)
|
|
36
|
+
|
|
37
|
+
if test_name
|
|
38
|
+
sh "ruby -Itest #{file_path} --name #{test_name}"
|
|
39
|
+
else
|
|
40
|
+
puts "Warning: Could not find test method at line #{line_number}"
|
|
41
|
+
ruby "-Itest #{file_path}"
|
|
42
|
+
end
|
|
43
|
+
else
|
|
44
|
+
ruby "-Itest #{file_path}"
|
|
45
|
+
end
|
|
46
|
+
else
|
|
47
|
+
Rake::Task['test:unit'].invoke
|
|
48
|
+
Rake::Task['test:integration'].invoke
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def find_test_name_at_line(file_path, line_number)
|
|
53
|
+
return nil unless File.exist?(file_path)
|
|
54
|
+
|
|
55
|
+
lines = File.readlines(file_path)
|
|
56
|
+
return nil if line_number > lines.length
|
|
57
|
+
|
|
58
|
+
(line_number - 1).downto(0) do |i|
|
|
59
|
+
line = lines[i]
|
|
60
|
+
if line =~ /^\s*def\s+(test_\w+)/
|
|
61
|
+
return Regexp.last_match(1)
|
|
62
|
+
elsif line =~ /^\s*test\s+['"](.+?)['"]\s+do/
|
|
63
|
+
test_desc = Regexp.last_match(1)
|
|
64
|
+
return "/#{test_desc}/"
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
nil
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def reset_environment_variables
|
|
72
|
+
ENV['OPENAI_API_KEY'] = nil
|
|
73
|
+
ENV['GEMINI_API_KEY'] = nil
|
|
74
|
+
ENV['ANTHROPIC_API_KEY'] = nil
|
|
75
|
+
ENV['DEEPSEEK_API_KEY'] = nil
|
|
15
76
|
end
|
metadata
CHANGED
|
@@ -1,32 +1,34 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: active_genie
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.30.
|
|
4
|
+
version: 0.30.8
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Radamés Roriz
|
|
8
|
-
autorequire:
|
|
9
8
|
bindir: bin
|
|
10
9
|
cert_chain: []
|
|
11
|
-
date:
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
12
11
|
dependencies:
|
|
13
12
|
- !ruby/object:Gem::Dependency
|
|
14
13
|
name: async
|
|
15
14
|
requirement: !ruby/object:Gem::Requirement
|
|
16
15
|
requirements:
|
|
17
|
-
- - "
|
|
16
|
+
- - "~>"
|
|
18
17
|
- !ruby/object:Gem::Version
|
|
19
18
|
version: '2.0'
|
|
20
19
|
type: :runtime
|
|
21
20
|
prerelease: false
|
|
22
21
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
22
|
requirements:
|
|
24
|
-
- - "
|
|
23
|
+
- - "~>"
|
|
25
24
|
- !ruby/object:Gem::Version
|
|
26
25
|
version: '2.0'
|
|
27
26
|
description: |
|
|
28
|
-
ActiveGenie is
|
|
29
|
-
|
|
27
|
+
ActiveGenie is an enabler for creating reliable GenAI features, offering powerful, model-agnostic tools across any provider. It allows you to settle subjective comparisons with a `ActibeGenie::Comparator` module that stages a political debate, get accurate scores from an AI jury using `ActiveGenie::Scorer`, and rank large datasets using `ActiveGenie::Ranker`'s tournament-style system.
|
|
28
|
+
This reliability is built on three core pillars:
|
|
29
|
+
- Custom Benchmarking: Testing for consistency with every new version and model update.
|
|
30
|
+
- Reasoning Prompting: Utilizing human reasoning techniques (like debate and jury review) to control a model's reasoning.
|
|
31
|
+
- Overfitting Prompts: Highly specialized, and potentially model-specific, prompt for each module's purpose.
|
|
30
32
|
email:
|
|
31
33
|
- radames@roriz.dev
|
|
32
34
|
executables: []
|
|
@@ -44,7 +46,7 @@ files:
|
|
|
44
46
|
- lib/active_genie/comparator/fight.json
|
|
45
47
|
- lib/active_genie/comparator/fight.prompt.md
|
|
46
48
|
- lib/active_genie/comparator/fight.rb
|
|
47
|
-
- lib/active_genie/configs/
|
|
49
|
+
- lib/active_genie/configs/base_config.rb
|
|
48
50
|
- lib/active_genie/configs/extractor_config.rb
|
|
49
51
|
- lib/active_genie/configs/lister_config.rb
|
|
50
52
|
- lib/active_genie/configs/llm_config.rb
|
|
@@ -56,11 +58,19 @@ files:
|
|
|
56
58
|
- lib/active_genie/configs/providers/provider_base.rb
|
|
57
59
|
- lib/active_genie/configs/providers_config.rb
|
|
58
60
|
- lib/active_genie/configs/ranker_config.rb
|
|
59
|
-
- lib/active_genie/configs/scorer_config.rb
|
|
60
61
|
- lib/active_genie/configuration.rb
|
|
62
|
+
- lib/active_genie/entities/player.rb
|
|
63
|
+
- lib/active_genie/entities/players.rb
|
|
64
|
+
- lib/active_genie/entities/result.rb
|
|
61
65
|
- lib/active_genie/errors/invalid_log_output_error.rb
|
|
66
|
+
- lib/active_genie/errors/invalid_model_error.rb
|
|
62
67
|
- lib/active_genie/errors/invalid_provider_error.rb
|
|
68
|
+
- lib/active_genie/errors/provider_server_error.rb
|
|
69
|
+
- lib/active_genie/errors/without_available_provider_error.rb
|
|
63
70
|
- lib/active_genie/extractor.rb
|
|
71
|
+
- lib/active_genie/extractor/data.json
|
|
72
|
+
- lib/active_genie/extractor/data.prompt.md
|
|
73
|
+
- lib/active_genie/extractor/data.rb
|
|
64
74
|
- lib/active_genie/extractor/explanation.json
|
|
65
75
|
- lib/active_genie/extractor/explanation.prompt.md
|
|
66
76
|
- lib/active_genie/extractor/explanation.rb
|
|
@@ -81,15 +91,17 @@ files:
|
|
|
81
91
|
- lib/active_genie/providers/unified_provider.rb
|
|
82
92
|
- lib/active_genie/ranker.rb
|
|
83
93
|
- lib/active_genie/ranker/elo.rb
|
|
84
|
-
- lib/active_genie/ranker/entities/player.rb
|
|
85
|
-
- lib/active_genie/ranker/entities/players.rb
|
|
86
94
|
- lib/active_genie/ranker/free_for_all.rb
|
|
87
95
|
- lib/active_genie/ranker/scoring.rb
|
|
88
96
|
- lib/active_genie/ranker/tournament.rb
|
|
89
97
|
- lib/active_genie/scorer.rb
|
|
90
98
|
- lib/active_genie/scorer/jury_bench.prompt.md
|
|
91
99
|
- lib/active_genie/scorer/jury_bench.rb
|
|
100
|
+
- lib/active_genie/utils/base_module.rb
|
|
101
|
+
- lib/active_genie/utils/call_wrapper.rb
|
|
102
|
+
- lib/active_genie/utils/deep_merge.rb
|
|
92
103
|
- lib/active_genie/utils/fiber_by_batch.rb
|
|
104
|
+
- lib/active_genie/utils/text_case.rb
|
|
93
105
|
- lib/tasks/benchmark.rake
|
|
94
106
|
- lib/tasks/install.rake
|
|
95
107
|
- lib/tasks/templates/active_genie.rb
|
|
@@ -103,7 +115,6 @@ metadata:
|
|
|
103
115
|
changelog_uri: https://github.com/Roriz/active_genie/blob/master/CHANGELOG.md
|
|
104
116
|
bug_tracker_uri: https://github.com/Roriz/active_genie/issues
|
|
105
117
|
rubygems_mfa_required: 'true'
|
|
106
|
-
post_install_message:
|
|
107
118
|
rdoc_options: []
|
|
108
119
|
require_paths:
|
|
109
120
|
- lib
|
|
@@ -118,8 +129,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
118
129
|
- !ruby/object:Gem::Version
|
|
119
130
|
version: '0'
|
|
120
131
|
requirements: []
|
|
121
|
-
rubygems_version: 3.
|
|
122
|
-
signing_key:
|
|
132
|
+
rubygems_version: 3.6.9
|
|
123
133
|
specification_version: 4
|
|
124
|
-
summary: 'The Lodash for GenAI:
|
|
134
|
+
summary: 'The Lodash for GenAI: Consistent + Model-Agnostic'
|
|
125
135
|
test_files: []
|