activeagent 1.9.0 → 1.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +219 -1
- data/lib/active_agent/base.rb +27 -2
- data/lib/active_agent/delegation/schema.rb +51 -13
- data/lib/active_agent/evals/comparison.rb +169 -0
- data/lib/active_agent/evals/report.rb +73 -7
- data/lib/active_agent/evals/report_html.rb +64 -2
- data/lib/active_agent/evals/runner.rb +7 -1
- data/lib/active_agent/evals/statistics.rb +26 -0
- data/lib/active_agent/evals.rb +2 -0
- data/lib/active_agent/providers/anthropic_provider.rb +3 -1
- data/lib/active_agent/providers/open_ai/chat_provider.rb +7 -0
- data/lib/active_agent/providers/open_ai/responses_provider.rb +37 -4
- data/lib/active_agent/schema_generator.rb +146 -22
- data/lib/active_agent/version.rb +1 -1
- metadata +4 -2
|
@@ -34,6 +34,21 @@ module ActiveAgent
|
|
|
34
34
|
#
|
|
35
35
|
# ContactForm.to_json_schema
|
|
36
36
|
#
|
|
37
|
+
# @example ActiveModel with a nested array of objects
|
|
38
|
+
# class Person
|
|
39
|
+
# include ActiveModel::Model
|
|
40
|
+
# include ActiveModel::Attributes
|
|
41
|
+
# include ActiveAgent::SchemaGenerator
|
|
42
|
+
#
|
|
43
|
+
# attribute :name, :string
|
|
44
|
+
# has_many :addresses, class_name: "Address"
|
|
45
|
+
# end
|
|
46
|
+
#
|
|
47
|
+
# Person.to_json_schema
|
|
48
|
+
# # => { name: "person", schema: { type: "object", properties: {
|
|
49
|
+
# # name: {...}, addresses: { type: "array", items: { "$ref": "#/$defs/address" } } },
|
|
50
|
+
# # ..., "$defs": { address: { type: "object", ... } } } }
|
|
51
|
+
#
|
|
37
52
|
# @see Builder
|
|
38
53
|
module SchemaGenerator
|
|
39
54
|
extend ActiveSupport::Concern
|
|
@@ -47,6 +62,13 @@ module ActiveAgent
|
|
|
47
62
|
# Fallback for any class that includes this module
|
|
48
63
|
extend ActiveModelClassMethods
|
|
49
64
|
end
|
|
65
|
+
|
|
66
|
+
# The railtie includes this module into ActiveRecord::Base too, whose
|
|
67
|
+
# has_many must stay ActiveRecord's.
|
|
68
|
+
unless respond_to?(:has_many)
|
|
69
|
+
extend ActiveModelAssociations
|
|
70
|
+
class_attribute :schema_associations, instance_accessor: false, default: {}.freeze
|
|
71
|
+
end
|
|
50
72
|
end
|
|
51
73
|
|
|
52
74
|
# Class methods added to ActiveRecord models.
|
|
@@ -106,6 +128,40 @@ module ActiveAgent
|
|
|
106
128
|
end
|
|
107
129
|
end
|
|
108
130
|
|
|
131
|
+
# Nested schemas for ActiveModel classes, which have no associations of
|
|
132
|
+
# their own. Added only to a class without a +has_many+ already.
|
|
133
|
+
module ActiveModelAssociations
|
|
134
|
+
# Declares an array of objects described by another schema class.
|
|
135
|
+
#
|
|
136
|
+
# {ActiveModelClassMethods#to_json_schema} emits the attribute as an
|
|
137
|
+
# array whose items are a +$ref+ to that class's schema, which goes under
|
|
138
|
+
# the root +$defs+ keyed by the singular name in lower camel case
|
|
139
|
+
# (+addresses+ becomes +address+, +line_items+ becomes +lineItem+).
|
|
140
|
+
# The prompt preserves both property and definition names.
|
|
141
|
+
#
|
|
142
|
+
# Also declares the attribute, defaulting to an empty array, so the
|
|
143
|
+
# model accepts the data the schema describes.
|
|
144
|
+
#
|
|
145
|
+
# @param name [Symbol] the attribute holding the array
|
|
146
|
+
# @param class_name [String] full constant name of the item class, which
|
|
147
|
+
# generates JSON Schema too; resolved when the schema is generated
|
|
148
|
+
# @return [void]
|
|
149
|
+
# @raise [ArgumentError] when the class does not include
|
|
150
|
+
# +ActiveModel::Attributes+, which declares the attribute
|
|
151
|
+
#
|
|
152
|
+
# @example
|
|
153
|
+
# has_many :addresses, class_name: "Address"
|
|
154
|
+
def has_many(name, class_name:)
|
|
155
|
+
unless respond_to?(:attribute)
|
|
156
|
+
raise ArgumentError, "#{self.name || inspect}.has_many :#{name} needs ActiveModel::Attributes to declare the attribute; include it before calling has_many"
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
self.schema_associations = schema_associations.merge(name.to_sym => class_name).freeze
|
|
160
|
+
|
|
161
|
+
attribute name, default: -> { [] }
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
|
|
109
165
|
# Internal builder class for constructing JSON Schemas from model classes.
|
|
110
166
|
#
|
|
111
167
|
# This class handles the actual schema generation logic, supporting both
|
|
@@ -122,24 +178,7 @@ module ActiveAgent
|
|
|
122
178
|
#
|
|
123
179
|
# @raise [ArgumentError] If model_class is not an ActiveRecord or ActiveModel class
|
|
124
180
|
def self.json_schema_from_model(model_class, options = {})
|
|
125
|
-
schema =
|
|
126
|
-
type: "object",
|
|
127
|
-
properties: {},
|
|
128
|
-
required: [],
|
|
129
|
-
additional_properties: options.fetch(:additional_properties, false)
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
if defined?(ActiveRecord::Base) && model_class < ActiveRecord::Base
|
|
133
|
-
schema = build_activerecord_schema(schema, model_class, options)
|
|
134
|
-
elsif defined?(ActiveModel::Model) && model_class.include?(ActiveModel::Model)
|
|
135
|
-
schema = build_activemodel_schema(schema, model_class, options)
|
|
136
|
-
else
|
|
137
|
-
raise ArgumentError, "Model must be an ActiveRecord or ActiveModel class"
|
|
138
|
-
end
|
|
139
|
-
|
|
140
|
-
# OpenAI strict mode requires all properties to be in the required array
|
|
141
|
-
# So we add all properties to required if strict mode is enabled
|
|
142
|
-
schema[:required] = schema[:properties].keys.sort if options[:strict]
|
|
181
|
+
schema = object_schema(model_class, options)
|
|
143
182
|
|
|
144
183
|
{
|
|
145
184
|
name: options[:name] || model_class.name.underscore,
|
|
@@ -151,6 +190,37 @@ module ActiveAgent
|
|
|
151
190
|
class << self
|
|
152
191
|
private
|
|
153
192
|
|
|
193
|
+
# Builds the object schema of a model class, without the name and
|
|
194
|
+
# strict flag {json_schema_from_model} wraps it in.
|
|
195
|
+
#
|
|
196
|
+
# @param model_class [Class] The ActiveRecord or ActiveModel class
|
|
197
|
+
# @param options [Hash] Schema generation options
|
|
198
|
+
# @param ancestors [Array<Class>] the classes nesting this one, outermost first
|
|
199
|
+
#
|
|
200
|
+
# @return [Hash] The object schema
|
|
201
|
+
def object_schema(model_class, options, ancestors = [])
|
|
202
|
+
schema = {
|
|
203
|
+
type: "object",
|
|
204
|
+
properties: {},
|
|
205
|
+
required: [],
|
|
206
|
+
additional_properties: options.fetch(:additional_properties, false)
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
if defined?(ActiveRecord::Base) && model_class < ActiveRecord::Base
|
|
210
|
+
schema = build_activerecord_schema(schema, model_class, options)
|
|
211
|
+
elsif defined?(ActiveModel::Model) && model_class.include?(ActiveModel::Model)
|
|
212
|
+
schema = build_activemodel_schema(schema, model_class, options, ancestors)
|
|
213
|
+
else
|
|
214
|
+
raise ArgumentError, "Model must be an ActiveRecord or ActiveModel class"
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
# OpenAI strict mode requires all properties to be in the required array
|
|
218
|
+
# So we add all properties to required if strict mode is enabled
|
|
219
|
+
schema[:required] = schema[:properties].keys.sort if options[:strict]
|
|
220
|
+
|
|
221
|
+
schema
|
|
222
|
+
end
|
|
223
|
+
|
|
154
224
|
# Builds a JSON Schema from an ActiveRecord model.
|
|
155
225
|
#
|
|
156
226
|
# Extracts column information, associations, and validations to build
|
|
@@ -187,14 +257,16 @@ module ActiveAgent
|
|
|
187
257
|
|
|
188
258
|
# Builds a JSON Schema from an ActiveModel class.
|
|
189
259
|
#
|
|
190
|
-
# Extracts attribute types and validations to build a
|
|
260
|
+
# Extracts attribute types, nested schemas and validations to build a
|
|
261
|
+
# schema representation.
|
|
191
262
|
#
|
|
192
263
|
# @param schema [Hash] The schema hash to populate
|
|
193
264
|
# @param model_class [Class] The ActiveModel class
|
|
194
265
|
# @param options [Hash] Schema generation options
|
|
266
|
+
# @param ancestors [Array<Class>] the classes nesting this one, outermost first
|
|
195
267
|
#
|
|
196
268
|
# @return [void]
|
|
197
|
-
def build_activemodel_schema(schema, model_class, options)
|
|
269
|
+
def build_activemodel_schema(schema, model_class, options, ancestors)
|
|
198
270
|
if model_class.respond_to?(:attribute_types)
|
|
199
271
|
model_class.attribute_types.each do |name, type|
|
|
200
272
|
next if options[:exclude]&.include?(name.to_sym)
|
|
@@ -206,6 +278,10 @@ module ActiveAgent
|
|
|
206
278
|
raise ArgumentError, "#{model_class.name} does not define any attributes. Use `attribute :name, :type` to define attributes."
|
|
207
279
|
end
|
|
208
280
|
|
|
281
|
+
if model_class.respond_to?(:schema_associations)
|
|
282
|
+
schema = add_activemodel_associations_to_schema(schema, model_class, options, ancestors)
|
|
283
|
+
end
|
|
284
|
+
|
|
209
285
|
if model_class.respond_to?(:validators)
|
|
210
286
|
schema = add_validations_to_schema(schema, model_class, options)
|
|
211
287
|
end
|
|
@@ -356,6 +432,51 @@ module ActiveAgent
|
|
|
356
432
|
schema
|
|
357
433
|
end
|
|
358
434
|
|
|
435
|
+
# Adds the arrays an ActiveModel class declares with
|
|
436
|
+
# {ActiveModelAssociations#has_many}, each item a +$ref+ to the nested
|
|
437
|
+
# class's schema under +$defs+.
|
|
438
|
+
#
|
|
439
|
+
# A +$ref+ resolves against the root of the schema, so the definitions
|
|
440
|
+
# a nested class collects move up into this schema's +$defs+.
|
|
441
|
+
#
|
|
442
|
+
# @param schema [Hash] The schema hash to populate
|
|
443
|
+
# @param model_class [Class] The ActiveModel class
|
|
444
|
+
# @param options [Hash] Schema generation options
|
|
445
|
+
# @param ancestors [Array<Class>] the classes nesting this one, outermost first
|
|
446
|
+
#
|
|
447
|
+
# @return [Hash]
|
|
448
|
+
# @raise [ArgumentError] when a class nests itself, or two different
|
|
449
|
+
# schemas would share a definition name
|
|
450
|
+
def add_activemodel_associations_to_schema(schema, model_class, options, ancestors)
|
|
451
|
+
path = ancestors + [ model_class ]
|
|
452
|
+
definitions = {}
|
|
453
|
+
|
|
454
|
+
model_class.schema_associations.each do |name, class_name|
|
|
455
|
+
next if options[:exclude]&.include?(name)
|
|
456
|
+
|
|
457
|
+
nested_class = class_name.constantize
|
|
458
|
+
if path.include?(nested_class)
|
|
459
|
+
raise ArgumentError, "#{model_class.name}.has_many :#{name} is recursive (#{(path + [ nested_class ]).map(&:name).join(' > ')}); recursive schemas are not supported"
|
|
460
|
+
end
|
|
461
|
+
|
|
462
|
+
key = name.to_s.singularize.camelize(:lower).to_sym
|
|
463
|
+
nested = object_schema(nested_class, options.slice(:strict, :additional_properties), path)
|
|
464
|
+
|
|
465
|
+
[ [ key, nested ], *nested.delete(:$defs) ].each do |definition_key, definition|
|
|
466
|
+
if definitions.key?(definition_key) && definitions[definition_key] != definition
|
|
467
|
+
raise ArgumentError, "#{model_class.name} nests two different schemas under the definition #{definition_key.inspect}"
|
|
468
|
+
end
|
|
469
|
+
|
|
470
|
+
definitions[definition_key] = definition
|
|
471
|
+
end
|
|
472
|
+
|
|
473
|
+
schema[:properties][name] = { type: "array", items: { "$ref": "#/$defs/#{key}" } }
|
|
474
|
+
end
|
|
475
|
+
|
|
476
|
+
schema[:$defs] = definitions if definitions.any?
|
|
477
|
+
schema
|
|
478
|
+
end
|
|
479
|
+
|
|
359
480
|
# Adds validation constraints to the schema.
|
|
360
481
|
#
|
|
361
482
|
# Translates ActiveModel validations (presence, length, numericality,
|
|
@@ -375,11 +496,14 @@ module ActiveAgent
|
|
|
375
496
|
when ActiveModel::Validations::PresenceValidator
|
|
376
497
|
schema[:required] << attribute.to_sym unless schema[:required].include?(attribute.to_sym)
|
|
377
498
|
when ActiveModel::Validations::LengthValidator
|
|
499
|
+
# An array, such as a has_many, is limited by its item count.
|
|
500
|
+
min_key, max_key = schema[:properties][attribute.to_sym][:type] == "array" ? [ :min_items, :max_items ] : [ :min_length, :max_length ]
|
|
501
|
+
|
|
378
502
|
if validator.options[:minimum]
|
|
379
|
-
schema[:properties][attribute.to_sym][
|
|
503
|
+
schema[:properties][attribute.to_sym][min_key] = validator.options[:minimum]
|
|
380
504
|
end
|
|
381
505
|
if validator.options[:maximum]
|
|
382
|
-
schema[:properties][attribute.to_sym][
|
|
506
|
+
schema[:properties][attribute.to_sym][max_key] = validator.options[:maximum]
|
|
383
507
|
end
|
|
384
508
|
when ActiveModel::Validations::NumericalityValidator
|
|
385
509
|
if validator.options[:greater_than]
|
data/lib/active_agent/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: activeagent
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.9.
|
|
4
|
+
version: 1.9.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Justin Bowen
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-10-
|
|
11
|
+
date: 2026-10-20 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: actionpack
|
|
@@ -456,6 +456,7 @@ files:
|
|
|
456
456
|
- lib/active_agent/deprecator.rb
|
|
457
457
|
- lib/active_agent/evals.rb
|
|
458
458
|
- lib/active_agent/evals/catalog.rb
|
|
459
|
+
- lib/active_agent/evals/comparison.rb
|
|
459
460
|
- lib/active_agent/evals/correlation.rb
|
|
460
461
|
- lib/active_agent/evals/design_tokens.rb
|
|
461
462
|
- lib/active_agent/evals/diagnosis.rb
|
|
@@ -472,6 +473,7 @@ files:
|
|
|
472
473
|
- lib/active_agent/evals/scenario.rb
|
|
473
474
|
- lib/active_agent/evals/scenario_parser.rb
|
|
474
475
|
- lib/active_agent/evals/scorer.rb
|
|
476
|
+
- lib/active_agent/evals/statistics.rb
|
|
475
477
|
- lib/active_agent/evals/suite.rb
|
|
476
478
|
- lib/active_agent/generation.rb
|
|
477
479
|
- lib/active_agent/generation_job.rb
|