activeagent 1.9.2 → 1.9.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +252 -0
- data/README.md +3 -0
- data/lib/active_agent/base.rb +2 -2
- data/lib/active_agent/concerns/preview.rb +2 -2
- data/lib/active_agent/evals/design_tokens.rb +12 -1
- data/lib/active_agent/evals/publisher.rb +19 -12
- data/lib/active_agent/evals/report.rb +12 -62
- data/lib/active_agent/evals/report_html.rb +85 -10
- data/lib/active_agent/evals/report_markdown.rb +202 -0
- data/lib/active_agent/evals.rb +1 -0
- data/lib/active_agent/inline_preview_interceptor.rb +1 -0
- data/lib/active_agent/preview/templates/layouts/active_agent/preview.html.erb +17 -0
- data/lib/active_agent/preview/templates/rails/agents/index.html.erb +14 -0
- data/lib/active_agent/preview/templates/rails/agents/preview.html.erb +3 -0
- data/lib/active_agent/providers/_base_provider.rb +2 -0
- data/lib/active_agent/providers/anthropic_provider.rb +1 -0
- data/lib/active_agent/providers/common/responses/base.rb +15 -2
- data/lib/active_agent/providers/common/usage.rb +4 -2
- data/lib/active_agent/providers/open_ai/chat_provider.rb +39 -0
- data/lib/active_agent/providers/open_ai/responses_provider.rb +3 -0
- data/lib/active_agent/providers/opper/_types.rb +16 -0
- data/lib/active_agent/providers/opper/options.rb +38 -0
- data/lib/active_agent/providers/opper_provider.rb +72 -0
- data/lib/active_agent/railtie.rb +11 -5
- data/lib/active_agent/version.rb +1 -1
- data/lib/generators/test_unit/agent_generator.rb +1 -1
- data/lib/generators/test_unit/templates/functional_test.rb.tt +1 -1
- data/lib/generators/test_unit/templates/preview.rb.tt +3 -3
- data/lib/rails/agents_controller.rb +43 -0
- metadata +10 -2
|
@@ -5,10 +5,15 @@ require "cgi"
|
|
|
5
5
|
module ActiveAgent
|
|
6
6
|
module Evals
|
|
7
7
|
# Report#to_html: the run as one self-contained page on the dashboard's
|
|
8
|
-
# design system (DesignTokens) — inline styles, no
|
|
9
|
-
#
|
|
10
|
-
#
|
|
11
|
-
#
|
|
8
|
+
# design system (DesignTokens) — inline styles, no external assets — so a
|
|
9
|
+
# run's outcome can be archived next to a CI run or handed to a teammate,
|
|
10
|
+
# and so the dashboard can serve the same page for a persisted run.
|
|
11
|
+
#
|
|
12
|
+
# Its one inline script (COPY_SCRIPT) backs the "Copy as Markdown"
|
|
13
|
+
# buttons: the whole report, the verdict, What to fix (for the model the
|
|
14
|
+
# filter shows), each fix card and each scenario. The Markdown each copies
|
|
15
|
+
# is ReportMarkdown's, embedded in a hidden textarea, so the page copies
|
|
16
|
+
# exactly what the dashboard and the CLI write.
|
|
12
17
|
#
|
|
13
18
|
# Same content as Report#to_markdown, laid out the way the dashboard's
|
|
14
19
|
# suite card is: header and stat tiles, the MODELS panel with the judge's
|
|
@@ -50,6 +55,8 @@ module ActiveAgent
|
|
|
50
55
|
#{html_footer}
|
|
51
56
|
</section>
|
|
52
57
|
</main>
|
|
58
|
+
#{markdown_store("md-report", to_markdown)}
|
|
59
|
+
<script>#{COPY_SCRIPT}</script>
|
|
53
60
|
</body>
|
|
54
61
|
</html>
|
|
55
62
|
HTML
|
|
@@ -237,7 +244,7 @@ module ActiveAgent
|
|
|
237
244
|
|
|
238
245
|
<<~HEADER
|
|
239
246
|
<header>
|
|
240
|
-
<h1>#{h(title)}</h1>
|
|
247
|
+
<div class="title-row"><h1>#{h(title)}</h1>#{copy_button('md-report', label: 'Copy as Markdown', title: 'Copy the whole report as GitHub-flavored Markdown')}</div>
|
|
241
248
|
<div class="chips">#{chips.join}</div>
|
|
242
249
|
</header>
|
|
243
250
|
HEADER
|
|
@@ -320,7 +327,8 @@ module ActiveAgent
|
|
|
320
327
|
|
|
321
328
|
<<~PANEL
|
|
322
329
|
<div class="panel">
|
|
323
|
-
<div class="panel-head"><span class="micro">Models</span><span class="right">judged by #{h(judged_by)}</span
|
|
330
|
+
<div class="panel-head"><span class="micro">Models</span><span class="right">judged by #{h(judged_by)}</span>#{copy_button('md-verdict', title: 'Copy the verdict and the model comparison as Markdown')}</div>
|
|
331
|
+
#{markdown_store('md-verdict', verdict_markdown)}
|
|
324
332
|
#{html_comparison_table if comparing?}
|
|
325
333
|
#{html_intervals}
|
|
326
334
|
#{blocks.join}
|
|
@@ -612,12 +620,13 @@ module ActiveAgent
|
|
|
612
620
|
expects = scenario.expected_tools.any? ? %(<span class="exp">expects <b>#{h(scenario.expected_tools.join(' or '))}</b></span>) : ""
|
|
613
621
|
labels = @models.map(&:label)
|
|
614
622
|
cards = cohort.sort_by { |result| labels.index(result.label) || labels.size }.map { |result| html_result_card(result) }
|
|
623
|
+
md_id = "md-#{anchor(scenario)}"
|
|
615
624
|
|
|
616
625
|
<<~BLOCK
|
|
617
626
|
<div id="#{h(anchor(scenario))}">
|
|
618
627
|
<details>
|
|
619
628
|
<summary><span class="chev">></span><span class="key">#{h(scenario.key)}</span><span class="prompt">#{h(scenario.prompt)}</span>#{expects}</summary>
|
|
620
|
-
<div class="drill">#{cards.join}</div>
|
|
629
|
+
<div class="drill">#{cards.join}<div class="drill-copy">#{copy_button(md_id, title: "Copy #{scenario.key} as Markdown")}#{markdown_store(md_id, scenario_markdown(scenario.key, answer_limit: ANSWER_LIMIT))}</div></div>
|
|
621
630
|
</details>
|
|
622
631
|
</div>
|
|
623
632
|
BLOCK
|
|
@@ -670,14 +679,16 @@ module ActiveAgent
|
|
|
670
679
|
"#{plural(faulted.map { |result| result.scenario.key }.uniq.size, 'scenario')}"
|
|
671
680
|
body =
|
|
672
681
|
if items.any?
|
|
673
|
-
%(<div class="fixes">#{items.map { |item| html_fix_card(item) }.join}</div>)
|
|
682
|
+
%(<div class="fixes">#{items.each_with_index.map { |item, index| html_fix_card(item, index) }.join}</div>)
|
|
674
683
|
else
|
|
675
684
|
%(<div class="nothing">[+] nothing to fix</div>)
|
|
676
685
|
end
|
|
677
686
|
|
|
678
687
|
<<~FIXES
|
|
679
688
|
<section class="section fix-section" aria-label="Recommendations">
|
|
680
|
-
<div class="section-head"><span class="micro">What to fix</span><span class="meta">#{h(meta)}</span
|
|
689
|
+
<div class="section-head"><span class="micro">What to fix</span><span class="meta">#{h(meta)}</span>#{copy_button('md-fixes', model_filtered: comparing? && items.any?, title: 'Copy What to fix as Markdown, for the model the filter shows')}</div>
|
|
690
|
+
#{markdown_store('md-fixes', fix_items_markdown)}
|
|
691
|
+
#{@models.each_index.map { |index| markdown_store("md-fixes-m#{index}", fix_items_markdown(model: @models[index].label)) }.join if comparing? && items.any?}
|
|
681
692
|
#{html_fix_filter(items) if comparing? && items.any?}
|
|
682
693
|
#{body}
|
|
683
694
|
</section>
|
|
@@ -706,7 +717,7 @@ module ActiveAgent
|
|
|
706
717
|
labels.filter_map { |label| (index = @models.index(model_by_label(label))) && "m#{index}" }.join(" ")
|
|
707
718
|
end
|
|
708
719
|
|
|
709
|
-
def html_fix_card(item)
|
|
720
|
+
def html_fix_card(item, index)
|
|
710
721
|
tone = item["kind"] == "instruction" ? "info" : "error"
|
|
711
722
|
glyph = tone == "info" ? "[i]" : "[!]"
|
|
712
723
|
title = fault_name(item["fault"]) + (item["count"].to_i > 1 ? " ×#{item['count']}" : "")
|
|
@@ -719,6 +730,7 @@ module ActiveAgent
|
|
|
719
730
|
parts << html_fix_server(item["server"]) if item["server"]
|
|
720
731
|
parts << %(<div class="note">#{h(item['note'])}</div>) if item["note"].present?
|
|
721
732
|
parts << html_fix_action(item["action"]) if item["action"]
|
|
733
|
+
parts << %(<div class="fix-copy">#{copy_button("md-fix-#{index}", title: 'Copy this fix as Markdown')}#{markdown_store("md-fix-#{index}", fix_item_markdown(item))}</div>)
|
|
722
734
|
models = fix_model_tokens(item)
|
|
723
735
|
%(<div class="fix"#{%( data-models="#{models}") if models.present?}>#{parts.join}</div>)
|
|
724
736
|
end
|
|
@@ -771,6 +783,19 @@ module ActiveAgent
|
|
|
771
783
|
"#{scenarios} · #{models}"
|
|
772
784
|
end
|
|
773
785
|
|
|
786
|
+
# --- Copy as Markdown -------------------------------------------------
|
|
787
|
+
|
|
788
|
+
# A button COPY_SCRIPT answers by copying the textarea +id+ names; with
|
|
789
|
+
# +model_filtered+, the one for the model the fix filter has checked.
|
|
790
|
+
def copy_button(id, label: "Copy as Markdown", title: nil, model_filtered: false)
|
|
791
|
+
%(<button type="button" class="copy-md" data-md="#{h(id)}"#{' data-md-model' if model_filtered}) +
|
|
792
|
+
%(#{%( title="#{h(title)}") if title}>#{h(label)}</button>)
|
|
793
|
+
end
|
|
794
|
+
|
|
795
|
+
def markdown_store(id, markdown)
|
|
796
|
+
%(<textarea class="md-store" id="#{h(id)}" hidden readonly>#{h(markdown)}</textarea>)
|
|
797
|
+
end
|
|
798
|
+
|
|
774
799
|
# --- footer ----------------------------------------------------------
|
|
775
800
|
|
|
776
801
|
# The run's terms: the judge, the criteria and the pass mark, what it
|
|
@@ -804,6 +829,48 @@ module ActiveAgent
|
|
|
804
829
|
|
|
805
830
|
# Colors only through the token variables; radii 4 badges · 6 chips ·
|
|
806
831
|
# 8 controls · 10 nested panels · 12 cards · 999 bars; no shadows.
|
|
832
|
+
# Copies the Markdown a button names: the async Clipboard API where the
|
|
833
|
+
# page may use it, else a selected textarea and the copy command, which
|
|
834
|
+
# still works on an http:// host and in the dashboard's frame.
|
|
835
|
+
COPY_SCRIPT = <<~JS.gsub(/\n\s*/, "").freeze
|
|
836
|
+
document.addEventListener("click", function (event) {
|
|
837
|
+
var button = event.target.closest && event.target.closest("button.copy-md");
|
|
838
|
+
if (!button) return;
|
|
839
|
+
event.preventDefault();
|
|
840
|
+
var id = button.getAttribute("data-md");
|
|
841
|
+
if (button.hasAttribute("data-md-model")) {
|
|
842
|
+
var picked = document.querySelector('input[name="fix-model"]:checked');
|
|
843
|
+
if (picked && picked.value !== "all") id += "-" + picked.value;
|
|
844
|
+
}
|
|
845
|
+
var store = document.getElementById(id);
|
|
846
|
+
if (!store) return;
|
|
847
|
+
var text = store.value;
|
|
848
|
+
var label = button.getAttribute("data-label") || button.textContent;
|
|
849
|
+
button.setAttribute("data-label", label);
|
|
850
|
+
var done = function (ok) {
|
|
851
|
+
button.textContent = ok ? "Copied" : "Copy failed";
|
|
852
|
+
setTimeout(function () { button.textContent = label; }, 1500);
|
|
853
|
+
};
|
|
854
|
+
var legacy = function () {
|
|
855
|
+
var area = document.createElement("textarea");
|
|
856
|
+
area.value = text;
|
|
857
|
+
area.style.position = "fixed";
|
|
858
|
+
area.style.opacity = "0";
|
|
859
|
+
document.body.appendChild(area);
|
|
860
|
+
area.select();
|
|
861
|
+
var ok = false;
|
|
862
|
+
try { ok = document.execCommand("copy"); } catch (error) {}
|
|
863
|
+
document.body.removeChild(area);
|
|
864
|
+
done(ok);
|
|
865
|
+
};
|
|
866
|
+
if (navigator.clipboard && navigator.clipboard.writeText) {
|
|
867
|
+
navigator.clipboard.writeText(text).then(function () { done(true); }, legacy);
|
|
868
|
+
} else {
|
|
869
|
+
legacy();
|
|
870
|
+
}
|
|
871
|
+
});
|
|
872
|
+
JS
|
|
873
|
+
|
|
807
874
|
STYLES = <<~CSS.freeze
|
|
808
875
|
* { box-sizing: border-box; }
|
|
809
876
|
html, body { margin: 0; min-height: 100%; }
|
|
@@ -913,6 +980,14 @@ module ActiveAgent
|
|
|
913
980
|
.action { display: flex; align-items: center; gap: 10px; margin-top: auto; padding-top: 2px; flex-wrap: wrap; }
|
|
914
981
|
.btn { display: inline-block; padding: 6px 12px; border-radius: 8px; font-size: 13px; font-weight: 500; color: var(--color-text-cell); border: 1px solid var(--color-border-strong); background: transparent; white-space: nowrap; }
|
|
915
982
|
.btn:hover { background: var(--color-hover); color: var(--color-text-primary); text-decoration: none; }
|
|
983
|
+
.copy-md { font-family: var(--font-mono); font-size: 11px; padding: 3px 8px; border-radius: 6px; border: 1px solid var(--color-border); background: transparent; color: var(--color-text-secondary); cursor: pointer; white-space: nowrap; }
|
|
984
|
+
.copy-md:hover { background: var(--color-hover); color: var(--color-text-primary); }
|
|
985
|
+
.title-row { display: flex; align-items: center; gap: 12px; flex-wrap: wrap; }
|
|
986
|
+
.section-head .copy-md, .panel-head .copy-md { margin-left: auto; }
|
|
987
|
+
.panel-head .right + .copy-md { margin-left: 0; }
|
|
988
|
+
.drill-copy, .fix-copy { display: flex; justify-content: flex-end; }
|
|
989
|
+
.drill-copy { grid-column: 1 / -1; }
|
|
990
|
+
@media print { .copy-md { display: none; } }
|
|
916
991
|
.hint { font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); white-space: nowrap; }
|
|
917
992
|
.matrix { border: 1px solid var(--color-border-light); border-radius: 10px; overflow-x: auto; }
|
|
918
993
|
.mx { display: grid; gap: 12px; padding: 10px 12px; border-top: 1px solid var(--color-border-light); }
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveAgent
|
|
4
|
+
module Evals
|
|
5
|
+
# Report#to_markdown: the run as GitHub-flavored Markdown with the same
|
|
6
|
+
# content as Report#to_html — the per-model table and the cost, the
|
|
7
|
+
# verdict with the model comparison, the scenario matrix, every answer
|
|
8
|
+
# (one `<details>` block per scenario, since GitHub strips the HTML
|
|
9
|
+
# page's styles), and What to fix from Report#fix_items — so it can be
|
|
10
|
+
# pasted into a pull request or an issue.
|
|
11
|
+
#
|
|
12
|
+
# Each piece is public on its own, for the pieces people paste one at a
|
|
13
|
+
# time: #fix_item_markdown, #fix_items_markdown (optionally for one
|
|
14
|
+
# model), #verdict_markdown and #scenario_markdown. The whole report is
|
|
15
|
+
# built from them, so a piece copied alone reads as it does in the report.
|
|
16
|
+
#
|
|
17
|
+
# Table cells escape `|` and fold line breaks; text set inside the HTML
|
|
18
|
+
# of a `<details>` block escapes `<`, `>` and `&`.
|
|
19
|
+
module ReportMarkdown
|
|
20
|
+
# How much of an answer the whole report quotes; a scenario copied on
|
|
21
|
+
# its own quotes the whole answer.
|
|
22
|
+
MARKDOWN_ANSWER_LIMIT = 1_500
|
|
23
|
+
|
|
24
|
+
def to_markdown
|
|
25
|
+
lines = [ "# #{markdown_title}", "" ]
|
|
26
|
+
lines << judged_by_line
|
|
27
|
+
lines << ""
|
|
28
|
+
lines.concat(summary_table)
|
|
29
|
+
lines << ""
|
|
30
|
+
lines.concat(total_lines)
|
|
31
|
+
lines.concat(verdict_lines)
|
|
32
|
+
lines.concat(comparison_lines)
|
|
33
|
+
lines.concat(matrix_table)
|
|
34
|
+
lines.concat([ "", "## Answers", "" ])
|
|
35
|
+
scenario_keys.each { |key| lines << scenario_markdown(key, details: true, answer_limit: MARKDOWN_ANSWER_LIMIT) << "" }
|
|
36
|
+
lines << fix_items_markdown(heading: "##")
|
|
37
|
+
"#{lines.join("\n").rstrip}\n"
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# One fix item (Report#fix_items) as a heading and its body: the fault
|
|
41
|
+
# and how often, the scenarios and models it covers, the
|
|
42
|
+
# recommendation, the judge's proposed instruction change as a quote,
|
|
43
|
+
# the tools involved, the server that serves them, the note and where
|
|
44
|
+
# in the dashboard the fix lives.
|
|
45
|
+
def fix_item_markdown(item, heading: "###")
|
|
46
|
+
item = item.to_h.stringify_keys
|
|
47
|
+
title = item["fault"].to_s.tr("_", " ") + (item["count"].to_i > 1 ? " ×#{item['count']}" : "")
|
|
48
|
+
lines = [ "#{heading} #{item['kind'] == 'instruction' ? 'ℹ️' : '❗'} #{title}", "" ]
|
|
49
|
+
|
|
50
|
+
scope = [ "Scenarios: #{Array(item['scenario_keys']).map { |key| "`#{key}`" }.join(', ')}" ]
|
|
51
|
+
scope << "models: #{Array(item['models']).map { |label| "`#{label}`" }.join(', ')}" if Array(item["models"]).any?
|
|
52
|
+
scope << "judge suggestion" if item["kind"] == "instruction"
|
|
53
|
+
lines << scope.join(" · ") << ""
|
|
54
|
+
lines << item["recommendation"].to_s.strip << "" if item["recommendation"].present?
|
|
55
|
+
lines.concat(quoted(item["quote"]).push("")) if item["quote"].present?
|
|
56
|
+
tools = Array(item["tools"])
|
|
57
|
+
if tools.any?
|
|
58
|
+
lines << "**#{item['tools_label'].presence || 'tools'}:** " + tools.map { |tool| tool_text(tool) }.join(", ")
|
|
59
|
+
lines << ""
|
|
60
|
+
end
|
|
61
|
+
if (server = item["server"])
|
|
62
|
+
state = server["status"] == "enabled" ? "enabled for #{@agent_name}" : "#{server['status'].presence || 'unknown'} · not enabled for #{@agent_name}"
|
|
63
|
+
lines << "Served by **#{server['name'].presence || server['key']}** (#{state})" << ""
|
|
64
|
+
end
|
|
65
|
+
lines << "_#{item['note'].to_s.strip}_" << "" if item["note"].present?
|
|
66
|
+
if (action = item["action"])
|
|
67
|
+
lines << "Fix: #{[ action['label'], action['hint'] ].compact_blank.join(' — ')}" << ""
|
|
68
|
+
end
|
|
69
|
+
lines.join("\n").rstrip
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# What to fix: every fix item, or with +model+ (a model label) the ones
|
|
73
|
+
# attributed to that model, as the report's model filter shows them.
|
|
74
|
+
def fix_items_markdown(model: nil, heading: "##")
|
|
75
|
+
items = fix_items
|
|
76
|
+
items = items.select { |item| Array(item["models"]).empty? || item["models"].include?(model) } if model.present?
|
|
77
|
+
lines = [ "#{heading} What to fix#{" for `#{model}`" if model.present?}", "" ]
|
|
78
|
+
if items.empty?
|
|
79
|
+
lines << "_Nothing to fix._"
|
|
80
|
+
else
|
|
81
|
+
sub = "#{heading}#"
|
|
82
|
+
lines << items.map { |item| fix_item_markdown(item, heading: sub) }.join("\n\n")
|
|
83
|
+
end
|
|
84
|
+
lines.join("\n")
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# The verdict — the best model, why, and who picked it — then the
|
|
88
|
+
# per-model table and, comparing models, the comparison with its
|
|
89
|
+
# intervals.
|
|
90
|
+
def verdict_markdown(heading: "##")
|
|
91
|
+
lines = [ "#{heading} Verdict", "", judged_by_line, "" ]
|
|
92
|
+
lines.concat(verdict_lines)
|
|
93
|
+
lines.concat(summary_table)
|
|
94
|
+
lines << ""
|
|
95
|
+
lines.concat(total_lines)
|
|
96
|
+
lines.concat(comparison_lines.map { |line| line.start_with?("## ") ? line.sub("## ", "#{heading}# ") : line })
|
|
97
|
+
lines.join("\n").rstrip
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# One scenario: its prompt and expectations, a table of each model's
|
|
101
|
+
# status, score, tool calls, fault and cost, then each model's answer.
|
|
102
|
+
# With +details+ it is a `<details>` block whose summary names the key,
|
|
103
|
+
# the prompt and each model's mark, the way the report folds it.
|
|
104
|
+
# Raises KeyError for a key the report has no result for.
|
|
105
|
+
def scenario_markdown(key, details: false, answer_limit: nil)
|
|
106
|
+
cohort = @results.select { |result| result.scenario.key == key.to_s }
|
|
107
|
+
raise KeyError, "no results for scenario #{key}" if cohort.empty?
|
|
108
|
+
|
|
109
|
+
scenario = cohort.first.scenario
|
|
110
|
+
labels = @models.map(&:label)
|
|
111
|
+
cohort = cohort.sort_by { |result| labels.index(result.label) || labels.size }
|
|
112
|
+
body = []
|
|
113
|
+
body << "**Prompt**" << "" << quoted(scenario.prompt).join("\n") << ""
|
|
114
|
+
expectations = expectation_lines(scenario)
|
|
115
|
+
body.concat(expectations).push("") if expectations.any?
|
|
116
|
+
body << "| Model | Status | Score | Tools | Fault | Cost |" << "|---|---|---|---|---|---|"
|
|
117
|
+
cohort.each do |result|
|
|
118
|
+
tools = result.replay.tool_calls.map { |call| "#{call['name']}#{' ✗' if call['error']}" }.join(", ")
|
|
119
|
+
body << "| #{[ "`#{result.label}`", status_text(result), (result.score ? Format.score(result.score) : '—'), tools.presence || '—',
|
|
120
|
+
result.fault&.tr('_', ' ') || '—', (result.replay.cost ? answer_cost(result) : '—') ].map { |text| cell(text) }.join(' | ')} |"
|
|
121
|
+
end
|
|
122
|
+
body << ""
|
|
123
|
+
cohort.each do |result|
|
|
124
|
+
body << "**`#{result.label}`** — #{status_text(result)}"
|
|
125
|
+
body << ""
|
|
126
|
+
body << "Fault: #{result.fault.tr('_', ' ')} — #{[ result.summary, result.recommendation ].compact.join(' ')}" << "" if result.fault
|
|
127
|
+
body << "Error: #{result.replay.error}" << "" if result.replay.error
|
|
128
|
+
answer = result.replay.answer.to_s
|
|
129
|
+
answer = answer.truncate(answer_limit) if answer_limit
|
|
130
|
+
body << (answer.present? ? quoted(answer).join("\n") : "_(no answer)_") << ""
|
|
131
|
+
end
|
|
132
|
+
text = body.join("\n").rstrip
|
|
133
|
+
return "### Scenario `#{scenario.key}`\n\n#{text}" unless details
|
|
134
|
+
|
|
135
|
+
marks = cohort.map { |result| "#{status_mark(result)} #{result.label}" }.join(" · ")
|
|
136
|
+
summary = "<code>#{html_text(scenario.key)}</code> #{html_text(scenario.prompt.truncate(80))} — #{html_text(marks)}"
|
|
137
|
+
"<details>\n<summary>#{summary}</summary>\n\n#{text}\n\n</details>"
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
private
|
|
141
|
+
|
|
142
|
+
def markdown_title
|
|
143
|
+
scenarios = scenario_keys.size
|
|
144
|
+
"Evaluation — #{scenarios} scenario#{'s' unless scenarios == 1} × #{@models.size} model#{'s' unless @models.size == 1}"
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def scenario_keys
|
|
148
|
+
@results.map { |result| result.scenario.key }.uniq
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def judged_by_line
|
|
152
|
+
label = @judge_label || @judge&.label
|
|
153
|
+
label ? "Judged by `#{label}`." : "No judge; scored on rules and expectations alone."
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# "**Best model: gpt-5-mini**" with the verdict's rationale under it
|
|
157
|
+
# and, when a judge (not the pass-rate ranking) wrote it, who did.
|
|
158
|
+
def verdict_lines
|
|
159
|
+
return [] unless winner
|
|
160
|
+
|
|
161
|
+
lines = [ "**Best model: #{winner}**", "" ]
|
|
162
|
+
rationale = verdict["rationale"].to_s.strip
|
|
163
|
+
judge = verdict["judge"].presence
|
|
164
|
+
rationale += " _(#{judge == Report::PASS_RATE_JUDGE ? 'ranked by pass rate' : "judged by #{judge}"})_" if judge && rationale.present?
|
|
165
|
+
lines << rationale << "" if rationale.present?
|
|
166
|
+
lines
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def expectation_lines(scenario)
|
|
170
|
+
lines = []
|
|
171
|
+
lines << "Expects tools: #{scenario.expected_tools.map { |name| "`#{name}`" }.join(' or ')}" if scenario.expected_tools.any?
|
|
172
|
+
lines << "Expects the answer to contain: #{scenario.expected_patterns.map { |text| "`#{text}`" }.join(', ')}" if scenario.expected_patterns.any?
|
|
173
|
+
lines << "Expects the answer not to contain: #{scenario.forbidden_patterns.map { |text| "`#{text}`" }.join(', ')}" if scenario.forbidden_patterns.any?
|
|
174
|
+
lines.map { |line| "- #{line}" }
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def status_mark(result)
|
|
178
|
+
result.passed? ? "✅" : (result.errored? ? "⚠️" : "❌")
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def status_text(result)
|
|
182
|
+
"#{status_mark(result)} #{result.status}"
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def tool_text(tool)
|
|
186
|
+
tool = tool.is_a?(Hash) ? tool.stringify_keys : { "name" => tool.to_s }
|
|
187
|
+
note = tool["note"].presence
|
|
188
|
+
"`#{tool['name']}`#{" (#{note})" if note}"
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# Each line of +text+ as a blockquote line, `<` and `>` escaped so an
|
|
192
|
+
# answer cannot open or close the HTML around it.
|
|
193
|
+
def quoted(text)
|
|
194
|
+
text.to_s.strip.split(/\r?\n/).map { |line| line.strip.empty? ? ">" : "> #{line.gsub('<', '<').gsub('>', '>')}" }
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def html_text(text)
|
|
198
|
+
text.to_s.gsub("&", "&").gsub("<", "<").gsub(">", ">").gsub(/\s+/, " ")
|
|
199
|
+
end
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
end
|
data/lib/active_agent/evals.rb
CHANGED
|
@@ -27,6 +27,7 @@ require_relative "evals/judge"
|
|
|
27
27
|
require_relative "evals/result"
|
|
28
28
|
require_relative "evals/design_tokens"
|
|
29
29
|
require_relative "evals/report_html"
|
|
30
|
+
require_relative "evals/report_markdown"
|
|
30
31
|
require_relative "evals/report"
|
|
31
32
|
require_relative "evals/runner"
|
|
32
33
|
require_relative "evals/correlation"
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="utf-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
6
|
+
<title>ActiveAgent Previews</title>
|
|
7
|
+
<style>
|
|
8
|
+
body { color-scheme: light dark; font: 16px/1.5 system-ui, sans-serif; max-width: 1000px; padding: 1rem; margin: auto; }
|
|
9
|
+
pre { white-space: pre-wrap; overflow-wrap: anywhere; border: 1px solid; border-radius: .25rem; padding: 1rem; }
|
|
10
|
+
a { text-underline-offset: .2em; }
|
|
11
|
+
</style>
|
|
12
|
+
</head>
|
|
13
|
+
<body>
|
|
14
|
+
<nav><%= link_to "All agent previews", rails_agent_previews_path %></nav>
|
|
15
|
+
<%= yield %>
|
|
16
|
+
</body>
|
|
17
|
+
</html>
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
<h1>Agent Previews</h1>
|
|
2
|
+
<% if @previews.empty? %>
|
|
3
|
+
<p>No previews found. Generate an agent or add an ActiveAgent::Preview subclass under test/agents/previews.</p>
|
|
4
|
+
<% end %>
|
|
5
|
+
<% @previews.each do |preview| %>
|
|
6
|
+
<section>
|
|
7
|
+
<h2><%= preview.preview_name %></h2>
|
|
8
|
+
<ul>
|
|
9
|
+
<% preview.prompts.each do |prompt| %>
|
|
10
|
+
<li><%= link_to prompt, rails_agent_preview_path("#{preview.preview_name}/#{prompt}") %></li>
|
|
11
|
+
<% end %>
|
|
12
|
+
</ul>
|
|
13
|
+
</section>
|
|
14
|
+
<% end %>
|
|
@@ -132,6 +132,7 @@ module ActiveAgent
|
|
|
132
132
|
:stream_broadcaster, :streaming, # Callback (Streams)
|
|
133
133
|
:stream_completion_pending, # Callback (Streams)
|
|
134
134
|
:stream_completion_result, # Callback (Streams)
|
|
135
|
+
:stream_finish_reason, # Callback (Streams)
|
|
135
136
|
:tools_function, # Callback (Tools)
|
|
136
137
|
:usage_stack, # Usage Tracking
|
|
137
138
|
:stream_usage_index, # Usage Tracking (Streams)
|
|
@@ -607,6 +608,7 @@ module ActiveAgent
|
|
|
607
608
|
messages:,
|
|
608
609
|
raw_request: prompt_request_type.serialize(request),
|
|
609
610
|
raw_response: api_response,
|
|
611
|
+
streamed_finish_reason: stream_finish_reason,
|
|
610
612
|
usages: usage_stack,
|
|
611
613
|
**attributes
|
|
612
614
|
)
|
|
@@ -237,6 +237,7 @@ module ActiveAgent
|
|
|
237
237
|
when :message_delta
|
|
238
238
|
delta = Anthropic::Transforms.gem_to_hash(api_response_chunk.delta)
|
|
239
239
|
self.message_stack.last.merge!(delta)
|
|
240
|
+
self.stream_finish_reason = delta[:stop_reason] if delta[:stop_reason]
|
|
240
241
|
|
|
241
242
|
# Message Completed [Full Message]
|
|
242
243
|
when :message_stop
|
|
@@ -63,6 +63,16 @@ module ActiveAgent
|
|
|
63
63
|
# @return [Array<Usage>]
|
|
64
64
|
attribute :usages, default: -> { [] }, writable: false
|
|
65
65
|
|
|
66
|
+
# @!attribute [r] streamed_finish_reason
|
|
67
|
+
# Why a streamed generation stopped, as the provider reported it.
|
|
68
|
+
#
|
|
69
|
+
# A streamed generation has no response body to read it from, so
|
|
70
|
+
# the provider records it from the stream. {#finish_reason} falls
|
|
71
|
+
# back to it.
|
|
72
|
+
#
|
|
73
|
+
# @return [String, nil]
|
|
74
|
+
attribute :streamed_finish_reason, writable: false
|
|
75
|
+
|
|
66
76
|
# Initializes response with deep-duplicated attributes.
|
|
67
77
|
#
|
|
68
78
|
# Deep duplication prevents external modifications from affecting internal state.
|
|
@@ -180,14 +190,17 @@ module ActiveAgent
|
|
|
180
190
|
# response.stop_reason #=> "stop" (alias)
|
|
181
191
|
def finish_reason
|
|
182
192
|
@finish_reason ||= begin
|
|
183
|
-
return
|
|
193
|
+
return streamed_finish_reason unless raw_response
|
|
184
194
|
|
|
185
195
|
if raw_response.is_a?(Hash)
|
|
186
196
|
# OpenAI format: choices[0].finish_reason or choices[0].message.finish_reason
|
|
187
197
|
raw_response.dig(:choices, 0, :finish_reason) ||
|
|
188
198
|
raw_response.dig(:choices, 0, :message, :finish_reason) ||
|
|
189
199
|
# Anthropic format: stop_reason
|
|
190
|
-
raw_response[:stop_reason]
|
|
200
|
+
raw_response[:stop_reason] ||
|
|
201
|
+
# OpenAI Responses format: why it was cut short, else its status
|
|
202
|
+
raw_response.dig(:incomplete_details, :reason) ||
|
|
203
|
+
raw_response[:status]
|
|
191
204
|
end
|
|
192
205
|
end
|
|
193
206
|
end
|
|
@@ -335,8 +335,10 @@ module ActiveAgent
|
|
|
335
335
|
# Detect Ollama by presence of nanosecond duration fields
|
|
336
336
|
if usage.key?(:total_duration)
|
|
337
337
|
from_ollama(usage_hash)
|
|
338
|
-
# Detect Anthropic by presence of cache_creation or service_tier
|
|
339
|
-
|
|
338
|
+
# Detect Anthropic by presence of cache_creation or service_tier, or by its
|
|
339
|
+
# cache token fields (a usage payload does not always carry the former two)
|
|
340
|
+
elsif usage.key?(:cache_creation) || usage.key?(:service_tier) ||
|
|
341
|
+
usage.key?(:cache_read_input_tokens) || usage.key?(:cache_creation_input_tokens)
|
|
340
342
|
from_anthropic(usage_hash)
|
|
341
343
|
# Detect OpenAI Responses API by input_tokens/output_tokens with details
|
|
342
344
|
elsif usage.key?(:input_tokens) && usage.key?(:input_tokens_details)
|
|
@@ -117,6 +117,10 @@ module ActiveAgent
|
|
|
117
117
|
return if api_message.nil?
|
|
118
118
|
|
|
119
119
|
# If we have a delta, we need to update a message in the stack
|
|
120
|
+
if (finish_reason = api_message.try(:finish_reason))
|
|
121
|
+
self.stream_finish_reason = finish_reason.to_s
|
|
122
|
+
end
|
|
123
|
+
|
|
120
124
|
message = find_or_create_message(api_message.index)
|
|
121
125
|
message_merge_delta(message, api_message.delta.deep_to_h)
|
|
122
126
|
|
|
@@ -263,6 +267,39 @@ module ActiveAgent
|
|
|
263
267
|
message_stack.last
|
|
264
268
|
end
|
|
265
269
|
|
|
270
|
+
# Finds the streamed tool call an unindexed delta continues
|
|
271
|
+
#
|
|
272
|
+
# OpenAI-compatible servers (Gemini's compatibility endpoint, some Ollama
|
|
273
|
+
# and OpenRouter backends) omit the `index` that ties argument fragments
|
|
274
|
+
# to their call. A delta carrying an `id` continues the call with that
|
|
275
|
+
# id; one without an id or a function name continues the last call;
|
|
276
|
+
# anything else starts a new call.
|
|
277
|
+
#
|
|
278
|
+
# @param tool_calls [Array<Hash>] calls accumulated so far
|
|
279
|
+
# @param delta_item [Hash] unindexed tool call delta
|
|
280
|
+
# @return [Hash, nil] the call to merge into, or nil for a new call
|
|
281
|
+
def unindexed_tool_call_target(tool_calls, delta_item)
|
|
282
|
+
if delta_item[:id].present?
|
|
283
|
+
tool_calls.find { _1[:id] == delta_item[:id] }
|
|
284
|
+
elsif delta_item.dig(:function, :name).blank?
|
|
285
|
+
tool_calls.last
|
|
286
|
+
end
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
# Strips the fields a server repeats on every fragment (`id`, `type`,
|
|
290
|
+
# the function name), so merging does not concatenate them.
|
|
291
|
+
#
|
|
292
|
+
# @param hash_item [Hash] the call the fragment continues
|
|
293
|
+
# @param delta_item [Hash] unindexed tool call delta
|
|
294
|
+
# @return [Hash] the fragment to merge
|
|
295
|
+
def unindexed_tool_call_fragment(hash_item, delta_item)
|
|
296
|
+
fragment = delta_item.except(:id, :type)
|
|
297
|
+
if fragment[:function].is_a?(Hash) && fragment.dig(:function, :name) == hash_item.dig(:function, :name)
|
|
298
|
+
fragment = fragment.merge(function: fragment[:function].except(:name))
|
|
299
|
+
end
|
|
300
|
+
fragment
|
|
301
|
+
end
|
|
302
|
+
|
|
266
303
|
# Recursively merges delta changes into a hash structure
|
|
267
304
|
#
|
|
268
305
|
# Handles complex delta merging for OpenAI's streaming API, including
|
|
@@ -285,6 +322,8 @@ module ActiveAgent
|
|
|
285
322
|
else
|
|
286
323
|
hash[key] << delta_item
|
|
287
324
|
end
|
|
325
|
+
elsif key == :tool_calls && delta_item.is_a?(Hash) && (hash_item = unindexed_tool_call_target(hash[key], delta_item))
|
|
326
|
+
hash_merge_delta(hash_item, unindexed_tool_call_fragment(hash_item, delta_item))
|
|
288
327
|
else
|
|
289
328
|
hash[key] << delta_item
|
|
290
329
|
end
|
|
@@ -181,6 +181,9 @@ module ActiveAgent
|
|
|
181
181
|
# usage from, so it is recorded from the final event.
|
|
182
182
|
record_stream_usage(api_response_event.response.usage)
|
|
183
183
|
|
|
184
|
+
response = api_response_event.response
|
|
185
|
+
self.stream_finish_reason = (response.incomplete_details&.reason || response.status).to_s
|
|
186
|
+
|
|
184
187
|
# Completion, tool loop included, runs from stream_finished! once
|
|
185
188
|
# the stream has drained, so each turn's tool calls run once.
|
|
186
189
|
self.stream_completion_pending = true
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../open_ai/chat/_types"
|
|
4
|
+
require_relative "options"
|
|
5
|
+
|
|
6
|
+
module ActiveAgent
|
|
7
|
+
module Providers
|
|
8
|
+
module Opper
|
|
9
|
+
# ActiveModel type for casting and serializing Opper requests.
|
|
10
|
+
#
|
|
11
|
+
# Opper is OpenAI-compatible, so requests use the same shape as the
|
|
12
|
+
# OpenAI Chat API. This delegates entirely to OpenAI::Chat::RequestType.
|
|
13
|
+
RequestType = ActiveAgent::Providers::OpenAI::Chat::RequestType
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../open_ai/options"
|
|
4
|
+
|
|
5
|
+
module ActiveAgent
|
|
6
|
+
module Providers
|
|
7
|
+
module Opper
|
|
8
|
+
# Configuration options for the Opper provider.
|
|
9
|
+
#
|
|
10
|
+
# Extends OpenAI::Options, overriding the base URL to point at Opper's
|
|
11
|
+
# OpenAI-compatible gateway and resolving the API key from OPPER_API_KEY.
|
|
12
|
+
# Opper does not use organization or project identifiers.
|
|
13
|
+
#
|
|
14
|
+
# @example Basic configuration
|
|
15
|
+
# options = Options.new(api_key: ENV["OPPER_API_KEY"])
|
|
16
|
+
#
|
|
17
|
+
# @see https://docs.opper.ai
|
|
18
|
+
# @see https://platform.opper.ai Opper API Keys
|
|
19
|
+
class Options < ActiveAgent::Providers::OpenAI::Options
|
|
20
|
+
# @!attribute base_url
|
|
21
|
+
# @return [String] API endpoint (default: "https://api.opper.ai/v3/compat")
|
|
22
|
+
attribute :base_url, :string, as: "https://api.opper.ai/v3/compat"
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
|
|
26
|
+
def resolve_api_key(kwargs)
|
|
27
|
+
kwargs[:api_key] ||
|
|
28
|
+
kwargs[:access_token] ||
|
|
29
|
+
ENV["OPPER_API_KEY"]
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Not used as part of Opper
|
|
33
|
+
def resolve_organization_id(kwargs) = nil
|
|
34
|
+
def resolve_project_id(kwargs) = nil
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|