activeagent 1.9.2 → 1.9.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +252 -0
  3. data/README.md +3 -0
  4. data/lib/active_agent/base.rb +2 -2
  5. data/lib/active_agent/concerns/preview.rb +2 -2
  6. data/lib/active_agent/evals/design_tokens.rb +12 -1
  7. data/lib/active_agent/evals/publisher.rb +19 -12
  8. data/lib/active_agent/evals/report.rb +12 -62
  9. data/lib/active_agent/evals/report_html.rb +85 -10
  10. data/lib/active_agent/evals/report_markdown.rb +202 -0
  11. data/lib/active_agent/evals.rb +1 -0
  12. data/lib/active_agent/inline_preview_interceptor.rb +1 -0
  13. data/lib/active_agent/preview/templates/layouts/active_agent/preview.html.erb +17 -0
  14. data/lib/active_agent/preview/templates/rails/agents/index.html.erb +14 -0
  15. data/lib/active_agent/preview/templates/rails/agents/preview.html.erb +3 -0
  16. data/lib/active_agent/providers/_base_provider.rb +2 -0
  17. data/lib/active_agent/providers/anthropic_provider.rb +1 -0
  18. data/lib/active_agent/providers/common/responses/base.rb +15 -2
  19. data/lib/active_agent/providers/common/usage.rb +4 -2
  20. data/lib/active_agent/providers/open_ai/chat_provider.rb +39 -0
  21. data/lib/active_agent/providers/open_ai/responses_provider.rb +3 -0
  22. data/lib/active_agent/providers/opper/_types.rb +16 -0
  23. data/lib/active_agent/providers/opper/options.rb +38 -0
  24. data/lib/active_agent/providers/opper_provider.rb +72 -0
  25. data/lib/active_agent/railtie.rb +11 -5
  26. data/lib/active_agent/version.rb +1 -1
  27. data/lib/generators/test_unit/agent_generator.rb +1 -1
  28. data/lib/generators/test_unit/templates/functional_test.rb.tt +1 -1
  29. data/lib/generators/test_unit/templates/preview.rb.tt +3 -3
  30. data/lib/rails/agents_controller.rb +43 -0
  31. metadata +10 -2
@@ -5,10 +5,15 @@ require "cgi"
5
5
  module ActiveAgent
6
6
  module Evals
7
7
  # Report#to_html: the run as one self-contained page on the dashboard's
8
- # design system (DesignTokens) — inline styles, no scripts, no external
9
- # assets — so a run's outcome can be archived next to a CI run or handed
10
- # to a teammate, and so the dashboard can serve the same page for a
11
- # persisted run.
8
+ # design system (DesignTokens) — inline styles, no external assets — so a
9
+ # run's outcome can be archived next to a CI run or handed to a teammate,
10
+ # and so the dashboard can serve the same page for a persisted run.
11
+ #
12
+ # Its one inline script (COPY_SCRIPT) backs the "Copy as Markdown"
13
+ # buttons: the whole report, the verdict, What to fix (for the model the
14
+ # filter shows), each fix card and each scenario. The Markdown each copies
15
+ # is ReportMarkdown's, embedded in a hidden textarea, so the page copies
16
+ # exactly what the dashboard and the CLI write.
12
17
  #
13
18
  # Same content as Report#to_markdown, laid out the way the dashboard's
14
19
  # suite card is: header and stat tiles, the MODELS panel with the judge's
@@ -50,6 +55,8 @@ module ActiveAgent
50
55
  #{html_footer}
51
56
  </section>
52
57
  </main>
58
+ #{markdown_store("md-report", to_markdown)}
59
+ <script>#{COPY_SCRIPT}</script>
53
60
  </body>
54
61
  </html>
55
62
  HTML
@@ -237,7 +244,7 @@ module ActiveAgent
237
244
 
238
245
  <<~HEADER
239
246
  <header>
240
- <h1>#{h(title)}</h1>
247
+ <div class="title-row"><h1>#{h(title)}</h1>#{copy_button('md-report', label: 'Copy as Markdown', title: 'Copy the whole report as GitHub-flavored Markdown')}</div>
241
248
  <div class="chips">#{chips.join}</div>
242
249
  </header>
243
250
  HEADER
@@ -320,7 +327,8 @@ module ActiveAgent
320
327
 
321
328
  <<~PANEL
322
329
  <div class="panel">
323
- <div class="panel-head"><span class="micro">Models</span><span class="right">judged by #{h(judged_by)}</span></div>
330
+ <div class="panel-head"><span class="micro">Models</span><span class="right">judged by #{h(judged_by)}</span>#{copy_button('md-verdict', title: 'Copy the verdict and the model comparison as Markdown')}</div>
331
+ #{markdown_store('md-verdict', verdict_markdown)}
324
332
  #{html_comparison_table if comparing?}
325
333
  #{html_intervals}
326
334
  #{blocks.join}
@@ -612,12 +620,13 @@ module ActiveAgent
612
620
  expects = scenario.expected_tools.any? ? %(<span class="exp">expects <b>#{h(scenario.expected_tools.join(' or '))}</b></span>) : ""
613
621
  labels = @models.map(&:label)
614
622
  cards = cohort.sort_by { |result| labels.index(result.label) || labels.size }.map { |result| html_result_card(result) }
623
+ md_id = "md-#{anchor(scenario)}"
615
624
 
616
625
  <<~BLOCK
617
626
  <div id="#{h(anchor(scenario))}">
618
627
  <details>
619
628
  <summary><span class="chev">&gt;</span><span class="key">#{h(scenario.key)}</span><span class="prompt">#{h(scenario.prompt)}</span>#{expects}</summary>
620
- <div class="drill">#{cards.join}</div>
629
+ <div class="drill">#{cards.join}<div class="drill-copy">#{copy_button(md_id, title: "Copy #{scenario.key} as Markdown")}#{markdown_store(md_id, scenario_markdown(scenario.key, answer_limit: ANSWER_LIMIT))}</div></div>
621
630
  </details>
622
631
  </div>
623
632
  BLOCK
@@ -670,14 +679,16 @@ module ActiveAgent
670
679
  "#{plural(faulted.map { |result| result.scenario.key }.uniq.size, 'scenario')}"
671
680
  body =
672
681
  if items.any?
673
- %(<div class="fixes">#{items.map { |item| html_fix_card(item) }.join}</div>)
682
+ %(<div class="fixes">#{items.each_with_index.map { |item, index| html_fix_card(item, index) }.join}</div>)
674
683
  else
675
684
  %(<div class="nothing">[+] nothing to fix</div>)
676
685
  end
677
686
 
678
687
  <<~FIXES
679
688
  <section class="section fix-section" aria-label="Recommendations">
680
- <div class="section-head"><span class="micro">What to fix</span><span class="meta">#{h(meta)}</span></div>
689
+ <div class="section-head"><span class="micro">What to fix</span><span class="meta">#{h(meta)}</span>#{copy_button('md-fixes', model_filtered: comparing? && items.any?, title: 'Copy What to fix as Markdown, for the model the filter shows')}</div>
690
+ #{markdown_store('md-fixes', fix_items_markdown)}
691
+ #{@models.each_index.map { |index| markdown_store("md-fixes-m#{index}", fix_items_markdown(model: @models[index].label)) }.join if comparing? && items.any?}
681
692
  #{html_fix_filter(items) if comparing? && items.any?}
682
693
  #{body}
683
694
  </section>
@@ -706,7 +717,7 @@ module ActiveAgent
706
717
  labels.filter_map { |label| (index = @models.index(model_by_label(label))) && "m#{index}" }.join(" ")
707
718
  end
708
719
 
709
- def html_fix_card(item)
720
+ def html_fix_card(item, index)
710
721
  tone = item["kind"] == "instruction" ? "info" : "error"
711
722
  glyph = tone == "info" ? "[i]" : "[!]"
712
723
  title = fault_name(item["fault"]) + (item["count"].to_i > 1 ? " ×#{item['count']}" : "")
@@ -719,6 +730,7 @@ module ActiveAgent
719
730
  parts << html_fix_server(item["server"]) if item["server"]
720
731
  parts << %(<div class="note">#{h(item['note'])}</div>) if item["note"].present?
721
732
  parts << html_fix_action(item["action"]) if item["action"]
733
+ parts << %(<div class="fix-copy">#{copy_button("md-fix-#{index}", title: 'Copy this fix as Markdown')}#{markdown_store("md-fix-#{index}", fix_item_markdown(item))}</div>)
722
734
  models = fix_model_tokens(item)
723
735
  %(<div class="fix"#{%( data-models="#{models}") if models.present?}>#{parts.join}</div>)
724
736
  end
@@ -771,6 +783,19 @@ module ActiveAgent
771
783
  "#{scenarios} · #{models}"
772
784
  end
773
785
 
786
+ # --- Copy as Markdown -------------------------------------------------
787
+
788
+ # A button COPY_SCRIPT answers by copying the textarea +id+ names; with
789
+ # +model_filtered+, the one for the model the fix filter has checked.
790
+ def copy_button(id, label: "Copy as Markdown", title: nil, model_filtered: false)
791
+ %(<button type="button" class="copy-md" data-md="#{h(id)}"#{' data-md-model' if model_filtered}) +
792
+ %(#{%( title="#{h(title)}") if title}>#{h(label)}</button>)
793
+ end
794
+
795
+ def markdown_store(id, markdown)
796
+ %(<textarea class="md-store" id="#{h(id)}" hidden readonly>#{h(markdown)}</textarea>)
797
+ end
798
+
774
799
  # --- footer ----------------------------------------------------------
775
800
 
776
801
  # The run's terms: the judge, the criteria and the pass mark, what it
@@ -804,6 +829,48 @@ module ActiveAgent
804
829
 
805
830
  # Colors only through the token variables; radii 4 badges · 6 chips ·
806
831
  # 8 controls · 10 nested panels · 12 cards · 999 bars; no shadows.
832
+ # Copies the Markdown a button names: the async Clipboard API where the
833
+ # page may use it, else a selected textarea and the copy command, which
834
+ # still works on an http:// host and in the dashboard's frame.
835
+ COPY_SCRIPT = <<~JS.gsub(/\n\s*/, "").freeze
836
+ document.addEventListener("click", function (event) {
837
+ var button = event.target.closest && event.target.closest("button.copy-md");
838
+ if (!button) return;
839
+ event.preventDefault();
840
+ var id = button.getAttribute("data-md");
841
+ if (button.hasAttribute("data-md-model")) {
842
+ var picked = document.querySelector('input[name="fix-model"]:checked');
843
+ if (picked && picked.value !== "all") id += "-" + picked.value;
844
+ }
845
+ var store = document.getElementById(id);
846
+ if (!store) return;
847
+ var text = store.value;
848
+ var label = button.getAttribute("data-label") || button.textContent;
849
+ button.setAttribute("data-label", label);
850
+ var done = function (ok) {
851
+ button.textContent = ok ? "Copied" : "Copy failed";
852
+ setTimeout(function () { button.textContent = label; }, 1500);
853
+ };
854
+ var legacy = function () {
855
+ var area = document.createElement("textarea");
856
+ area.value = text;
857
+ area.style.position = "fixed";
858
+ area.style.opacity = "0";
859
+ document.body.appendChild(area);
860
+ area.select();
861
+ var ok = false;
862
+ try { ok = document.execCommand("copy"); } catch (error) {}
863
+ document.body.removeChild(area);
864
+ done(ok);
865
+ };
866
+ if (navigator.clipboard && navigator.clipboard.writeText) {
867
+ navigator.clipboard.writeText(text).then(function () { done(true); }, legacy);
868
+ } else {
869
+ legacy();
870
+ }
871
+ });
872
+ JS
873
+
807
874
  STYLES = <<~CSS.freeze
808
875
  * { box-sizing: border-box; }
809
876
  html, body { margin: 0; min-height: 100%; }
@@ -913,6 +980,14 @@ module ActiveAgent
913
980
  .action { display: flex; align-items: center; gap: 10px; margin-top: auto; padding-top: 2px; flex-wrap: wrap; }
914
981
  .btn { display: inline-block; padding: 6px 12px; border-radius: 8px; font-size: 13px; font-weight: 500; color: var(--color-text-cell); border: 1px solid var(--color-border-strong); background: transparent; white-space: nowrap; }
915
982
  .btn:hover { background: var(--color-hover); color: var(--color-text-primary); text-decoration: none; }
983
+ .copy-md { font-family: var(--font-mono); font-size: 11px; padding: 3px 8px; border-radius: 6px; border: 1px solid var(--color-border); background: transparent; color: var(--color-text-secondary); cursor: pointer; white-space: nowrap; }
984
+ .copy-md:hover { background: var(--color-hover); color: var(--color-text-primary); }
985
+ .title-row { display: flex; align-items: center; gap: 12px; flex-wrap: wrap; }
986
+ .section-head .copy-md, .panel-head .copy-md { margin-left: auto; }
987
+ .panel-head .right + .copy-md { margin-left: 0; }
988
+ .drill-copy, .fix-copy { display: flex; justify-content: flex-end; }
989
+ .drill-copy { grid-column: 1 / -1; }
990
+ @media print { .copy-md { display: none; } }
916
991
  .hint { font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); white-space: nowrap; }
917
992
  .matrix { border: 1px solid var(--color-border-light); border-radius: 10px; overflow-x: auto; }
918
993
  .mx { display: grid; gap: 12px; padding: 10px 12px; border-top: 1px solid var(--color-border-light); }
@@ -0,0 +1,202 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveAgent
4
+ module Evals
5
+ # Report#to_markdown: the run as GitHub-flavored Markdown with the same
6
+ # content as Report#to_html — the per-model table and the cost, the
7
+ # verdict with the model comparison, the scenario matrix, every answer
8
+ # (one `<details>` block per scenario, since GitHub strips the HTML
9
+ # page's styles), and What to fix from Report#fix_items — so it can be
10
+ # pasted into a pull request or an issue.
11
+ #
12
+ # Each piece is public on its own, for the pieces people paste one at a
13
+ # time: #fix_item_markdown, #fix_items_markdown (optionally for one
14
+ # model), #verdict_markdown and #scenario_markdown. The whole report is
15
+ # built from them, so a piece copied alone reads as it does in the report.
16
+ #
17
+ # Table cells escape `|` and fold line breaks; text set inside the HTML
18
+ # of a `<details>` block escapes `<`, `>` and `&`.
19
+ module ReportMarkdown
20
+ # How much of an answer the whole report quotes; a scenario copied on
21
+ # its own quotes the whole answer.
22
+ MARKDOWN_ANSWER_LIMIT = 1_500
23
+
24
+ def to_markdown
25
+ lines = [ "# #{markdown_title}", "" ]
26
+ lines << judged_by_line
27
+ lines << ""
28
+ lines.concat(summary_table)
29
+ lines << ""
30
+ lines.concat(total_lines)
31
+ lines.concat(verdict_lines)
32
+ lines.concat(comparison_lines)
33
+ lines.concat(matrix_table)
34
+ lines.concat([ "", "## Answers", "" ])
35
+ scenario_keys.each { |key| lines << scenario_markdown(key, details: true, answer_limit: MARKDOWN_ANSWER_LIMIT) << "" }
36
+ lines << fix_items_markdown(heading: "##")
37
+ "#{lines.join("\n").rstrip}\n"
38
+ end
39
+
40
+ # One fix item (Report#fix_items) as a heading and its body: the fault
41
+ # and how often, the scenarios and models it covers, the
42
+ # recommendation, the judge's proposed instruction change as a quote,
43
+ # the tools involved, the server that serves them, the note and where
44
+ # in the dashboard the fix lives.
45
+ def fix_item_markdown(item, heading: "###")
46
+ item = item.to_h.stringify_keys
47
+ title = item["fault"].to_s.tr("_", " ") + (item["count"].to_i > 1 ? " ×#{item['count']}" : "")
48
+ lines = [ "#{heading} #{item['kind'] == 'instruction' ? 'ℹ️' : '❗'} #{title}", "" ]
49
+
50
+ scope = [ "Scenarios: #{Array(item['scenario_keys']).map { |key| "`#{key}`" }.join(', ')}" ]
51
+ scope << "models: #{Array(item['models']).map { |label| "`#{label}`" }.join(', ')}" if Array(item["models"]).any?
52
+ scope << "judge suggestion" if item["kind"] == "instruction"
53
+ lines << scope.join(" · ") << ""
54
+ lines << item["recommendation"].to_s.strip << "" if item["recommendation"].present?
55
+ lines.concat(quoted(item["quote"]).push("")) if item["quote"].present?
56
+ tools = Array(item["tools"])
57
+ if tools.any?
58
+ lines << "**#{item['tools_label'].presence || 'tools'}:** " + tools.map { |tool| tool_text(tool) }.join(", ")
59
+ lines << ""
60
+ end
61
+ if (server = item["server"])
62
+ state = server["status"] == "enabled" ? "enabled for #{@agent_name}" : "#{server['status'].presence || 'unknown'} · not enabled for #{@agent_name}"
63
+ lines << "Served by **#{server['name'].presence || server['key']}** (#{state})" << ""
64
+ end
65
+ lines << "_#{item['note'].to_s.strip}_" << "" if item["note"].present?
66
+ if (action = item["action"])
67
+ lines << "Fix: #{[ action['label'], action['hint'] ].compact_blank.join(' — ')}" << ""
68
+ end
69
+ lines.join("\n").rstrip
70
+ end
71
+
72
+ # What to fix: every fix item, or with +model+ (a model label) the ones
73
+ # attributed to that model, as the report's model filter shows them.
74
+ def fix_items_markdown(model: nil, heading: "##")
75
+ items = fix_items
76
+ items = items.select { |item| Array(item["models"]).empty? || item["models"].include?(model) } if model.present?
77
+ lines = [ "#{heading} What to fix#{" for `#{model}`" if model.present?}", "" ]
78
+ if items.empty?
79
+ lines << "_Nothing to fix._"
80
+ else
81
+ sub = "#{heading}#"
82
+ lines << items.map { |item| fix_item_markdown(item, heading: sub) }.join("\n\n")
83
+ end
84
+ lines.join("\n")
85
+ end
86
+
87
+ # The verdict — the best model, why, and who picked it — then the
88
+ # per-model table and, comparing models, the comparison with its
89
+ # intervals.
90
+ def verdict_markdown(heading: "##")
91
+ lines = [ "#{heading} Verdict", "", judged_by_line, "" ]
92
+ lines.concat(verdict_lines)
93
+ lines.concat(summary_table)
94
+ lines << ""
95
+ lines.concat(total_lines)
96
+ lines.concat(comparison_lines.map { |line| line.start_with?("## ") ? line.sub("## ", "#{heading}# ") : line })
97
+ lines.join("\n").rstrip
98
+ end
99
+
100
+ # One scenario: its prompt and expectations, a table of each model's
101
+ # status, score, tool calls, fault and cost, then each model's answer.
102
+ # With +details+ it is a `<details>` block whose summary names the key,
103
+ # the prompt and each model's mark, the way the report folds it.
104
+ # Raises KeyError for a key the report has no result for.
105
+ def scenario_markdown(key, details: false, answer_limit: nil)
106
+ cohort = @results.select { |result| result.scenario.key == key.to_s }
107
+ raise KeyError, "no results for scenario #{key}" if cohort.empty?
108
+
109
+ scenario = cohort.first.scenario
110
+ labels = @models.map(&:label)
111
+ cohort = cohort.sort_by { |result| labels.index(result.label) || labels.size }
112
+ body = []
113
+ body << "**Prompt**" << "" << quoted(scenario.prompt).join("\n") << ""
114
+ expectations = expectation_lines(scenario)
115
+ body.concat(expectations).push("") if expectations.any?
116
+ body << "| Model | Status | Score | Tools | Fault | Cost |" << "|---|---|---|---|---|---|"
117
+ cohort.each do |result|
118
+ tools = result.replay.tool_calls.map { |call| "#{call['name']}#{' ✗' if call['error']}" }.join(", ")
119
+ body << "| #{[ "`#{result.label}`", status_text(result), (result.score ? Format.score(result.score) : '—'), tools.presence || '—',
120
+ result.fault&.tr('_', ' ') || '—', (result.replay.cost ? answer_cost(result) : '—') ].map { |text| cell(text) }.join(' | ')} |"
121
+ end
122
+ body << ""
123
+ cohort.each do |result|
124
+ body << "**`#{result.label}`** — #{status_text(result)}"
125
+ body << ""
126
+ body << "Fault: #{result.fault.tr('_', ' ')} — #{[ result.summary, result.recommendation ].compact.join(' ')}" << "" if result.fault
127
+ body << "Error: #{result.replay.error}" << "" if result.replay.error
128
+ answer = result.replay.answer.to_s
129
+ answer = answer.truncate(answer_limit) if answer_limit
130
+ body << (answer.present? ? quoted(answer).join("\n") : "_(no answer)_") << ""
131
+ end
132
+ text = body.join("\n").rstrip
133
+ return "### Scenario `#{scenario.key}`\n\n#{text}" unless details
134
+
135
+ marks = cohort.map { |result| "#{status_mark(result)} #{result.label}" }.join(" · ")
136
+ summary = "<code>#{html_text(scenario.key)}</code> #{html_text(scenario.prompt.truncate(80))} — #{html_text(marks)}"
137
+ "<details>\n<summary>#{summary}</summary>\n\n#{text}\n\n</details>"
138
+ end
139
+
140
+ private
141
+
142
+ def markdown_title
143
+ scenarios = scenario_keys.size
144
+ "Evaluation — #{scenarios} scenario#{'s' unless scenarios == 1} × #{@models.size} model#{'s' unless @models.size == 1}"
145
+ end
146
+
147
+ def scenario_keys
148
+ @results.map { |result| result.scenario.key }.uniq
149
+ end
150
+
151
+ def judged_by_line
152
+ label = @judge_label || @judge&.label
153
+ label ? "Judged by `#{label}`." : "No judge; scored on rules and expectations alone."
154
+ end
155
+
156
+ # "**Best model: gpt-5-mini**" with the verdict's rationale under it
157
+ # and, when a judge (not the pass-rate ranking) wrote it, who did.
158
+ def verdict_lines
159
+ return [] unless winner
160
+
161
+ lines = [ "**Best model: #{winner}**", "" ]
162
+ rationale = verdict["rationale"].to_s.strip
163
+ judge = verdict["judge"].presence
164
+ rationale += " _(#{judge == Report::PASS_RATE_JUDGE ? 'ranked by pass rate' : "judged by #{judge}"})_" if judge && rationale.present?
165
+ lines << rationale << "" if rationale.present?
166
+ lines
167
+ end
168
+
169
+ def expectation_lines(scenario)
170
+ lines = []
171
+ lines << "Expects tools: #{scenario.expected_tools.map { |name| "`#{name}`" }.join(' or ')}" if scenario.expected_tools.any?
172
+ lines << "Expects the answer to contain: #{scenario.expected_patterns.map { |text| "`#{text}`" }.join(', ')}" if scenario.expected_patterns.any?
173
+ lines << "Expects the answer not to contain: #{scenario.forbidden_patterns.map { |text| "`#{text}`" }.join(', ')}" if scenario.forbidden_patterns.any?
174
+ lines.map { |line| "- #{line}" }
175
+ end
176
+
177
+ def status_mark(result)
178
+ result.passed? ? "✅" : (result.errored? ? "⚠️" : "❌")
179
+ end
180
+
181
+ def status_text(result)
182
+ "#{status_mark(result)} #{result.status}"
183
+ end
184
+
185
+ def tool_text(tool)
186
+ tool = tool.is_a?(Hash) ? tool.stringify_keys : { "name" => tool.to_s }
187
+ note = tool["note"].presence
188
+ "`#{tool['name']}`#{" (#{note})" if note}"
189
+ end
190
+
191
+ # Each line of +text+ as a blockquote line, `<` and `>` escaped so an
192
+ # answer cannot open or close the HTML around it.
193
+ def quoted(text)
194
+ text.to_s.strip.split(/\r?\n/).map { |line| line.strip.empty? ? ">" : "> #{line.gsub('<', '&lt;').gsub('>', '&gt;')}" }
195
+ end
196
+
197
+ def html_text(text)
198
+ text.to_s.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;").gsub(/\s+/, " ")
199
+ end
200
+ end
201
+ end
202
+ end
@@ -27,6 +27,7 @@ require_relative "evals/judge"
27
27
  require_relative "evals/result"
28
28
  require_relative "evals/design_tokens"
29
29
  require_relative "evals/report_html"
30
+ require_relative "evals/report_markdown"
30
31
  require_relative "evals/report"
31
32
  require_relative "evals/runner"
32
33
  require_relative "evals/correlation"
@@ -28,6 +28,7 @@ module ActiveAgent
28
28
  end
29
29
 
30
30
  def transform! # :nodoc:
31
+ return context unless context.respond_to?(:html_part)
31
32
  return context if html_part.blank?
32
33
 
33
34
  html_part.body = html_part.decoded.gsub(PATTERN) do |match|
@@ -0,0 +1,17 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <title>ActiveAgent Previews</title>
7
+ <style>
8
+ body { color-scheme: light dark; font: 16px/1.5 system-ui, sans-serif; max-width: 1000px; padding: 1rem; margin: auto; }
9
+ pre { white-space: pre-wrap; overflow-wrap: anywhere; border: 1px solid; border-radius: .25rem; padding: 1rem; }
10
+ a { text-underline-offset: .2em; }
11
+ </style>
12
+ </head>
13
+ <body>
14
+ <nav><%= link_to "All agent previews", rails_agent_previews_path %></nav>
15
+ <%= yield %>
16
+ </body>
17
+ </html>
@@ -0,0 +1,14 @@
1
+ <h1>Agent Previews</h1>
2
+ <% if @previews.empty? %>
3
+ <p>No previews found. Generate an agent or add an ActiveAgent::Preview subclass under test/agents/previews.</p>
4
+ <% end %>
5
+ <% @previews.each do |preview| %>
6
+ <section>
7
+ <h2><%= preview.preview_name %></h2>
8
+ <ul>
9
+ <% preview.prompts.each do |prompt| %>
10
+ <li><%= link_to prompt, rails_agent_preview_path("#{preview.preview_name}/#{prompt}") %></li>
11
+ <% end %>
12
+ </ul>
13
+ </section>
14
+ <% end %>
@@ -0,0 +1,3 @@
1
+ <h1><%= @preview.preview_name %> / <%= @prompt %></h1>
2
+ <p><%= link_to "Download Markdown", "#{download_rails_agent_preview_path("#{@preview.preview_name}/#{@prompt}")}?#{request.query_parameters.to_query}" %></p>
3
+ <pre><%= @content %></pre>
@@ -132,6 +132,7 @@ module ActiveAgent
132
132
  :stream_broadcaster, :streaming, # Callback (Streams)
133
133
  :stream_completion_pending, # Callback (Streams)
134
134
  :stream_completion_result, # Callback (Streams)
135
+ :stream_finish_reason, # Callback (Streams)
135
136
  :tools_function, # Callback (Tools)
136
137
  :usage_stack, # Usage Tracking
137
138
  :stream_usage_index, # Usage Tracking (Streams)
@@ -607,6 +608,7 @@ module ActiveAgent
607
608
  messages:,
608
609
  raw_request: prompt_request_type.serialize(request),
609
610
  raw_response: api_response,
611
+ streamed_finish_reason: stream_finish_reason,
610
612
  usages: usage_stack,
611
613
  **attributes
612
614
  )
@@ -237,6 +237,7 @@ module ActiveAgent
237
237
  when :message_delta
238
238
  delta = Anthropic::Transforms.gem_to_hash(api_response_chunk.delta)
239
239
  self.message_stack.last.merge!(delta)
240
+ self.stream_finish_reason = delta[:stop_reason] if delta[:stop_reason]
240
241
 
241
242
  # Message Completed [Full Message]
242
243
  when :message_stop
@@ -63,6 +63,16 @@ module ActiveAgent
63
63
  # @return [Array<Usage>]
64
64
  attribute :usages, default: -> { [] }, writable: false
65
65
 
66
+ # @!attribute [r] streamed_finish_reason
67
+ # Why a streamed generation stopped, as the provider reported it.
68
+ #
69
+ # A streamed generation has no response body to read it from, so
70
+ # the provider records it from the stream. {#finish_reason} falls
71
+ # back to it.
72
+ #
73
+ # @return [String, nil]
74
+ attribute :streamed_finish_reason, writable: false
75
+
66
76
  # Initializes response with deep-duplicated attributes.
67
77
  #
68
78
  # Deep duplication prevents external modifications from affecting internal state.
@@ -180,14 +190,17 @@ module ActiveAgent
180
190
  # response.stop_reason #=> "stop" (alias)
181
191
  def finish_reason
182
192
  @finish_reason ||= begin
183
- return nil unless raw_response
193
+ return streamed_finish_reason unless raw_response
184
194
 
185
195
  if raw_response.is_a?(Hash)
186
196
  # OpenAI format: choices[0].finish_reason or choices[0].message.finish_reason
187
197
  raw_response.dig(:choices, 0, :finish_reason) ||
188
198
  raw_response.dig(:choices, 0, :message, :finish_reason) ||
189
199
  # Anthropic format: stop_reason
190
- raw_response[:stop_reason]
200
+ raw_response[:stop_reason] ||
201
+ # OpenAI Responses format: why it was cut short, else its status
202
+ raw_response.dig(:incomplete_details, :reason) ||
203
+ raw_response[:status]
191
204
  end
192
205
  end
193
206
  end
@@ -335,8 +335,10 @@ module ActiveAgent
335
335
  # Detect Ollama by presence of nanosecond duration fields
336
336
  if usage.key?(:total_duration)
337
337
  from_ollama(usage_hash)
338
- # Detect Anthropic by presence of cache_creation or service_tier
339
- elsif usage.key?(:cache_creation) || usage.key?(:service_tier)
338
+ # Detect Anthropic by presence of cache_creation or service_tier, or by its
339
+ # cache token fields (a usage payload does not always carry the former two)
340
+ elsif usage.key?(:cache_creation) || usage.key?(:service_tier) ||
341
+ usage.key?(:cache_read_input_tokens) || usage.key?(:cache_creation_input_tokens)
340
342
  from_anthropic(usage_hash)
341
343
  # Detect OpenAI Responses API by input_tokens/output_tokens with details
342
344
  elsif usage.key?(:input_tokens) && usage.key?(:input_tokens_details)
@@ -117,6 +117,10 @@ module ActiveAgent
117
117
  return if api_message.nil?
118
118
 
119
119
  # If we have a delta, we need to update a message in the stack
120
+ if (finish_reason = api_message.try(:finish_reason))
121
+ self.stream_finish_reason = finish_reason.to_s
122
+ end
123
+
120
124
  message = find_or_create_message(api_message.index)
121
125
  message_merge_delta(message, api_message.delta.deep_to_h)
122
126
 
@@ -263,6 +267,39 @@ module ActiveAgent
263
267
  message_stack.last
264
268
  end
265
269
 
270
+ # Finds the streamed tool call an unindexed delta continues
271
+ #
272
+ # OpenAI-compatible servers (Gemini's compatibility endpoint, some Ollama
273
+ # and OpenRouter backends) omit the `index` that ties argument fragments
274
+ # to their call. A delta carrying an `id` continues the call with that
275
+ # id; one without an id or a function name continues the last call;
276
+ # anything else starts a new call.
277
+ #
278
+ # @param tool_calls [Array<Hash>] calls accumulated so far
279
+ # @param delta_item [Hash] unindexed tool call delta
280
+ # @return [Hash, nil] the call to merge into, or nil for a new call
281
+ def unindexed_tool_call_target(tool_calls, delta_item)
282
+ if delta_item[:id].present?
283
+ tool_calls.find { _1[:id] == delta_item[:id] }
284
+ elsif delta_item.dig(:function, :name).blank?
285
+ tool_calls.last
286
+ end
287
+ end
288
+
289
+ # Strips the fields a server repeats on every fragment (`id`, `type`,
290
+ # the function name), so merging does not concatenate them.
291
+ #
292
+ # @param hash_item [Hash] the call the fragment continues
293
+ # @param delta_item [Hash] unindexed tool call delta
294
+ # @return [Hash] the fragment to merge
295
+ def unindexed_tool_call_fragment(hash_item, delta_item)
296
+ fragment = delta_item.except(:id, :type)
297
+ if fragment[:function].is_a?(Hash) && fragment.dig(:function, :name) == hash_item.dig(:function, :name)
298
+ fragment = fragment.merge(function: fragment[:function].except(:name))
299
+ end
300
+ fragment
301
+ end
302
+
266
303
  # Recursively merges delta changes into a hash structure
267
304
  #
268
305
  # Handles complex delta merging for OpenAI's streaming API, including
@@ -285,6 +322,8 @@ module ActiveAgent
285
322
  else
286
323
  hash[key] << delta_item
287
324
  end
325
+ elsif key == :tool_calls && delta_item.is_a?(Hash) && (hash_item = unindexed_tool_call_target(hash[key], delta_item))
326
+ hash_merge_delta(hash_item, unindexed_tool_call_fragment(hash_item, delta_item))
288
327
  else
289
328
  hash[key] << delta_item
290
329
  end
@@ -181,6 +181,9 @@ module ActiveAgent
181
181
  # usage from, so it is recorded from the final event.
182
182
  record_stream_usage(api_response_event.response.usage)
183
183
 
184
+ response = api_response_event.response
185
+ self.stream_finish_reason = (response.incomplete_details&.reason || response.status).to_s
186
+
184
187
  # Completion, tool loop included, runs from stream_finished! once
185
188
  # the stream has drained, so each turn's tool calls run once.
186
189
  self.stream_completion_pending = true
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../open_ai/chat/_types"
4
+ require_relative "options"
5
+
6
+ module ActiveAgent
7
+ module Providers
8
+ module Opper
9
+ # ActiveModel type for casting and serializing Opper requests.
10
+ #
11
+ # Opper is OpenAI-compatible, so requests use the same shape as the
12
+ # OpenAI Chat API. This delegates entirely to OpenAI::Chat::RequestType.
13
+ RequestType = ActiveAgent::Providers::OpenAI::Chat::RequestType
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../open_ai/options"
4
+
5
+ module ActiveAgent
6
+ module Providers
7
+ module Opper
8
+ # Configuration options for the Opper provider.
9
+ #
10
+ # Extends OpenAI::Options, overriding the base URL to point at Opper's
11
+ # OpenAI-compatible gateway and resolving the API key from OPPER_API_KEY.
12
+ # Opper does not use organization or project identifiers.
13
+ #
14
+ # @example Basic configuration
15
+ # options = Options.new(api_key: ENV["OPPER_API_KEY"])
16
+ #
17
+ # @see https://docs.opper.ai
18
+ # @see https://platform.opper.ai Opper API Keys
19
+ class Options < ActiveAgent::Providers::OpenAI::Options
20
+ # @!attribute base_url
21
+ # @return [String] API endpoint (default: "https://api.opper.ai/v3/compat")
22
+ attribute :base_url, :string, as: "https://api.opper.ai/v3/compat"
23
+
24
+ private
25
+
26
+ def resolve_api_key(kwargs)
27
+ kwargs[:api_key] ||
28
+ kwargs[:access_token] ||
29
+ ENV["OPPER_API_KEY"]
30
+ end
31
+
32
+ # Not used as part of Opper
33
+ def resolve_organization_id(kwargs) = nil
34
+ def resolve_project_id(kwargs) = nil
35
+ end
36
+ end
37
+ end
38
+ end