activeagent 1.9.0 → 1.9.2

data/lib/active_agent/evals/report.rb

@@ -37,9 +37,12 @@ module ActiveAgent
       # @param release [Hash, nil] the release of the agent under evaluation,
       #   `{ "digest", "revision", "label" }`, so the report names the code
       #   it scored: in `to_h["release"]` and as a header chip.
+      # @param comparison [Hash, nil] a comparison already recorded for these
+      #   results (Comparison#to_h). A report rebuilt from a persisted run
+      #   renders the intervals that run recorded instead of resampling them.
       def initialize(results:, models:, judge: nil, instructions: nil, threshold: PASS_THRESHOLD, metadata: {},
                      tool_resolver: nil, agent_name: nil, links: {}, verdict: nil, judge_label: nil,
-                     judge_usage: nil, release: nil)
+                     judge_usage: nil, release: nil, comparison: nil)
         @results = results
         @models = models
         @judge = judge
@@ -53,6 +56,7 @@ module ActiveAgent
         @recorded_verdict = verdict.is_a?(Hash) ? verdict.to_h.stringify_keys.presence : nil
         @run_judge_usage = judge_usage.is_a?(Hash) ? judge_usage.to_h.stringify_keys.presence : nil
         @release = release.is_a?(Hash) ? release.to_h.stringify_keys.compact.presence : nil
+        @recorded_comparison = comparison.is_a?(Hash) ? comparison.to_h.deep_stringify_keys.presence : nil
       end
 
       # The release of the agent this report scored, `{ "digest",
@@ -64,7 +68,8 @@ module ActiveAgent
       end
 
       # Per model, keyed by label: scenario count, passes, errors, pass rate,
-      # mean score, mean latency, tokens, cost and fault counts. "cost" sums
+      # mean score, latency (mean, and the P50 and P95 by nearest rank,
+      # Statistics.percentile), tokens, cost and fault counts. "cost" sums
       # the replays that carried a cost — "priced" of the "scenarios", of
       # which "reported" came from the caller and "estimated" from tokens ×
       # a model rate (Result#cost_source) — and is nil when none did.
@@ -90,6 +95,8 @@ module ActiveAgent
             "avg_score" => scored.any? ? (scored.sum / scored.size).round(3) : nil,
             "avg_task_completion" => task_scores.any? ? (task_scores.sum / task_scores.size).round(3) : nil,
             "avg_duration_ms" => durations.any? ? (durations.sum.to_f / durations.size).round : nil,
+            "p50_duration_ms" => Statistics.percentile(durations, 50)&.round,
+            "p95_duration_ms" => Statistics.percentile(durations, 95)&.round,
             "input_tokens" => cohort.sum { |result| result.replay.input_tokens.to_i },
             "output_tokens" => cohort.sum { |result| result.replay.output_tokens.to_i },
             "cost" => costs.any? ? costs.sum.to_f.round(6) : nil,
@@ -267,14 +274,27 @@ module ActiveAgent
         verdict&.dig("winner")
       end
 
-      # Adds "judge_usage" and "release" only when there is one to add, so
-      # a report built the way it always was serializes exactly as before.
+      # Whether the gap between the models is real (Comparison#to_h): each
+      # model's pass rate with its interval, every pair's difference with
+      # its interval and whether it is decisive, and the conclusion. The
+      # comparison recorded for these results when one was handed in. nil
+      # for a single model.
+      def comparison
+        return nil unless comparing?
+
+        @comparison ||= @recorded_comparison || Comparison.new(results: @results, models: @models).to_h
+      end
+
+      # Adds "judge_usage" and "release" only when there is one to add, and
+      # "comparison" only when comparing models, so a single-model report
+      # built the way it always was serializes exactly as before.
       def to_h
         {
           "models" => summary_by_model,
           "criteria" => criterion_scores,
           "recommendations" => recommendations,
           "verdict" => verdict,
+          "comparison" => comparison,
           "judge" => @judge_label || @judge&.label,
           "judge_usage" => judge_usage,
           "release" => release,
@@ -298,6 +318,7 @@ module ActiveAgent
         lines.concat(total_lines)
         lines << "**Best model: #{winner}**" if winner
         lines << ""
+        lines.concat(comparison_lines)
         lines.concat(matrix_table)
         lines.concat(detail_lines)
         lines.concat(recommendation_lines)
@@ -525,22 +546,27 @@ module ActiveAgent
 
       # --- Markdown ----------------------------------------------------------
 
-      # The per-model table. A Judge column joins it when a judge spent
-      # anything, so the agent's cost and the evaluation's stay two numbers.
+      # The per-model table. Cost / 1,000 puts every model on one
+      # denominator however many of its scenarios ran. A Judge column joins
+      # it when a judge spent anything, so the agent's cost and the
+      # evaluation's stay two numbers.
       def summary_table
         judged = judge_usage.present?
-        columns = [ "Model", "Passed", "Mean score", "Mean latency", "Tokens in/out", "Cost", ("Judge" if judged), "Faults" ].compact
+        columns = [ "Model", "Passed", "Mean score", "Mean latency", "Median / P95", "Tokens in/out", "Cost", "Cost / 1,000", ("Judge" if judged), "Faults" ].compact
         header = [ "| #{columns.join(' | ')} |", "|#{columns.map { '---' }.join('|')}|" ]
         rows = summary_by_model.map do |label, stats|
           faults = stats["faults"].map { |fault, count| "#{fault.tr('_', ' ')} ×#{count}" }.join(", ")
           latency = stats["avg_duration_ms"] ? "#{stats['avg_duration_ms']} ms" : "—"
+          percentiles = stats["p50_duration_ms"] ? "#{stats['p50_duration_ms']} / #{stats['p95_duration_ms']} ms" : "—"
           cells = [
             "`#{label}`",
             Format.passes(stats["passed"], stats["scenarios"], style: :markdown),
             Format.score(stats["avg_score"]),
             latency,
+            percentiles,
             "#{stats['input_tokens']}/#{stats['output_tokens']}",
             Format.money(stats["cost"], estimated: estimated_cost?(stats)),
+            per_thousand_cell(stats),
             (judge_cell(stats) if judged),
             faults.presence || "—"
           ].compact
@@ -549,6 +575,14 @@ module ActiveAgent
         header + rows
       end
 
+      # "$3.6300": what 1,000 scenario runs cost at the model's mean cost
+      # per priced replay, marked "~" when that mean is estimated or covers
+      # only some of the replays. "—" when none was priced.
+      def per_thousand_cell(stats)
+        per = cost_per_priced(stats)
+        per ? Format.money(per * 1_000, estimated: estimated_cost?(stats)) : "—"
+      end
+
       # "~$0.0030 (3 calls)" for a model's judge spend, "—" when the judge
       # was not asked about its results.
       def judge_cell(stats)
@@ -572,6 +606,38 @@ module ActiveAgent
         lines << ""
       end
 
+      # Each model's pass rate with its interval, best first, then every
+      # pair's difference with its interval, the conclusion, and how the
+      # intervals were computed. Nothing for a single model.
+      def comparison_lines
+        return [] unless comparison
+
+        method = comparison["method"]
+        confidence = "#{(method['confidence'].to_f * 100).round}% CI"
+        lines = [ "## Comparison", "", "| Model | Pass rate | #{confidence} | Scenarios |", "|---|---|---|---|" ]
+        comparison["ranking"].each do |label|
+          stats = comparison["models"][label]
+          lines << "| `#{label}` | #{rate_text(stats['pass_rate'])} | #{interval_text(stats['interval'])} | #{stats['scenarios']} |"
+        end
+        lines += [ "", "| Pair | Difference | #{confidence} | Shared scenarios | Decisive |", "|---|---|---|---|---|" ]
+        comparison["pairs"].each do |pair|
+          a, b = pair["models"]
+          difference = pair["difference"] ? format("%+.1f points", pair["difference"]) : "—"
+          lines << "| `#{a}` vs `#{b}` | #{difference} | #{interval_text(pair['interval'])} | #{pair['scenarios']} | #{pair['decisive'] ? 'yes' : 'no'} |"
+        end
+        lines += [ "", comparison["conclusion"].to_s, "",
+                   "_#{confidence} from #{Format.delimited(method['resamples'])} resamples of whole scenarios (seed #{method['seed']})._", "" ]
+      end
+
+      def rate_text(rate)
+        rate ? "#{rate}%" : "—"
+      end
+
+      # "73.8 to 95.2", or "—" when there were too few scenarios.
+      def interval_text(interval)
+        interval ? "#{interval.first} to #{interval.last}" : "—"
+      end
+
       # The scenario matrix, with a trailing Cost column: what the scenario
       # cost across every model, and in brackets what the judge spent on it.
       def matrix_table

data/lib/active_agent/evals/report_html.rb

@@ -151,6 +151,14 @@ module ActiveAgent
         "#{format("%.#{n >= 10_000 ? 1 : 2}f", n / 1_000).sub(/\.?0+\z/, '')}s"
       end
 
+      # "4.02s / 22.2s": a model's P50 and P95 latency; "—" when no replay
+      # was timed.
+      def fmt_percentiles(stats)
+        return "—" if stats["p50_duration_ms"].nil?
+
+        "#{fmt_ms(stats['p50_duration_ms'])} / #{fmt_ms(stats['p95_duration_ms'])}"
+      end
+
       # Money reads "~$0.0243" when any part of it was estimated from tokens
       # × a model rate rather than reported (Format.money); the footer's
       # legend explains the mark once.
@@ -314,6 +322,7 @@ module ActiveAgent
           <div class="panel">
           <div class="panel-head"><span class="micro">Models</span><span class="right">judged by #{h(judged_by)}</span></div>
           #{html_comparison_table if comparing?}
+          #{html_intervals}
           #{blocks.join}
           #{verdict_row}
           </div>
@@ -335,12 +344,49 @@ module ActiveAgent
 
         <<~TABLE
           <div class="compare"><table>
-          <thead><tr><th>Model</th><th class="num">Passed</th><th class="num">Mean score</th><th class="num">Avg latency</th><th class="num" title="Average input + output tokens per scenario">Avg tokens</th><th class="num" title="Cohort spend, and per scenario">Cost</th>#{judge_head}<th class="fault">Typical fault</th></tr></thead>
+          <thead><tr><th>Model</th><th class="num">Passed</th><th class="num">Mean score</th><th class="num">Avg latency</th><th class="num" title="Median and 95th percentile latency, nearest rank">P50 / P95</th><th class="num" title="Average input + output tokens per scenario">Avg tokens</th><th class="num" title="Cohort spend, and per scenario">Cost</th>#{judge_head}<th class="fault">Typical fault</th></tr></thead>
           <tbody>#{rows.map { |label, stats| html_comparison_row(label, stats) }.join}</tbody>
           </table></div>
         TABLE
       end
 
+      # The ranked interval plot under the comparison table (Report#comparison):
+      # one row per model, best first, its pass rate a dot inside its interval
+      # on a fixed 0-100% axis, so a narrow gap is not magnified, with the
+      # figures written out beside it; then the conclusion and how the
+      # intervals were computed. Nothing for a single model. The rows and the
+      # axis share one grid (.iv-row is display: contents), so every track
+      # spans the same width and a tick sits under the dots it names.
+      def html_intervals
+        return "" unless comparison
+
+        method = comparison["method"]
+        confidence = "#{(method['confidence'].to_f * 100).round}% CI"
+        rows = comparison["ranking"].map { |label| html_interval_row(label, comparison["models"][label] || {}) }
+        ticks = [ 0, 25, 50, 75, 100 ].map { |tick| %(<span class="iv-tick" style="left:#{tick}%">#{tick}%</span>) }
+
+        <<~PLOT
+          <div class="intervals">
+          <div class="micro sm">Pass rate · #{h(confidence)}</div>
+          #{rows.join}
+          <div class="iv-row iv-axis" aria-hidden="true"><span></span><span class="iv-track">#{ticks.join}</span><span></span></div>
+          <p class="iv-conclusion">#{h(comparison['conclusion'])}</p>
+          <p class="iv-caption">#{h(confidence)} from #{h(Format.delimited(method['resamples']))} resamples of whole scenarios</p>
+          </div>
+        PLOT
+      end
+
+      def html_interval_row(label, stats)
+        rate = stats["pass_rate"]
+        low, high = stats["interval"]
+        range = high ? %(<span class="iv-range" style="left:#{low}%;width:#{(high - low).round(1)}%"></span>) : ""
+        point = rate ? %(<span class="iv-point" style="left:#{rate}%"></span>) : ""
+        figures = "<b>#{rate ? "#{rate}%" : '—'}</b> #{high ? "#{low} to #{high}" : 'no interval'} · #{h(plural(stats['scenarios'].to_i, 'scenario'))}"
+
+        %(<div class="iv-row"><span class="iv-name">#{h(short_name(model_by_label(label)))}</span>) +
+          %(<span class="iv-track">#{range}#{point}</span><span class="iv-figures">#{figures}</span></div>\n)
+      end
+
       def html_comparison_row(label, stats)
         short, provider = split_label(model_by_label(label))
         total = stats["scenarios"].to_i
@@ -362,6 +408,7 @@ module ActiveAgent
           <td class="num ratio tone-#{tone_for(ratio)}">#{h(fmt_passes(stats['passed'], total))}</td>
           <td class="num">#{h(fmt_score(stats['avg_score']))}</td>
           <td class="num">#{h(fmt_ms(stats['avg_duration_ms']))}</td>
+          <td class="num">#{h(fmt_percentiles(stats))}</td>
           <td class="num"#{tokens_title}>#{tokens_cell}</td>
           <td class="num"#{cost_title_attr(results_for(label), estimated: estimated)}>#{cost_cell}</td>
           #{judge_cell}<td class="fault">#{typical_fault_text(label, stats)}</td>
@@ -417,7 +464,7 @@ module ActiveAgent
         <<~BLOCK
           <div class="model">
           <div class="line"><span class="name">#{h(short)}</span><span class="provider">#{h(provider)}</span>#{pick}<span class="pass"><span class="bar bar-#{tone}"><span style="width:#{(ratio * 100).round}%"></span></span><span class="ratio tone-#{tone}">#{h(fmt_passes(stats['passed'], total))}</span></span></div>
-          <div class="stats-line"><span>score <b>#{h(fmt_score(stats['avg_score']))}</b></span><span>latency <b>#{h(fmt_ms(stats['avg_duration_ms']))}</b></span><span class="tok"><span class="in">in</span> #{h(fmt_k(stats['input_tokens']))} · <span class="out">out</span> #{h(fmt_k(stats['output_tokens']))}</span><span#{cost_title_attr(results_for(label), estimated: estimated)}>cost <b>#{h(fmt_cost(stats['cost'], estimated: estimated))}</b></span>#{judge_line}</div>
+          <div class="stats-line"><span>score <b>#{h(fmt_score(stats['avg_score']))}</b></span><span>latency <b>#{h(fmt_ms(stats['avg_duration_ms']))}</b></span><span>p50 / p95 <b>#{h(fmt_percentiles(stats))}</b></span><span class="tok"><span class="in">in</span> #{h(fmt_k(stats['input_tokens']))} · <span class="out">out</span> #{h(fmt_k(stats['output_tokens']))}</span><span#{cost_title_attr(results_for(label), estimated: estimated)}>cost <b>#{h(fmt_cost(stats['cost'], estimated: estimated))}</b></span>#{judge_line}</div>
           <div class="faults">#{faults_html}</div>
           </div>
         BLOCK
@@ -830,6 +877,21 @@ module ActiveAgent
         .pick-model input { position: absolute; opacity: 0; width: 0; height: 0; }
         .pick-model:has(input:checked) { border-color: var(--color-accent-ui); background: var(--color-accent-ui-tint); color: var(--color-accent-ui); }
         .clean { font-family: var(--font-mono); font-size: 11px; color: var(--color-success-text); }
+        .intervals { padding: 10px 12px; border-top: 1px solid var(--color-border-light); display: grid; grid-template-columns: minmax(120px, 200px) minmax(160px, 1fr) minmax(190px, auto); align-items: center; gap: 6px 12px; }
+        .intervals > .micro, .iv-conclusion, .iv-caption { grid-column: 1 / -1; }
+        .iv-row { display: contents; }
+        .iv-name { font-family: var(--font-mono); font-size: 12px; font-weight: 600; color: var(--color-text-primary); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+        .iv-track { position: relative; height: 14px; }
+        .iv-track::before { content: ""; position: absolute; left: 0; right: 0; top: 6px; height: 2px; background: var(--color-muted); }
+        .iv-range { position: absolute; top: 3px; height: 8px; box-sizing: border-box; border: 1px solid var(--color-info); border-radius: 999px; background: var(--color-info-soft); }
+        .iv-point { position: absolute; top: 2px; width: 10px; height: 10px; margin-left: -5px; border-radius: 50%; background: var(--color-info); }
+        .iv-figures { font-family: var(--font-mono); font-size: 11px; color: var(--color-text-secondary); white-space: nowrap; }
+        .iv-figures b { color: var(--color-text-primary); }
+        .iv-axis .iv-track::before { display: none; }
+        .iv-tick { position: absolute; top: 0; transform: translateX(-50%); font-family: var(--font-mono); font-size: 10px; color: var(--color-text-muted); }
+        .iv-conclusion { margin: 4px 0 0; font-size: 12px; line-height: 18px; color: var(--color-text-cell); }
+        .iv-caption { margin: 0; font-family: var(--font-mono); font-size: 10px; color: var(--color-text-muted); }
+        @media (max-width: 640px) { .intervals { grid-template-columns: minmax(0, 1fr); row-gap: 4px; } .iv-row + .iv-row .iv-name { margin-top: 6px; } .iv-axis { display: none; } }
         .verdict { padding: 10px 12px; border-top: 1px solid var(--color-border-light); font-size: 12px; line-height: 18px; color: var(--color-text-cell); }
         .verdict .micro { margin-right: 8px; }
         .fixes { display: grid; grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); gap: 12px; }

data/lib/active_agent/evals/runner.rb

@@ -54,10 +54,14 @@ module ActiveAgent
       # @param release [Hash, nil] the release of the agent under evaluation,
       #   `{ "digest", "revision", "label" }`, carried onto the Report so a
       #   published run names the code it scored (see Report.new)
+      # @param refine_gate [#call, nil] asked before each recommendation call,
+      #   after `judge_limit`; false skips it. A caller that scores each
+      #   scenario in its own Runner (a background job per scenario) shares one
+      #   run-wide limit this way, since `judge_limit` counts per instance.
       def initialize(scenarios:, models:, replay:, criteria: [], judge: nil, judge_task: true, available_tools: {},
                      instructions: nil, agent_name: "The agent", threshold: PASS_THRESHOLD,
                      refine_faults: DEFAULT_REFINE_FAULTS, judge_limit: DEFAULT_JUDGE_LIMIT, on_result: nil,
-                     around_evaluation: nil, require_judge_scores: false, metadata: {}, release: nil)
+                     around_evaluation: nil, require_judge_scores: false, metadata: {}, release: nil, refine_gate: nil)
         @scenarios = scenarios
         @models = models
         @replay = replay
@@ -75,6 +79,7 @@ module ActiveAgent
         @require_judge_scores = require_judge_scores
         @metadata = metadata
         @release = release
+        @refine_gate = refine_gate
         @judge_calls = 0
         @scorer = Scorer.new(criteria: criteria, judge: judge)
       end
@@ -187,6 +192,7 @@ module ActiveAgent
       def refine!(diagnosis, scenario, replay, result)
         return unless @judge && @refine_faults.include?(result.fault)
         return if @judge_calls >= @judge_limit
+        return if @refine_gate && !@refine_gate.call
 
         @judge_calls += 1
         refined = @judge.recommend(scenario: scenario, replay: replay, diagnosis: result,

data/lib/active_agent/evals/statistics.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # The summary statistics a report computes over its results.
+    module Statistics
+      module_function
+
+      # Nearest-rank percentile: the value at rank ceil(pct / 100 × n) of the
+      # values sorted ascending, so the result is always one of the values. The
+      # dashboard's Metrics view uses the same rule (ActionAgent::MetricsReport),
+      # so a model's P95 reads the same in both. nil for no values.
+      #
+      # @param values [Array<Numeric>] in any order; nils are ignored
+      # @param pct [Numeric] 0..100
+      # @return [Numeric, nil]
+      def percentile(values, pct)
+        sorted = values.compact.sort
+        return nil if sorted.empty?
+
+        rank = (sorted.size * pct / 100.0).ceil
+        sorted[rank.clamp(1, sorted.size) - 1]
+      end
+    end
+  end
+end

data/lib/active_agent/evals.rb

@@ -19,6 +19,8 @@ require_relative "evals/catalog"
 require_relative "evals/model_spec"
 require_relative "evals/replay"
 require_relative "evals/format"
+require_relative "evals/statistics"
+require_relative "evals/comparison"
 require_relative "evals/scorer"
 require_relative "evals/diagnosis"
 require_relative "evals/judge"

data/lib/active_agent/providers/anthropic_provider.rb

@@ -43,7 +43,7 @@ module ActiveAgent
       def initialize(kwargs = {})
         super
 
-        self.json_format_retry_count = kwargs[:max_retries] || ::Anthropic::Client::DEFAULT_MAX_RETRIES
+        self.json_format_retry_count = options.max_retries
       end
 
       # @todo Add support for Anthropic::BedrockClient and Anthropic::VertexClient
@@ -222,6 +222,8 @@ module ActiveAgent
             # No-Op; Wait for Function call to be complete
           when :thinking_delta, :signature_delta
             # TODO: Add with thinking rendering support
+          when :citations_delta
+            # No-Op; the citations arrive with the full block at content_block_stop
           else
             raise "Unexpected delta type: #{api_delta.type}"
           end

data/lib/active_agent/providers/open_ai/chat_provider.rb

@@ -143,6 +143,13 @@ module ActiveAgent
             # => {type: :"tool_calls.function.arguments.delta", name: "get_current_weather", index: 0, arguments: "", parsed: nil, arguments_delta: ""}
           when :"tool_calls.function.arguments.done"
             # => => {type: :"tool_calls.function.arguments.done", index: 0, name: "get_current_weather", arguments: "{\"location\":\"Boston, MA\"}", parsed: nil}
+          when :"refusal.delta"
+            # No-Op; the refusal is merged into the message from the :chunk event
+          when :"refusal.done"
+            # A refused message ends here, with no content.done
+            self.stream_completion_pending = true
+          when :"logprobs.content.delta", :"logprobs.content.done", :"logprobs.refusal.delta", :"logprobs.refusal.done"
+            # No-Op; sent when the request asks for logprobs
           else
             fail "Unexpected Response Event Type: #{api_response_event.type}"
           end

data/lib/active_agent/providers/open_ai/responses_provider.rb

@@ -15,6 +15,18 @@ module ActiveAgent
       class ResponsesProvider < Base
         include ToolChoiceClearing
 
+        # Output items of OpenAI's built-in tools (web search, file search,
+        # image generation, code interpreter, remote MCP).
+        BUILT_IN_TOOL_ITEM_TYPES = %i[
+          web_search_call file_search_call image_generation_call code_interpreter_call mcp_call mcp_list_tools
+        ].freeze
+
+        # Prefixes of the progress events those tools stream.
+        BUILT_IN_TOOL_EVENT_PREFIXES = %w[
+          response.web_search_call. response.file_search_call. response.image_generation_call.
+          response.code_interpreter_call response.mcp_call response.mcp_list_tools.
+        ].freeze
+
         # @return [Class]
         def self.options_klass
           Options
@@ -99,6 +111,7 @@ module ActiveAgent
         # - `:"response.content_part.done"` - content part completed
         # - `:"response.output_item.done"` - message, reasoning or function call completed
         # - `:"response.completed"` - response finished
+        # - `:"response.incomplete"` - response cut short (e.g. max_output_tokens)
         #
         # @param api_response_event [Hash] streaming chunk with :type key
         # @return [void]
@@ -106,6 +119,12 @@ module ActiveAgent
         def process_stream_chunk(api_response_event)
           instrument("stream_chunk.active_agent", chunk_type: api_response_event.type)
 
+          # Built-in tools report their progress with their own events; the
+          # finished tool call arrives as an output item.
+          if BUILT_IN_TOOL_EVENT_PREFIXES.any? { api_response_event.type.to_s.start_with?(_1) }
+            return
+          end
+
           case api_response_event.type
           # Response Created
           when :"response.created", :"response.in_progress"
@@ -124,12 +143,20 @@ module ActiveAgent
             message[:content] += api_response_event.delta
             broadcast_stream_update(message, api_response_event.delta)
 
+          # -> -> -> Content Text Citation (e.g. from web search)
+          when :"response.output_text.annotation.added"
+          # No-Op: The text is complete at response.output_text.done
+
           # -> -> -> Content Text Completed [Full Text]
           when :"response.output_text.done"
             message = message_stack.find { _1[:id] == api_response_event.item_id }
             message[:content] = api_response_event.text
             broadcast_stream_update(message, nil) # Don't double send content
 
+          # -> -> -> Refusal (the response still ends with response.completed)
+          when :"response.refusal.delta", :"response.refusal.done"
+          # No-Op: The message has no refusal field to stream it into
+
           # -> -> -> Content Function Call Append
           when :"response.function_call_arguments.delta", :"response.function_call_arguments.done"
           # No-Op: Wait for FC to Land
@@ -147,8 +174,13 @@ module ActiveAgent
           when :"response.output_item.done"
             process_stream_output_item_done(api_response_event)
 
-          # Response Completed
-          when :"response.completed"
+          # Response Completed, or cut short (e.g. by max_output_tokens) with
+          # the output it has so far
+          when :"response.completed", :"response.incomplete"
+            # The streamed response carries no body for resolve_prompt to read
+            # usage from, so it is recorded from the final event.
+            record_stream_usage(api_response_event.response.usage)
+
             # Completion, tool loop included, runs from stream_finished! once
             # the stream has drained, so each turn's tool calls run once.
             self.stream_completion_pending = true
@@ -174,7 +206,7 @@ module ActiveAgent
             # PATCH: API returns an empty array instead of empty string due to a bug in their serialization
             item_hash = Responses::Transforms.gem_to_hash(api_response_event.item).compact_blank
             message_stack << { content: "" }.merge(item_hash)
-          when :function_call, :reasoning
+          when :function_call, :reasoning, *BUILT_IN_TOOL_ITEM_TYPES
             # No-Op: Wait for the item to land (-> response.output_item.done)
           else
             raise "Unexpected Item Type: #{api_response_event.item.type}"
@@ -194,7 +226,8 @@ module ActiveAgent
           case api_response_event.item.type
           when :message
             # No-Op: Message Up to Date
-          when :function_call, :reasoning
+          when :function_call, :reasoning, *BUILT_IN_TOOL_ITEM_TYPES
+            # Kept like the non-streamed output, which holds every item
             item_hash = Responses::Transforms.gem_to_hash(api_response_event.item)
             message_stack << item_hash
           else