activeagent 1.9.6 → 1.9.7

data/lib/active_agent/evals/pass_thresholds.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # The bars a suite declares for a run: the score every answer must reach
+    # (the Runner's and the Report's threshold), the share of a model's
+    # answers that must pass, and the groups whose results are reported but
+    # not gated. Report#gates compares each group's pass rate against them.
+    #
+    #   suite: support_desk
+    #   pass:
+    #     threshold: 0.7
+    #     min_pass_rate: 0.8
+    #   groups:
+    #     - key: orders
+    #       pass: { min_pass_rate: 1.0 }
+    #     - key: experimental
+    #       pass: { gated: false }
+    #
+    # A group's `min_pass_rate` falls back to the suite's, and the suite's to
+    # DEFAULT_MIN_PASS_RATE; a group is gated unless it says `gated: false`.
+    # Layered documents override key by key at both levels, so a document
+    # that sets one group's bar leaves the suite's threshold and the other
+    # groups alone.
+    class PassThresholds
+      DEFAULT_MIN_PASS_RATE = 1.0
+      SUITE_KEYS = %w[threshold min_pass_rate].freeze
+      GROUP_KEYS = %w[min_pass_rate gated].freeze
+
+      # Layers each document's top-level `pass` and each group's `pass`
+      # (groups under `groups`, keyed by their `key`), later documents
+      # winning key by key. A group without a key has no scenarios to gate,
+      # so a `pass` block on one is refused.
+      #
+      # @param documents [Array<Hash>] parsed YAML documents, base first
+      def self.from_documents(documents)
+        suite = {}
+        groups = {}
+        Array(documents).each do |document|
+          document = document.to_h.deep_stringify_keys
+          suite.merge!(pass_block(document["pass"], "the suite"))
+          Array(document["groups"]).each do |group|
+            group = group.to_h.deep_stringify_keys
+            key = group["key"].to_s
+            raise ArgumentError, "\"pass\" of a group without a key gates nothing" if key.empty? && !group["pass"].nil?
+
+            groups[key] = groups.fetch(key, {}).merge(pass_block(group["pass"], "group #{key}"))
+          end
+        end
+        new(suite: suite, groups: groups)
+      end
+
+      def self.default
+        new
+      end
+
+      # A `pass` block as a string-keyed Hash; nothing declared reads as {},
+      # and anything but a mapping raises ArgumentError naming `place`.
+      def self.pass_block(value, place)
+        return {} if value.nil?
+        raise ArgumentError, "\"pass\" of #{place} must be a mapping, got #{value.inspect}" unless value.is_a?(Hash)
+
+        value.deep_stringify_keys
+      end
+
+      # `pass` as a string-keyed Hash of the `allowed` keys (SUITE_KEYS or
+      # GROUP_KEYS), each refusal naming `place` — the suite, a group, or
+      # whatever a caller holds a block for: a bar is a number from 0 to 1,
+      # kept as a Float so `1` and `1.0` read, serialize and hash the same,
+      # and `gated` is true or false.
+      def self.validate(pass, allowed, place)
+        pass_block(pass, place).to_h do |key, value|
+          raise ArgumentError, "unknown key #{key.inspect} in \"pass\" of #{place}; expected one of #{allowed.join(', ')}" unless allowed.include?(key)
+
+          if key == "gated"
+            raise ArgumentError, "\"gated\" in \"pass\" of #{place} must be true or false, got #{value.inspect}" unless [ true, false ].include?(value)
+
+            [ key, value ]
+          elsif value.is_a?(Numeric) && value.finite? && value.between?(0, 1)
+            [ key, value.to_f ]
+          else
+            raise ArgumentError, "#{key.inspect} in \"pass\" of #{place} must be a number from 0 to 1, got #{value.inspect}"
+          end
+        end
+      end
+
+      # @param suite [Hash] the suite's bars, `threshold` and `min_pass_rate`
+      # @param groups [Hash{String=>Hash}] each group's bars by group key,
+      #   `min_pass_rate` and `gated`
+      def initialize(suite: {}, groups: {})
+        @suite = self.class.validate(suite, SUITE_KEYS, "the suite")
+        @groups = groups.to_h.deep_stringify_keys.filter_map do |key, pass|
+          declared = self.class.validate(pass, GROUP_KEYS, "group #{key}")
+          [ key.to_s, declared ] if declared.any?
+        end.to_h
+      end
+
+      # The score every answer must reach, 0..1.
+      def threshold
+        (@suite["threshold"] || PASS_THRESHOLD).to_f
+      end
+
+      # The share of a model's answers that must pass, 0..1: the group's own
+      # bar when it declared one, else the suite's, else DEFAULT_MIN_PASS_RATE.
+      def min_pass_rate(group = nil)
+        own = @groups.dig(group.to_s, "min_pass_rate") unless group.nil?
+        (own || @suite["min_pass_rate"] || DEFAULT_MIN_PASS_RATE).to_f
+      end
+
+      # False only for a group that declared `gated: false`; nil and unknown
+      # groups are gated.
+      def gated?(group)
+        return true if group.nil?
+
+        @groups.dig(group.to_s, "gated") != false
+      end
+
+      # The bars in effect, with each group's declared keys only.
+      def to_h
+        {
+          "threshold" => threshold,
+          "min_pass_rate" => min_pass_rate,
+          "groups" => @groups.transform_values(&:dup)
+        }
+      end
+
+      def ==(other)
+        other.is_a?(PassThresholds) && to_h == other.to_h
+      end
+      alias eql? ==
+
+      def hash
+        to_h.hash
+      end
+    end
+  end
+end

data/lib/active_agent/evals/patterns.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # Regexes for a figure an answer is expected to state, built from the data
+    # the scenario was seeded with: a count ("3 of 4 tickets") or a share of
+    # a total ("75%"). Each is a String that drops into a suite's
+    # `expect.contains` / `not_contains` or a `contains` criterion, which
+    # Scorer.matches_pattern? tries as a substring and then as a regex:
+    #
+    #   "expect" => { "contains" => [ Patterns.number(3), Patterns.share(3, 4) ] }
+    module Patterns
+      MONTHS = %w[january february march april may june july august september october november december].freeze
+
+      # `text` as a pattern that matches it in any case, each letter a
+      # character class: any_case("Oct.") is "[Oo][Cc][Tt]\\.".
+      def self.any_case(text)
+        text.chars.map { |char| char.match?(/[a-z]/i) ? "[#{char.upcase}#{char.downcase}]" : Regexp.escape(char) }.join
+      end
+      private_class_method :any_case
+
+      # A day of the month written beside its month, in any case: "Oct 12",
+      # "Sept. 12" and "October 12" before the figure, "12 Oct" and "12th of
+      # October" after it. A lookbehind takes alternatives of one length only,
+      # so the spellings before the figure are one lookbehind per length, and
+      # each letter in it is a character class rather than `(?i:...)`: the
+      # dashes make the pattern UTF-8, where case folding lets "st" match the
+      # one-character ligature "st", and Ruby 3.2 refuses a lookbehind whose
+      # length that leaves open. "may" is left out after the figure, where it
+      # is usually the verb.
+      MONTH_BEFORE = MONTHS.flat_map { |month| [ month, month[0, 3], "#{month[0, 3]}." ] }.push("sept", "sept.").uniq
+        .group_by(&:length).sort.map { |_length, spellings| "(?<!#{spellings.map { |spelling| "\\b#{any_case("#{spelling} ")}" }.join('|')})" }
+        .join.freeze
+      MONTH_AFTER = (MONTHS - %w[may]).map { |month| "#{month[0, 3]}(?:#{Regexp.escape(month[3..])})?" }.push("sept").join("|").freeze
+
+      module_function
+
+      # The integer `count` as a figure on its own: number(10) matches
+      # "10 sessions", "(10)" and "10." at a sentence end, not "2026-10-01",
+      # "10/1", "10:00", "1-10" or "1–10", "110", "10.5", "10,000", "10%",
+      # "Oct 10" or "10 October". A ratio such as "10/22" does not match
+      # either, because "10/12" reads as a date as often as a count; expect
+      # "10 of 22". `count` is read as a whole number from its text, so 3 and
+      # "3" work and 1.5, 10.0, "a" or a negative count raise ArgumentError.
+      def number(count)
+        count = whole_number(count, "count")
+        "(?<![\\d/:.,\\-\u2013\u2014])#{MONTH_BEFORE}\\b#{count}\\b(?![/:\\-\u2013\u2014])(?![.,]\\d)(?! ?%)" \
+          "(?!(?i:(?:st|nd|rd|th)?,? (?:of )?(?:#{MONTH_AFTER})\\b))"
+      end
+
+      # `count` of `total` as a percentage, whole or to one or two decimals,
+      # with an optional space before the sign: share(3, 4) matches "75%",
+      # "75 %", "75.0%" and "75.00%", not "175%", "7.5%", "75.5%" or "0.75".
+      # The whole percent rounds half up like Format.whole and the decimals
+      # like Float#round, so share(1, 8) matches "13%", "12.5%" and "12.50%",
+      # and share(12, 22) matches "55%", "54.5%" and "54.55%". Both arguments are read like
+      # `number`'s; a total below one and a count above the total raise
+      # ArgumentError.
+      def share(count, total)
+        count = whole_number(count, "count")
+        total = whole_number(total, "total", minimum: 1)
+        raise ArgumentError, "count must not exceed total" if count > total
+
+        percent = count * 100.0 / total
+        decimals = [ 2, 1 ].map { |places| Regexp.escape(format("%.#{places}f", percent.round(places))) }
+        "(?<![\\d.,])\\b(?:#{decimals.join('|')}|#{Format.whole(percent)}) ?%"
+      end
+
+      # `value` as an Integer of at least `minimum`, read from its text so
+      # that 1.5 is rejected rather than truncated: 3 and "3" → 3; 1.5, "a"
+      # and a value below `minimum` raise ArgumentError.
+      def whole_number(value, name, minimum: 0)
+        number = Integer(value.to_s, 10)
+        raise ArgumentError, "#{name} must be #{minimum.positive? ? 'positive' : 'zero or positive'}" if number < minimum
+
+        number
+      end
+    end
+  end
+end

data/lib/active_agent/evals/publisher.rb

@@ -20,12 +20,21 @@ module ActiveAgent
     # ArgumentError before anything is sent.
     class Publisher
       DEFAULT_ENDPOINT = "https://api.activeagents.ai/v1/evaluations"
+      # The hosted dashboard people sign in to; the hosted collector answers
+      # on the API host, where a dashboard session does not reach.
+      HOSTED_DASHBOARD_URL = "https://activeagents.ai"
       MAX_BYTES = 2 * 1024 * 1024
       DETAIL_LIMIT = 200
 
       # @return [String] the collector URL reports go to
       attr_reader :endpoint
 
+      # @return [String] the address a receipt's "url" path resolves against: the
+      #   dashboard_url given, or by default the endpoint's origin (its scheme,
+      #   host and any non-default port, with no path), except that the hosted
+      #   API host resolves to HOSTED_DASHBOARD_URL
+      attr_reader :dashboard_url
+
       # Raised for every failed delivery. Only a network failure keeps the
       # underlying error as its +cause+, so neither the response nor the
       # report reaches a log through the exception chain.
@@ -68,7 +77,14 @@ module ActiveAgent
       # The key is sent as a bearer token and filtered from the collector's
       # explanation, so it is limited to visible ASCII: the sanitizer in
       # +collector_detail+ never alters it, and an echo of it always matches.
-      def initialize(api_key:, endpoint: DEFAULT_ENDPOINT, timeout: 10, open_timeout: 10)
+      #
+      # @param dashboard_url [String, nil] the address people open the dashboard at,
+      #   where a receipt's "url" path resolves; by default the endpoint's origin,
+      #   or HOSTED_DASHBOARD_URL for the hosted collector.
+      #   The receipt's path, which carries the engine's mount, replaces any path on it
+      # @raise [ArgumentError] "Evaluation dashboard_url is not an http(s) URL" when it is anything else, or
+      #   "Evaluation dashboard_url must be an http(s) URL without credentials, query or fragment" when it carries one of those
+      def initialize(api_key:, endpoint: DEFAULT_ENDPOINT, dashboard_url: nil, timeout: 10, open_timeout: 10)
         @uri = URI.parse(endpoint.to_s)
         unless @uri.is_a?(URI::HTTP) && @uri.host && !@uri.userinfo && !@uri.query && !@uri.fragment
           raise ArgumentError, "Evaluation endpoint must be an HTTP(S) URL without credentials, query or fragment"
@@ -78,6 +94,8 @@ module ActiveAgent
         end
         @endpoint = @uri.to_s
 
+        @dashboard_url = dashboard_url.nil? ? default_dashboard_url(@uri) : checked_dashboard_url(dashboard_url)
+
         @api_key = api_key.to_s.strip
         raise ArgumentError, "Evaluation API key is required" if @api_key.empty?
         unless @api_key.match?(/\A[\x21-\x7E]+\z/)
@@ -93,9 +111,42 @@ module ActiveAgent
         raise ArgumentError, "Evaluation endpoint is not a valid URL"
       end
 
-      # report may be a Report or its saved JSON hash. Full prompts, answers and
-      # tool results are included; applications should make publication opt-in.
-      def call(report:, run_id:, source:, agent_name:, suite:)
+      # Returns a publisher built from +env+: ACTIVEAGENTS_API_KEY is the key,
+      # ACTIVEAGENTS_EVALUATIONS_ENDPOINT the collector (DEFAULT_ENDPOINT when
+      # blank) and ACTIVEAGENTS_DASHBOARD_URL the dashboard a receipt's "url"
+      # path resolves against (the endpoint's origin when blank), so
+      # publishing is one line:
+      #
+      #   Publisher.from_env.call(report: report, source: "support-app", agent_name: "SupportBot", suite: "support_desk")
+      #
+      # @param env [#[]] the environment to read, ENV by default
+      # @raise [ArgumentError] "ACTIVEAGENTS_API_KEY is not set" for a blank key, and what +new+ raises for the other two
+      # @return [Publisher]
+      def self.from_env(env = ENV)
+        api_key = env["ACTIVEAGENTS_API_KEY"].to_s.strip
+        raise ArgumentError, "ACTIVEAGENTS_API_KEY is not set" if api_key.empty?
+
+        endpoint = env["ACTIVEAGENTS_EVALUATIONS_ENDPOINT"].to_s.strip
+        dashboard_url = env["ACTIVEAGENTS_DASHBOARD_URL"].to_s.strip
+        new(api_key: api_key, endpoint: endpoint.empty? ? DEFAULT_ENDPOINT : endpoint, dashboard_url: dashboard_url.empty? ? nil : dashboard_url)
+      end
+
+      # Returns the collector's receipt for the completed +report+, a Report
+      # or its saved JSON hash. Full prompts, answers and tool results are
+      # included; applications should make publication opt-in.
+      #
+      # The receipt's "url" is the run's dashboard page, made absolute: a path
+      # beginning with "/" is joined onto +dashboard_url+. An absolute URL, a
+      # url that is neither (a relative reference without a leading slash, a
+      # network-path reference, or a path that is not a valid URI) and a
+      # receipt without one are left as the collector sent them.
+      #
+      # @param run_id [String, nil] the run's identity, kept across retries; by default
+      #   the one the report's metadata records under "run_id" or :run_id, which
+      #   Correlation#with_run puts there
+      # @raise [ArgumentError] when no run_id is given and the report's metadata records none as a nonempty string
+      def call(report:, run_id: nil, source:, agent_name:, suite:)
+        run_id = resolve_run_id(report, run_id)
         envelope = self.class.envelope(report: report, run_id: run_id, source: source, agent_name: agent_name, suite: suite)
         publish(envelope, accepted: %w[complete], receipt: "completion receipt")
       end
@@ -104,13 +155,18 @@ module ActiveAgent
       # which the collector shows as a running run under +run_id+. Each call
       # replaces the results the previous one sent; +call+ with the finished
       # report under the same run_id completes the run. The receipt's status
-      # is "running", or "complete" when the run already finished.
+      # is "running", or "complete" when the run already finished, and its
+      # "url" is made absolute as +call+ makes it.
       #
       # Raises Error, without sending the report, when the collector does not
       # list "progress" among its capabilities (see +supports_progress?+): a
       # collector that predates progress would store the partial report as
       # complete.
-      def progress(report:, run_id:, source:, agent_name:, suite:)
+      #
+      # @param run_id [String, nil] as for +call+: by default the one the report's metadata records
+      # @raise [ArgumentError] when no run_id is given and the report's metadata records none as a nonempty string
+      def progress(report:, run_id: nil, source:, agent_name:, suite:)
+        run_id = resolve_run_id(report, run_id)
         unless supports_progress?
           raise Error, "Evaluation collector does not accept progress; publish the finished report with call"
         end
@@ -186,8 +242,54 @@ module ActiveAgent
 
       private
 
+      # The run_id a delivery goes under: +run_id+ when given, otherwise the
+      # one the report's metadata records under a String or Symbol key, which
+      # Correlation#with_run puts there. Raises ArgumentError for a report
+      # that is neither a Report nor a hash, as +envelope+ does, and when
+      # neither +run_id+ nor the metadata names a nonempty string.
+      def resolve_run_id(report, run_id)
+        return run_id unless run_id.nil?
+
+        hash = report.to_h if report.respond_to?(:to_h) && !report.nil? && !report.is_a?(Array)
+        raise ArgumentError, "report must be a Report or its saved JSON hash" unless hash.is_a?(Hash)
+
+        metadata = hash["metadata"]
+        recorded = metadata["run_id"] || metadata[:run_id] if metadata.is_a?(Hash)
+        return recorded if recorded.is_a?(String) && !recorded.strip.empty?
+
+        raise ArgumentError, "run_id must be given, or the report's metadata must carry one (Correlation#with_run records it)"
+      end
+
+      # The hosted collector's API host serves no dashboard session, so its
+      # receipts resolve against the hosted dashboard; any other collector is
+      # its own dashboard.
+      def default_dashboard_url(uri)
+        uri.hostname == URI.parse(DEFAULT_ENDPOINT).hostname ? HOSTED_DASHBOARD_URL : origin(uri)
+      end
+
+      # Returns the dashboard_url given as a String, raising ArgumentError
+      # unless it is an http(s) URL with a host and without credentials, a
+      # query or a fragment.
+      def checked_dashboard_url(value)
+        uri = URI.parse(value.to_s)
+        raise ArgumentError, "Evaluation dashboard_url is not an http(s) URL" unless uri.is_a?(URI::HTTP) && !uri.host.to_s.empty?
+        if uri.userinfo || uri.query || uri.fragment
+          raise ArgumentError, "Evaluation dashboard_url must be an http(s) URL without credentials, query or fragment"
+        end
+
+        value.to_s
+      rescue URI::InvalidURIError
+        raise ArgumentError, "Evaluation dashboard_url is not an http(s) URL"
+      end
+
+      # The origin of +uri+: its scheme, host and any non-default port, with no path.
+      def origin(uri)
+        uri.dup.tap { |without_path| without_path.path = "" }.to_s
+      end
+
       # Delivers +envelope+ and returns the collector's receipt, which must name
-      # the envelope's run_id, a stored run, and one of the +accepted+ statuses.
+      # the envelope's run_id, a stored run, and one of the +accepted+ statuses,
+      # with a "url" path made absolute on the dashboard.
       def publish(envelope, accepted:, receipt:)
         body = encode(envelope)
         raise Error, "Evaluation report exceeds the 2 MiB delivery limit; publish a smaller selection" if body.bytesize > MAX_BYTES
@@ -201,10 +303,24 @@ module ActiveAgent
                  answer["id"] && answer["evaluation_id"]
             raise Error.new("Evaluation collector returned an invalid #{receipt}; retain the report and run_id for retry", retryable: true)
           end
-          answer
+          resolve_url(answer)
         end
       end
 
+      # Returns +receipt+ with a "url" that is a path beginning with "/" joined
+      # onto +dashboard_url+. Any other url (an absolute one, a relative
+      # reference without a leading slash, a network-path reference, a path
+      # that is not a valid URI) and a receipt without one are left as the
+      # collector sent them.
+      def resolve_url(receipt)
+        url = receipt["url"]
+        return receipt unless url.is_a?(String) && url.start_with?("/") && !url.start_with?("//")
+
+        receipt.merge("url" => URI.join(@dashboard_url, url).to_s)
+      rescue URI::InvalidURIError
+        receipt
+      end
+
       # The string entries of a JSON object body's +capabilities+ array, or none.
       def collector_capabilities(body)
         parsed = JSON.parse(body.to_s)