actionagent 1.7.2 → 1.8.1

data/app/models/action_agent/model_pricing.rb

@@ -1,16 +1,40 @@
 # frozen_string_literal: true
 
+require "digest"
+
 module ActionAgent
   # Estimates LLM spend from token counts. The activeagent gem's telemetry
   # records tokens only; the platform layers pricing on top for the cost
-  # figures shown in Traces and Metrics.
+  # figures shown in Traces, Metrics and Evaluations.
   #
   # Rates come from RubyLLM's model registry (USD per million tokens,
-  # maintained upstream per model) when the model is known there; the static
-  # pattern table below is the fallback for aliases/self-hosted models, and
-  # a conservative blended rate covers everything else so totals stay
-  # meaningful. Costs are always presented as estimates.
+  # maintained upstream per model) when the model is known there — looked
+  # up under the provider it ran on when the caller knows it, since
+  # `gpt-5.5` through OpenRouter and `gpt-5.5` at OpenAI are two entries.
+  # The static tables below are the fallback for aliases, self-hosted
+  # models and an install without RubyLLM: exact rows (STATIC_RATES) for
+  # the current frontier models, then name patterns (PRICES), then a
+  # conservative blended rate so totals stay meaningful. Costs are always
+  # presented as estimates, and every rate says where it came from
+  # (`source`), so a figure worked out at a fallback rate can say so.
   class ModelPricing
+    # Exact rates for models the pattern table would otherwise misprice —
+    # claude-sonnet-5 is not a $3/$15 Sonnet, and the gpt-5 family is not
+    # one price — keyed by the name with its vendor prefix and date suffix
+    # removed and its dots and dashes normalised (see .normalize).
+    STATIC_RATES = {
+      "claude-sonnet-5" => [ 2.00, 10.00 ],
+      "gpt-5-5" => [ 5.00, 30.00 ],
+      "gpt-5-5-mini" => [ 0.25, 2.00 ],
+      "gpt-5-5-nano" => [ 0.05, 0.40 ],
+      "gpt-5-1" => [ 1.25, 10.00 ],
+      "gpt-5-1-mini" => [ 0.25, 2.00 ],
+      "gpt-5-1-nano" => [ 0.05, 0.40 ],
+      "gpt-5" => [ 1.25, 10.00 ],
+      "gpt-5-mini" => [ 0.25, 2.00 ],
+      "gpt-5-nano" => [ 0.05, 0.40 ]
+    }.freeze
+
     PRICES = [
       # [pattern, input $/1M, output $/1M]
       [ /gpt-4o-mini/i, 0.15, 0.60 ],
@@ -18,10 +42,15 @@ module ActionAgent
       [ /gpt-4\.1-nano/i, 0.10, 0.40 ],
       [ /gpt-4\.1-mini/i, 0.40, 1.60 ],
       [ /gpt-4\.1/i, 2.00, 8.00 ],
+      [ /gpt-5\.?5/i, 5.00, 30.00 ],
+      [ /gpt-5(\.\d+)?-nano/i, 0.05, 0.40 ],
+      [ /gpt-5(\.\d+)?-mini/i, 0.25, 2.00 ],
+      [ /gpt-5/i, 1.25, 10.00 ],
       [ /o3-mini|o4-mini/i, 1.10, 4.40 ],
       [ /claude.*(fable|mythos)/i, 10.00, 50.00 ],
       [ /claude.*haiku-?4/i, 1.00, 5.00 ],
       [ /claude.*(haiku)/i, 0.80, 4.00 ],
+      [ /claude.*sonnet-?5/i, 2.00, 10.00 ],
       [ /claude.*(sonnet)/i, 3.00, 15.00 ],
       [ /claude.*opus-(5|4-[5-9])/i, 5.00, 25.00 ],
       [ /claude.*(opus)/i, 15.00, 75.00 ],
@@ -36,45 +65,196 @@ module ActionAgent
     # Fallback blended rate for unknown models ($/1M input, $/1M output)
     DEFAULT_RATE = [ 1.00, 4.00 ].freeze
 
-    # @return [Float, nil] estimated USD cost, nil when there is nothing to price
-    def self.estimate(model:, input_tokens:, output_tokens:)
-      input = input_tokens.to_i
-      output = output_tokens.to_i
-      return nil if input.zero? && output.zero?
+    # Where a rate came from: RubyLLM's bundled catalog, a registry the host
+    # keeps itself (RubyLLM's model table or a refreshed listing), a row of
+    # STATIC_RATES or PRICES, or DEFAULT_RATE. The last two are a guess at
+    # the model's price rather than its listing.
+    SOURCES = %w[catalog remote pattern default].freeze
+    FALLBACK_SOURCES = %w[pattern default].freeze
 
-      input_rate, output_rate = rate_for(model)
-      ((input * input_rate) + (output * output_rate)) / 1_000_000.0
-    end
+    # A gateway's prefix on a model name, which names the provider rather
+    # than the model.
+    GATEWAY_PREFIXES = %w[openrouter requesty].freeze
+    # Vendor prefixes a gateway (or a host) puts before a model name, and
+    # the provider the bare name is listed under.
+    VENDOR_PROVIDERS = {
+      "openai" => "openai", "anthropic" => "anthropic", "google" => "gemini", "gemini" => "gemini",
+      "meta-llama" => nil, "mistralai" => "mistral", "mistral" => "mistral", "deepseek" => "deepseek",
+      "qwen" => nil, "x-ai" => "xai", "xai" => "xai", "cohere" => nil, "perplexity" => "perplexity"
+    }.freeze
 
-    def self.rate_for(model)
-      return DEFAULT_RATE if model.blank?
+    class << self
+      # @return [Float, nil] estimated USD cost, nil when there is nothing to price
+      def estimate(model:, input_tokens:, output_tokens:, provider: nil)
+        estimate_detailed(model: model, input_tokens: input_tokens, output_tokens: output_tokens, provider: provider)&.fetch(:cost)
+      end
 
-      registry_rate(model) || static_rate(model)
-    end
+      # The estimate with its working: `{ cost:, input_rate:, output_rate:,
+      # source: }`, rates in $ per million tokens. nil when both token
+      # counts are zero — a $0.00 is the caller's to decide on, since
+      # nothing generated and nothing recorded look alike here.
+      def estimate_detailed(model:, input_tokens:, output_tokens:, provider: nil)
+        input = input_tokens.to_i
+        output = output_tokens.to_i
+        return nil if input.zero? && output.zero?
+
+        rate = rate_detail(model, provider: provider)
+        {
+          cost: ((input * rate[:input]) + (output * rate[:output])) / 1_000_000.0,
+          input_rate: rate[:input],
+          output_rate: rate[:output],
+          source: rate[:source]
+        }
+      end
+
+      # @return [Array(Float, Float)] input and output rates in $ per million tokens
+      def rate_for(model, provider = nil)
+        rate = rate_detail(model, provider: provider)
+        [ rate[:input], rate[:output] ]
+      end
+
+      # `{ input:, output:, source: }` for a model under a provider, memoized
+      # per [provider, model]: the registry scan is not free and trace
+      # serialization asks per row.
+      def rate_detail(model, provider: nil)
+        return { input: DEFAULT_RATE[0], output: DEFAULT_RATE[1], source: "default" } if model.blank?
+
+        key = [ provider.to_s, model.to_s ]
+        @rates ||= {}
+        return @rates[key] if @rates.key?(key)
+
+        @rates[key] = registry_rate(model.to_s, provider.to_s.presence) || static_rate(model.to_s)
+      end
 
-    # Exact per-model rates from RubyLLM's registry. Lookups are memoized —
-    # the registry scan is not free and trace serialization calls this per
-    # row.
-    def self.registry_rate(model)
-      @registry_rates ||= {}
-      return @registry_rates[model] if @registry_rates.key?(model)
-
-      @registry_rates[model] = begin
-        info = RubyLLM.models.find(model.to_s)
-        tokens = info&.pricing&.text_tokens
-        if tokens&.input && tokens&.output
-          [ tokens.input, tokens.output ]
+      # Forgets memoized rates — after RubyLLM is loaded, or in tests.
+      def reset!
+        @rates = {}
+      end
+
+      # Names the rate tables in force, so a cache of figures worked out at
+      # them is invalidated when they change: the static rows, the default,
+      # and the version of the registry the install has (or none).
+      def fingerprint
+        registry = defined?(::RubyLLM) && ::RubyLLM.const_defined?(:VERSION) ? ::RubyLLM::VERSION.to_s : "none"
+        tables = [ STATIC_RATES, PRICES.map { |pattern, input, output| [ pattern.source, input, output ] }, DEFAULT_RATE, registry ]
+        Digest::SHA256.hexdigest(JSON.generate(tables))[0, 12]
+      end
+
+      # "openrouter/anthropic/claude-sonnet-4-5-20250929" → "claude-sonnet-4-5":
+      # the name without a gateway or vendor prefix, a date suffix or a
+      # colon-tagged size, with dots as dashes, so the same model under two
+      # spellings keys one row.
+      def normalize(model)
+        name = model.to_s.strip.downcase
+        name = name.split("/").last.to_s
+        name = name.sub(/-\d{8}\z/, "").sub(/-\d{4}-\d{2}-\d{2}\z/, "")
+        name.tr(".", "-")
+      end
+
+      private
+
+      # The registry's rate for the first spelling it knows: the name as
+      # given under the provider it ran on, then bare, then with a gateway
+      # or vendor prefix stripped (under the vendor's own provider), each
+      # with its dots and dashes swapped and its date suffix dropped.
+      def registry_rate(model, provider)
+        return nil unless registry_available?
+
+        lookup_candidates(model, provider).each do |id, candidate_provider|
+          info = registry_find(id, candidate_provider)
+          tokens = info&.pricing&.text_tokens
+          next unless tokens.respond_to?(:input) && tokens.input && tokens.output
+
+          return { input: tokens.input.to_f, output: tokens.output.to_f, source: registry_source }
         end
+        nil
       rescue StandardError
         nil
       end
-    end
 
-    def self.static_rate(model)
-      PRICES.each do |pattern, input_rate, output_rate|
-        return [ input_rate, output_rate ] if model.to_s.match?(pattern)
+      def registry_available?
+        defined?(::RubyLLM) && ::RubyLLM.respond_to?(:models)
+      end
+
+      # A host that keeps its own model registry (RubyLLM's model table or
+      # a refreshed listing) prices from it rather than from the gem's
+      # bundled catalog.
+      def registry_source
+        config = ::RubyLLM.respond_to?(:config) ? ::RubyLLM.config : nil
+        return "remote" if config.respond_to?(:model_registry_class) && config.model_registry_class.present?
+
+        "catalog"
+      end
+
+      # RubyLLM 2 takes the provider as a keyword; 1.x took it positionally.
+      def registry_find(id, provider)
+        models = ::RubyLLM.models
+        return models.find(id) if provider.blank?
+
+        if registry_find_keyword?(models)
+          models.find(id, provider: provider)
+        else
+          models.find(id, provider)
+        end
+      rescue StandardError
+        nil
+      end
+
+      def registry_find_keyword?(models)
+        models.method(:find).parameters.any? { |type, name| name == :provider && %i[key keyreq].include?(type) }
+      end
+
+      # [[id, provider], ...] to try in order, without repeats.
+      def lookup_candidates(model, provider)
+        name = model.strip
+        provider = provider&.downcase
+        candidates = []
+
+        head, rest = name.split("/", 2)
+        if rest.present? && GATEWAY_PREFIXES.include?(head.downcase)
+          provider ||= head.downcase
+          name = rest
+          head, rest = name.split("/", 2)
+        end
+
+        names = [ name ]
+        if rest.present? && VENDOR_PROVIDERS.key?(head.downcase)
+          names << rest
+          vendor_provider = VENDOR_PROVIDERS[head.downcase]
+        end
+
+        names.each do |candidate|
+          spellings(candidate).each do |spelling|
+            candidates << [ spelling, provider ] if provider
+            candidates << [ spelling, vendor_provider ] if vendor_provider && vendor_provider != provider && candidate == rest
+            candidates << [ spelling, nil ]
+          end
+        end
+        candidates.uniq
+      end
+
+      # The name as given, then with its date suffix dropped, each with its
+      # dots and dashes swapped: "claude-sonnet-4.5" and "claude-sonnet-4-5"
+      # are one model.
+      def spellings(name)
+        undated = name.sub(/-\d{8}\z/, "").sub(/-\d{4}-\d{2}-\d{2}\z/, "")
+        [ name, undated ].uniq.flat_map do |spelling|
+          [ spelling, spelling.gsub(/(\d)\.(\d)/, '\1-\2'), spelling.gsub(/(\d)-(\d)/, '\1.\2') ]
+        end.uniq
+      end
+
+      # An exact row for the normalised name, else the first pattern the
+      # name as given matches, else the default.
+      def static_rate(model)
+        if (exact = STATIC_RATES[normalize(model)])
+          return { input: exact[0], output: exact[1], source: "pattern" }
+        end
+
+        PRICES.each do |pattern, input_rate, output_rate|
+          return { input: input_rate, output: output_rate, source: "pattern" } if model.match?(pattern)
+        end
+        { input: DEFAULT_RATE[0], output: DEFAULT_RATE[1], source: "default" }
       end
-      DEFAULT_RATE
     end
   end
 end

data/app/models/action_agent/provider_key.rb

@@ -5,22 +5,53 @@ module ActionAgent
   # Generation runs (AgentExecutionService and the evaluation LLM judge)
   # prefer these over the platform's ENV-configured keys, so users can run
   # agents with their own OpenAI/Anthropic/OpenRouter accounts — or point
-  # ollama at their own host (e.g. a tunnel to a locally running instance).
+  # ollama at their own host: a locally running instance, a tunnel to one, or
+  # a remote/cloud server that additionally needs a Bearer API key.
   #
-  # The credential is encrypted at rest with Active Record Encryption. API
-  # keys are never rendered back to the client — only a masked hint; ollama
-  # hosts are not secret and are shown in full (see #display_hint).
+  # The credential (and the optional api_key) is encrypted at rest with
+  # Active Record Encryption. API keys are never rendered back to the client —
+  # only a masked hint; ollama hosts are not secret and are shown in full
+  # (see #display_hint).
+  #
+  # A connection credential (Claude Code) is stored the same way but is not a
+  # generation provider: no agent runs "on" it. It is handed to runtimes that
+  # need it — a checkout sandbox runs Claude Code sessions with it — through
+  # #runtime_environment. For Claude Code that is an Anthropic API key only
+  # (see CLAUDE_CODE_CREDENTIAL).
   class ProviderKey < ApplicationRecord
     # Providers that authenticate with an API key.
     KEY_PROVIDERS = %w[openai anthropic openrouter].freeze
-    # Providers addressed by host URL instead of a key.
+    # Providers addressed by host URL instead of a key (with an optional key
+    # for remote servers).
     HOST_PROVIDERS = %w[ollama].freeze
-    PROVIDERS = (KEY_PROVIDERS + HOST_PROVIDERS).freeze
+    # Tools connected with a credential, configured beside the providers
+    # (Settings -> Integrations) but never offered to the agent builder.
+    CONNECTION_PROVIDERS = %w[claude_code codex].freeze
+    PROVIDERS = (KEY_PROVIDERS + HOST_PROVIDERS + CONNECTION_PROVIDERS).freeze
+
+    # Only an Anthropic API key (sk-ant-api03-…, from the Claude Console or
+    # a supported cloud provider). Anthropic does not let third-party
+    # products collect, store or route requests through Claude.ai
+    # subscription credentials (a `claude setup-token` token, sk-ant-oat…):
+    # https://code.claude.com/docs/en/legal-and-compliance.md. A developer
+    # who wants their own subscription on their own machine uses
+    # ActionAgent.claude_code_auth = :local_login instead, where the
+    # dashboard never touches the credential.
+    CLAUDE_CODE_CREDENTIAL = /\Ask-ant-api\d{2}-[A-Za-z0-9_-]+\z/
+    CODEX_CREDENTIAL = /\Ask-(?!ant-|or-)[A-Za-z0-9_-]+\z/
+    # A Claude subscription token, as earlier versions stored. Recognized so
+    # a stored one is never handed out (see #needs_replacing?).
+    SUBSCRIPTION_TOKEN_PREFIX = "sk-ant-oat"
 
     include Ownable
     owned_by :account, :user
 
-    encrypts :credential if ActionAgent.encrypt_credentials
+    if ActionAgent.encrypt_credentials
+      encrypts :credential
+      encrypts :api_key
+    end
+
+    before_validation :normalize_host_credential, if: :host_based?
 
     validates :provider, presence: true, inclusion: { in: PROVIDERS }
     # One credential per provider per owner; which column that means
@@ -29,26 +60,133 @@ module ActionAgent
     validates :credential, presence: true, length: { maximum: 500 }
     validates :credential, format: { with: %r{\Ahttps?://\S+\z}, message: "must be an http(s):// URL" },
       if: :host_based?
+    validates :credential, format: {
+      with: CLAUDE_CODE_CREDENTIAL,
+      message: "must be an Anthropic API key (sk-ant-api…) from the Claude Console (https://platform.claude.com). " \
+        "Claude subscription tokens (`claude setup-token`, sk-ant-oat…) cannot be stored: Anthropic does not allow " \
+        "third-party apps to hold Claude.ai credentials. To use your own Claude login on this machine, set " \
+        "ActionAgent.claude_code_auth = :local_login with the :local sandbox backend instead"
+    }, if: -> { provider == "claude_code" }
+    validates :credential, format: { with: CODEX_CREDENTIAL, message: "must be an OpenAI API key (sk-…)" },
+      if: -> { provider == "codex" }
+    validates :api_key, length: { maximum: 500 }, allow_nil: true
+
+    # Deletes every Claude Code connection that still holds a Claude
+    # subscription token (see #needs_replacing?), whoever owns it. The
+    # credential is encrypted, so each is read to tell. Their owners see
+    # Claude Code as not connected, and connect an API key again.
+    #
+    # @return [Integer] how many were deleted
+    def self.purge_subscription_tokens!
+      where(provider: "claude_code").find_each.count do |key|
+        key.needs_replacing? && key.destroy!
+      end
+    end
+
+    def self.kind_of_provider(provider)
+      if HOST_PROVIDERS.include?(provider) then "host"
+      elsif CONNECTION_PROVIDERS.include?(provider) then "connection"
+      else "key"
+      end
+    end
+
+    # Ollama's OpenAI-compatible API lives under /v1. Accept the bare server
+    # address people naturally paste (http://localhost:11434, a tunnel
+    # hostname) and add the path; trailing slashes are dropped so the client
+    # can join paths cleanly. An explicit non-root path is left alone, for
+    # servers behind a reverse proxy.
+    def self.normalize_host(value)
+      host = value.to_s.strip.chomp("/")
+      return host if host.blank?
+
+      uri = URI.parse(host)
+      return host unless uri.is_a?(URI::HTTP)
+
+      uri.path = "/v1" if uri.path.blank? || uri.path == "/"
+      uri.to_s.chomp("/")
+    rescue URI::InvalidURIError
+      host
+    end
 
     def host_based?
       HOST_PROVIDERS.include?(provider)
     end
 
+    def connection?
+      CONNECTION_PROVIDERS.include?(provider)
+    end
+
+    # Only host-based providers carry an optional key (a remote Ollama behind
+    # an authenticating proxy, or Ollama Cloud).
+    def api_key?
+      host_based? && api_key.present?
+    end
+
     # Options merged into generate_with for runs owned by this key's owner,
-    # overriding the host app's config/active_agent.yml credentials.
+    # overriding the host app's config/active_agent.yml credentials. A
+    # connection credential configures no generation.
     def generation_options
-      host_based? ? { host: credential } : { access_token: credential }
+      return {} if connection?
+      return { access_token: credential } unless host_based?
+
+      { host: credential, access_token: api_key.presence }.compact
+    end
+
+    # Environment variables a runtime needs to use this credential, for the
+    # credentials that are consumed by a process rather than a provider
+    # client: Claude Code reads an API key from ANTHROPIC_API_KEY.
+    #
+    # A subscription token stored before those were refused is never handed
+    # to a process: it yields nothing, as if Claude Code were not connected,
+    # until it is replaced with an API key.
+    #
+    # @return [Hash{String => String}]
+    def runtime_environment
+      if provider == "codex"
+        return CODEX_CREDENTIAL.match?(credential.to_s) ? { "CODEX_API_KEY" => credential } : {}
+      end
+      return {} unless provider == "claude_code"
+      return {} if needs_replacing? || !CLAUDE_CODE_CREDENTIAL.match?(credential.to_s)
+
+      { "ANTHROPIC_API_KEY" => credential }
+    end
+
+    # A Claude Code connection that still holds a Claude subscription token
+    # (sk-ant-oat…), stored by an earlier version. It is no longer used and
+    # must be replaced with an API key; `bin/rails
+    # action_agent:claude_code:purge_subscription_tokens` deletes them all.
+    def needs_replacing?
+      provider == "claude_code" && credential.to_s.start_with?(SUBSCRIPTION_TOKEN_PREFIX)
     end
 
     # "sk-a…Q2z9" for keys; hosts are shown in full.
     def display_hint
       return credential if host_based?
 
-      "#{credential.first(4)}…#{credential.last(4)}"
+      mask(credential)
+    end
+
+    # Masked hint for the optional host-provider key, nil when none is set.
+    def api_key_hint
+      api_key? ? mask(api_key) : nil
+    end
+
+    # Reachability + served models for host-based providers.
+    def probe
+      OllamaHostProbe.call(host: credential, api_key: api_key)
     end
 
     private
 
+    def mask(value)
+      "#{value.first(4)}…#{value.last(4)}"
+    end
+
+    def normalize_host_credential
+      self.credential = self.class.normalize_host(credential) if credential.present?
+      self.api_key = api_key.presence&.strip
+    end
+
     def provider_unique_within_owner
       return if provider.blank?