active_harness 0.2.6 → 0.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c32abcde2f732633681db8c314c0066d668a6e90818d0bb10324f7c4fcb26f20
-  data.tar.gz: 32081332c6997ef6bc1296008b1841be30c333afa7ffd8d1701af1a7f182d718
+  metadata.gz: d7139860d4aa461b1d86eca144c4929d4bf362192099498eadf61401b85acf8d
+  data.tar.gz: 3f00f021d6fd660748767349fb6a00ba9187c6baf507d18b01ac668c52c67904
 SHA512:
-  metadata.gz: 469e75529882ee17fb2610aad11823fab222cd3c060cb6a583990e1b22c4d9b8d1af5173b4158ef8ded2facb11b0ad6cb3fd9101be213e7327ca464106b720b6
-  data.tar.gz: 291624ff7f7765ae5af466d8417cee75e66596b604afefc01263771c091c738f5739dd8ead25712075e4e1a32f28252c475b62ae0942e02ce62d2eb483f42eff
+  metadata.gz: 1c335d3ecce967dc10424ef0cbe15922561627539e7cbf5eb2a34ad7e672c964bd7fd796ed129229a41a7f32992c22241353e16b8e314aec7cd71878ea86148d
+  data.tar.gz: 930e5ad7ad85e5e269772e90c45453fb6672dc2f8752c3d546f9888d3d2c8cd2020a24190f7391f8217bf1c8e24cde67945788870c3b1d1ec60e01747c0d2698

data/lib/active_harness/agent/models.rb

@@ -134,8 +134,15 @@ module ActiveHarness
       @models = []
     end
 
-    def use(provider:, model:, temperature: nil)
-      @models << { provider: provider, model: model, temperature: temperature }.compact
+    def use(provider:, model:, temperature: nil, name: nil, retry_attempts: nil, retry_delay: nil)
+      @models << {
+        provider:       provider,
+        model:          model,
+        temperature:    temperature,
+        name:           name,
+        retry_attempts: retry_attempts,
+        retry_delay:    retry_delay
+      }.compact
     end
 
     alias fallback use

data/lib/active_harness/agent/providers.rb

@@ -1,36 +1,54 @@
 module ActiveHarness
   class Agent
-    # Errors that allow retrying the next model in the chain
+    # Errors that allow retrying the next model in the chain.
+    # InvalidRequestError is included here so that a bad model name (or any
+    # per-model request failure) does not abort the entire chain — the next
+    # fallback model will be attempted instead.
     RETRYABLE_ERRORS = [
       Errors::TimeoutError,
       Errors::RateLimitError,
       Errors::ServerError,
-      Errors::ProviderUnavailableError
+      Errors::ProviderUnavailableError,
+      Errors::InvalidRequestError
     ].freeze
 
-    # Errors that abort the entire chain immediately
+    # Errors that abort the entire chain immediately.
+    # InvalidApiKeyError  — the key is wrong for every model, retrying is pointless.
+    # SafetyBlockedError  — the input itself is blocked; a different model won't help.
     STOP_ERRORS = [
-      Errors::InvalidRequestError,
       Errors::InvalidApiKeyError,
       Errors::SafetyBlockedError
     ].freeze
 
     PROVIDERS = {
-      openai:     -> { Providers::OpenAI.new },
-      openrouter: -> { Providers::OpenRouter.new },
-      groq:       -> { Providers::Groq.new },
-      gemini:     -> { Providers::Gemini.new },
-      anthropic:  -> { Providers::Anthropic.new }
+      openai:      -> { Providers::OpenAI.new },
+      openrouter:  -> { Providers::OpenRouter.new },
+      groq:        -> { Providers::Groq.new },
+      gemini:      -> { Providers::Gemini.new },
+      anthropic:   -> { Providers::Anthropic.new },
+      xai:         -> { Providers::XAI.new },
+      deepseek:    -> { Providers::DeepSeek.new },
+      mistral:     -> { Providers::Mistral.new },
+      ollama:      -> { Providers::Ollama.new },
+      perplexity:  -> { Providers::Perplexity.new },
+      gpustack:    -> { Providers::GPUStack.new },
+      azure:       -> { Providers::Azure.new },
+      bedrock:     -> { Providers::Bedrock.new },
+      vertexai:    -> { Providers::VertexAI.new },
+      custom:      -> { Providers::Custom.new }
     }.freeze
 
     private
 
     def attempt_model(entry, system_prompt)
+      return attempt_via_ruby_llm(entry, system_prompt) if @config[:ruby_llm_backend]
+
       provider = resolve_provider(entry[:provider])
       messages = build_messages(system_prompt, @input)
       opts = { model: entry[:model], messages: messages }
       opts[:temperature] = entry[:temperature] if entry[:temperature]
       opts[:stream]      = @stream             if @stream
+      opts[:name]        = entry[:name]        if entry[:name]
       provider.call(**opts)
     end
 

data/lib/active_harness/agent/ruby_llm_backend.rb

@@ -0,0 +1,90 @@
+module ActiveHarness
+  class Agent
+    # -------------------------------------------------------------------------
+    # RubyLLM backend DSL
+    #
+    # Allows an agent to delegate HTTP calls to the `ruby_llm` gem instead of
+    # ActiveHarness's built-in Net::HTTP providers.
+    #
+    # Usage:
+    #
+    #   ruby_llm_backend do |params|
+    #     RubyLLM.chat(
+    #       model:               params.model,
+    #       provider:            params.provider,
+    #       assume_model_exists: true
+    #     ).tap { |c| c.with_temperature(params.temperature) if params.temperature }
+    #   end
+    #
+    # The block receives a BackendParams struct and must return a RubyLLM::Chat.
+    # ActiveHarness calls chat.ask(@input) and maps the result to its Result format.
+    #
+    # All existing features work unchanged:
+    #   - model do / use / fallback  (order of attempts)
+    #   - retry_attempts / retry_delay  (per-model retry policy)
+    #   - fallback chain on error
+    #   - hooks (:setup, :before_call, :retry, :failure, …)
+    #   - streaming via stream: lambda
+    # -------------------------------------------------------------------------
+
+    # Passed to the ruby_llm_backend block for each model attempt.
+    BackendParams = Struct.new(:model, :provider, :temperature, keyword_init: true)
+
+    class << self
+      # Define the RubyLLM backend block for this agent class.
+      def ruby_llm_backend(&block)
+        agent_config[:ruby_llm_backend] = block
+      end
+    end
+
+    private
+
+    # Called from attempt_model when ruby_llm_backend is configured.
+    def attempt_via_ruby_llm(entry, system_prompt)
+      require "ruby_llm"
+
+      backend = @config[:ruby_llm_backend]
+
+      params = BackendParams.new(
+        model:       entry[:model],
+        provider:    entry[:provider].to_s,
+        temperature: entry[:temperature]
+      )
+
+      chat = backend.call(params)
+      chat.with_instructions(system_prompt) if system_prompt
+
+      if @stream
+        response = chat.ask(@input) { |chunk| @stream.call(chunk.content) if chunk.content }
+      else
+        response = chat.ask(@input)
+      end
+
+      { content: response.content, usage: ruby_llm_usage(response) }
+    rescue ::RubyLLM::UnauthorizedError => e
+      raise Errors::InvalidApiKeyError, e.message
+    rescue ::RubyLLM::RateLimitError, ::RubyLLM::OverloadedError => e
+      raise Errors::RateLimitError, e.message
+    rescue ::RubyLLM::ServerError, ::RubyLLM::ServiceUnavailableError => e
+      raise Errors::ServerError, e.message
+    rescue ::RubyLLM::BadRequestError, ::RubyLLM::ContextLengthExceededError => e
+      raise Errors::InvalidRequestError, e.message
+    rescue ::RubyLLM::Error => e
+      raise Errors::ProviderError, e.message
+    rescue LoadError
+      raise Errors::ProviderUnavailableError,
+            "The `ruby_llm` gem is required. Add `gem \"ruby_llm\"` to your Gemfile."
+    end
+
+    def ruby_llm_usage(response)
+      t = response.tokens
+      return nil unless t
+
+      {
+        input_tokens:  t.input,
+        output_tokens: t.output,
+        total_tokens:  (t.input.to_i + t.output.to_i)
+      }.compact
+    end
+  end
+end

data/lib/active_harness/agent.rb

@@ -61,9 +61,15 @@ module ActiveHarness
       run_hook(:before_call)
       attempts = []
 
+      cfg = ActiveHarness.config
+
       model_list.each do |entry|
+        retry_policy = Http::RetryPolicy.new(
+          max_attempts: entry[:retry_attempts] || cfg.retry_default_attempts,
+          base_delay:   entry[:retry_delay]    || cfg.retry_default_delay
+        )
         t0       = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-        response = attempt_model(entry, @system_prompt)
+        response = retry_policy.run { attempt_model(entry, @system_prompt) }
         elapsed  = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0).round(3)
         result   = build_result(response, entry, attempts, elapsed)
         save_to_memory(result)
@@ -139,4 +145,5 @@ require_relative "agent/hooks"
 require_relative "agent/models"
 require_relative "agent/providers"
 require_relative "agent/output_parser"
+require_relative "agent/ruby_llm_backend"
 

data/lib/active_harness/configuration.rb

@@ -0,0 +1,173 @@
+module ActiveHarness
+  # Central configuration object.
+  #
+  # Usage in config/initializers/active_harness.rb:
+  #
+  #   ActiveHarness.configure do |config|
+  #     config.openai_api_key  = ENV["OPENAI_API_KEY"]
+  #     config.openai_api_url  = "https://api.openai.com/v1/chat/completions"
+  #     # ...
+  #   end
+  #
+  # If a value is not explicitly set, it is read from the corresponding
+  # environment variable so all existing ENV-based setups keep working.
+  class Configuration
+    # -------------------------------------------------------------------------
+    # Global
+    # -------------------------------------------------------------------------
+    attr_accessor :request_timeout
+
+    # Retry policy for a single model (exponential backoff).
+    # Set retry_default_attempts to 1 to disable retries entirely.
+    # Per-model values can be set via retry_attempts: / retry_delay: in the DSL.
+    attr_accessor :retry_default_attempts
+    attr_accessor :retry_default_delay
+
+    # -------------------------------------------------------------------------
+    # OpenAI
+    # -------------------------------------------------------------------------
+    attr_accessor :openai_api_key
+    attr_accessor :openai_api_url
+
+    # -------------------------------------------------------------------------
+    # Anthropic
+    # -------------------------------------------------------------------------
+    attr_accessor :anthropic_api_key
+    attr_accessor :anthropic_api_url
+
+    # -------------------------------------------------------------------------
+    # Google Gemini (OpenAI-compatible REST endpoint)
+    # -------------------------------------------------------------------------
+    attr_accessor :gemini_api_key
+    attr_accessor :gemini_api_url
+
+    # -------------------------------------------------------------------------
+    # Groq
+    # -------------------------------------------------------------------------
+    attr_accessor :groq_api_key
+    attr_accessor :groq_api_url
+
+    # -------------------------------------------------------------------------
+    # OpenRouter
+    # -------------------------------------------------------------------------
+    attr_accessor :openrouter_api_key
+    attr_accessor :openrouter_api_url
+    attr_accessor :openrouter_http_referer
+
+    # -------------------------------------------------------------------------
+    # xAI (Grok)
+    # -------------------------------------------------------------------------
+    attr_accessor :xai_api_key
+    attr_accessor :xai_api_url
+
+    # -------------------------------------------------------------------------
+    # DeepSeek
+    # -------------------------------------------------------------------------
+    attr_accessor :deepseek_api_key
+    attr_accessor :deepseek_api_url
+
+    # -------------------------------------------------------------------------
+    # Mistral
+    # -------------------------------------------------------------------------
+    attr_accessor :mistral_api_key
+    attr_accessor :mistral_api_url
+
+    # -------------------------------------------------------------------------
+    # Ollama (local — key is optional)
+    # -------------------------------------------------------------------------
+    attr_accessor :ollama_api_key
+    attr_accessor :ollama_api_base
+
+    # -------------------------------------------------------------------------
+    # Perplexity
+    # -------------------------------------------------------------------------
+    attr_accessor :perplexity_api_key
+    attr_accessor :perplexity_api_url
+
+    # -------------------------------------------------------------------------
+    # GPUStack (self-hosted — key is optional)
+    # -------------------------------------------------------------------------
+    attr_accessor :gpustack_api_key
+    attr_accessor :gpustack_api_base
+
+    # -------------------------------------------------------------------------
+    # Azure OpenAI Service
+    # -------------------------------------------------------------------------
+    attr_accessor :azure_api_key        # api-key header (preferred)
+    attr_accessor :azure_ai_auth_token  # Bearer token (alternative to api-key)
+    attr_accessor :azure_api_base       # e.g. "https://my-resource.openai.azure.com"
+    attr_accessor :azure_api_version    # e.g. "2024-05-01-preview"
+
+    # -------------------------------------------------------------------------
+    # Custom providers
+    #
+    # Register any OpenAI-compatible endpoint under an arbitrary name:
+    #
+    #   ActiveHarness.configure do |config|
+    #     config.custom["MyLocal"]["url"]     = "http://localhost:8080/v1/chat/completions"
+    #     config.custom["MyLocal"]["api_key"] = ENV["MYLOCAL_API_KEY"]  # omit if no auth
+    #
+    #     config.custom["SecondProvider"]["url"]     = "https://second.example.com/v1/chat/completions"
+    #     config.custom["SecondProvider"]["api_key"] = ENV["SECOND_API_KEY"]
+    #   end
+    #
+    # Use in an agent:
+    #   model do
+    #     use      provider: :custom, name: "MyLocal",       model: "llama3.2"
+    #     fallback provider: :custom, name: "SecondProvider", model: "mixtral"
+    #   end
+    # -------------------------------------------------------------------------
+    def custom
+      @custom ||= Hash.new { |h, k| h[k] = {} }
+    end
+
+    # -------------------------------------------------------------------------
+    # Defaults — all keys fall back to the corresponding ENV variable so that
+    # existing ENV-based setups keep working without any changes.
+    # -------------------------------------------------------------------------
+    def initialize
+      @request_timeout    = 10
+      @retry_default_attempts = 3
+      @retry_default_delay   = 1.0
+
+      @openai_api_key  = ENV["OPENAI_API_KEY"]
+      @openai_api_url  = "https://api.openai.com/v1/chat/completions"
+
+      @anthropic_api_key = ENV["ANTHROPIC_API_KEY"]
+      @anthropic_api_url = "https://api.anthropic.com/v1/messages"
+
+      @gemini_api_key = ENV["GEMINI_API_KEY"]
+      @gemini_api_url = "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions"
+
+      @groq_api_key = ENV["GROQ_API_KEY"]
+      @groq_api_url = "https://api.groq.com/openai/v1/chat/completions"
+
+      @openrouter_api_key      = ENV["OPENROUTER_API_KEY"]
+      @openrouter_api_url      = "https://openrouter.ai/api/v1/chat/completions"
+      @openrouter_http_referer = "https://github.com/the-teacher/ActiveHarness"
+
+      @xai_api_key = ENV["XAI_API_KEY"]
+      @xai_api_url = "https://api.x.ai/v1/chat/completions"
+
+      @deepseek_api_key = ENV["DEEPSEEK_API_KEY"]
+      @deepseek_api_url = "https://api.deepseek.com/v1/chat/completions"
+
+      @mistral_api_key = ENV["MISTRAL_API_KEY"]
+      @mistral_api_url = "https://api.mistral.ai/v1/chat/completions"
+
+      @ollama_api_key  = ENV["OLLAMA_API_KEY"]   # nil if not set — key is optional
+      @ollama_api_base = ENV.fetch("OLLAMA_API_BASE", "http://localhost:11434")
+
+      @perplexity_api_key = ENV["PERPLEXITY_API_KEY"]
+      @perplexity_api_url = "https://api.perplexity.ai/chat/completions"
+
+      @gpustack_api_key  = ENV["GPUSTACK_API_KEY"]   # nil if not set — key is optional
+      @gpustack_api_base = ENV["GPUSTACK_API_BASE"]
+
+      @azure_api_key       = ENV["AZURE_API_KEY"]
+      @azure_ai_auth_token = ENV["AZURE_AI_AUTH_TOKEN"]
+      @azure_api_base      = ENV["AZURE_API_BASE"]
+      @azure_api_version   = ENV.fetch("AZURE_API_VERSION", "2024-05-01-preview")
+    end
+  end
+end

data/lib/active_harness/http/retry_policy.rb

@@ -0,0 +1,47 @@
+module ActiveHarness
+  module Http
+    # Wraps a block with automatic retry on transient errors.
+    # Uses exponential backoff: delay doubles after each failed attempt.
+    #
+    # Example:
+    #   RetryPolicy.new(max_attempts: 3, base_delay: 0.5).run do
+    #     http_client.post(...)
+    #   end
+    #
+    # Disable retries entirely:
+    #   RetryPolicy.new(max_attempts: 1).run { ... }
+    #
+    class RetryPolicy
+      RETRYABLE_ERRORS = [
+        Errors::TimeoutError,
+        Errors::RateLimitError,
+        Errors::ProviderUnavailableError,
+        Errors::ServerError
+      ].freeze
+
+      # @param max_attempts [Integer]  total number of attempts (1 = no retries)
+      # @param base_delay   [Float]    seconds before 1st retry; doubles each round
+      # @param errors       [Array]    error classes that trigger a retry
+      def initialize(max_attempts: 3, base_delay: 1.0, errors: RETRYABLE_ERRORS)
+        @max_attempts = max_attempts
+        @base_delay   = base_delay
+        @errors       = errors
+      end
+
+      # @yieldreturn [Object] result of the block on success
+      # @raise last error after all attempts are exhausted
+      def run
+        attempt = 0
+        begin
+          attempt += 1
+          yield
+        rescue *@errors => e
+          raise if attempt >= @max_attempts
+
+          sleep(@base_delay * (2**(attempt - 1)))
+          retry
+        end
+      end
+    end
+  end
+end

data/lib/active_harness/providers/anthropic.rb

@@ -5,7 +5,6 @@ module ActiveHarness
     # Anthropic Claude — native Messages API (not OpenAI-compatible).
     # https://docs.anthropic.com/en/api/messages
     class Anthropic < Base
-      API_URL         = URI("https://api.anthropic.com/v1/messages")
       ANTHROPIC_VERSION = "2023-06-01"
       DEFAULT_MAX_TOKENS = 1024
 
@@ -20,7 +19,7 @@ module ActiveHarness
         }
         body[:system] = system_msg if system_msg
 
-        raw  = post_json(API_URL,
+        raw  = post_json(URI(config.anthropic_api_url),
           headers: {
             "Content-Type"      => "application/json",
             "x-api-key"         => api_key,
@@ -52,8 +51,8 @@ module ActiveHarness
       end
 
       def api_key
-        key = ENV["ANTHROPIC_API_KEY"].to_s
-        raise Errors::InvalidApiKeyError, "ANTHROPIC_API_KEY is not set" if key.empty?
+        key = config.anthropic_api_key.to_s
+        raise Errors::InvalidApiKeyError, "anthropic_api_key is not configured" if key.empty?
         key
       end
 

data/lib/active_harness/providers/azure.rb

@@ -0,0 +1,97 @@
+require "uri"
+
+module ActiveHarness
+  module Providers
+    # Azure OpenAI Service — deployment-based API.
+    # https://learn.microsoft.com/en-us/azure/ai-services/openai/reference
+    #
+    # The `model:` parameter is treated as the **deployment name** you created
+    # in the Azure portal (not the underlying model name).
+    #
+    # Required config (or ENV fallback):
+    #   config.azure_api_base    — "https://my-resource.openai.azure.com"
+    #   config.azure_api_key     — your resource API key
+    #                              (alternatively: config.azure_ai_auth_token for OAuth bearer)
+    #
+    # Optional config:
+    #   config.azure_api_version — defaults to "2024-05-01-preview"
+    #
+    # Resulting endpoint:
+    #   POST {azure_api_base}/openai/deployments/{deployment}/chat/completions
+    #        ?api-version={azure_api_version}
+    #
+    # Example agent config:
+    #   model do
+    #     use provider: :azure, model: "my-gpt4o-deployment", temperature: 0.7
+    #   end
+    class Azure < Base
+      def call(model:, messages:, temperature: 0.7)
+        url = build_url(model)
+
+        raw  = post_json(url,
+          headers: {
+            "Content-Type" => "application/json"
+          }.merge(auth_header),
+          body: { messages: messages, temperature: temperature }
+        )
+        data = parse!(raw)
+        handle_error!(data)
+
+        {
+          content:  data.dig("choices", 0, "message", "content").to_s.strip,
+          provider: :azure,
+          model:    data["model"] || model,
+          usage:    extract_usage_openai(data)
+        }
+      end
+
+      private
+
+      def build_url(deployment)
+        URI("#{api_base}/openai/deployments/#{deployment}/chat/completions?api-version=#{config.azure_api_version}")
+      end
+
+      def api_base
+        base = config.azure_api_base.to_s
+        raise Errors::InvalidRequestError, "azure_api_base is not configured" if base.empty?
+        base.chomp("/")
+      end
+
+      # Azure accepts either a resource API key (header: api-key)
+      # or an OAuth2 bearer token (header: Authorization).
+      def auth_header
+        if (key = config.azure_api_key.to_s) && !key.empty?
+          { "api-key" => key }
+        elsif (token = config.azure_ai_auth_token.to_s) && !token.empty?
+          { "Authorization" => "Bearer #{token}" }
+        else
+          raise Errors::InvalidApiKeyError,
+            "Neither azure_api_key nor azure_ai_auth_token is configured"
+        end
+      end
+
+      def handle_error!(data)
+        return unless data["error"]
+
+        msg      = data.dig("error", "message").to_s
+        code     = data.dig("error", "code").to_s
+        type     = data.dig("error", "innererror", "code").to_s
+        metadata = data["error"].reject { |k, _| %w[message code innererror].include?(k) }
+        metadata = nil if metadata.empty?
+
+        case code
+        when "401", "invalid_api_key", "unauthorized", "AccessDenied"
+          raise Errors::InvalidApiKeyError.new(msg,       error_code: code, metadata: metadata)
+        when "429", "TooManyRequests"
+          raise Errors::RateLimitError.new(msg,           error_code: code, metadata: metadata)
+        when "ContentFilter", "content_filter"
+          raise Errors::SafetyBlockedError.new(msg,       error_code: code, metadata: metadata)
+        when "500", "502", "503", "504", "ServiceUnavailable"
+          raise Errors::ProviderUnavailableError.new(msg, error_code: code, metadata: metadata)
+        else
+          raise Errors::InvalidRequestError.new(msg,      error_code: code, metadata: metadata)
+        end
+      end
+    end
+  end
+end

data/lib/active_harness/providers/base.rb

@@ -8,6 +8,10 @@ module ActiveHarness
 
       private
 
+      def config
+        ActiveHarness.config
+      end
+
       def post_json(url, headers:, body:, timeout: 30)
         HTTP.post(url, headers: headers, body: body.to_json, timeout: timeout)
       end

data/lib/active_harness/providers/bedrock.rb

@@ -0,0 +1,29 @@
+module ActiveHarness
+  module Providers
+    # AWS Bedrock — stub provider.
+    #
+    # Bedrock requires AWS Signature V4 request signing, which is non-trivial
+    # to implement and carries AWS SDK dependencies. This stub raises a clear
+    # error so that the agent falls through to the next model in its fallback chain.
+    #
+    # To use Bedrock in production, please look for a dedicated gem, for example:
+    #   gem "active_harness-bedrock"  (not yet released — contributions welcome)
+    #
+    # Example agent config (will fall through to the next fallback):
+    #   model do
+    #     use      provider: :bedrock, model: "anthropic.claude-3-5-sonnet-20241022-v2:0"
+    #     fallback provider: :anthropic, model: "claude-3-5-sonnet-20241022"
+    #   end
+    class Bedrock < Base
+      STUB_MESSAGE = <<~MSG.strip
+        ActiveHarness: AWS Bedrock provider is not built-in.
+        Bedrock requires AWS Signature V4 signing — please use a dedicated gem.
+        Falling through to the next model in the fallback chain.
+      MSG
+
+      def call(model:, messages:, temperature: 0.7) # rubocop:disable Lint/UnusedMethodArgument
+        raise Errors::ProviderUnavailableError, STUB_MESSAGE
+      end
+    end
+  end
+end