agent-harness 0.7.4 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 278785d86727fd759e55bcd8fd4fb4124a13c8f6ae818a40b2ae49bcbbb3b18f
-  data.tar.gz: 717338d556ef335ebf3d4e2f0fbdb4a9d92bbbe52bbb1739fcb8afaba7b0c1ac
+  metadata.gz: a5bc12df7dec39319082c66c7aa4fc47bfc3f3d1765f572c4faa49677e18bad1
+  data.tar.gz: '08138f93d009f4d390917d7f09721adf7e1fbfe0eb2bedb75352b3c2250b245d'
 SHA512:
-  metadata.gz: 76cd57c3875f38271390f3f7ebe29153d40924988315807d79fd85d37fdedde109e7c465a6eeeb889c858a6da53faac5cd48dc8a62862fed5d6843e73b4036a7
-  data.tar.gz: 6d74d1ac89feb72339a87bb08b413ee6996b0dc1b0b7cb7ff2446ac3ec12539434a00f57187e1632ec889b5fba5bab10ce20d2b91a7aaee5b21e39c837101b04
+  metadata.gz: 0b0870b435dd46293c04eb382476138bb5b2cd1c8e6fc665a9c41fa0b4ed20be09906242a9e492f8c8f4eebd6c2da17c51ddb17860750ed634da220a54c80846
+  data.tar.gz: 1cc610554e295e6d9a65c7ac2f44209f99c45eab8b713256397a595d95c8bcd88a8a3463509dc55881b3a22701e142bb70632846b99c2f4d30a05be0888e8fd9

data/.release-please-manifest.json

@@ -1,3 +1,3 @@
 {
-  ".": "0.7.4"
+  ".": "0.8.0"
 }

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 ## [Unreleased]
 
+## [0.8.0](https://github.com/viamin/agent-harness/compare/agent-harness/v0.7.4...agent-harness/v0.8.0) (2026-04-19)
+
+
+### Features
+
+* **providers:** add config_file_content, notify_hook_content, and auth_lock_config ([#131](https://github.com/viamin/agent-harness/issues/131)) ([e95117e](https://github.com/viamin/agent-harness/commit/e95117e8000002972ca0fb31cb90dec035aa88fd))
+* **providers:** add env var name mappings to provider classes ([#122](https://github.com/viamin/agent-harness/issues/122)) ([#133](https://github.com/viamin/agent-harness/issues/133)) ([6be9015](https://github.com/viamin/agent-harness/commit/6be901592afb02337eb2a5269f08e3025c7511c1))
+* **providers:** add error_classification_patterns, noisy_error_patterns, and translate_error to provider classes ([#128](https://github.com/viamin/agent-harness/issues/128)) ([e2dfbed](https://github.com/viamin/agent-harness/commit/e2dfbed064fa26b2cae5691e6586e79900d19d28))
+* **providers:** add parse_rate_limit_reset to provider base class ([#134](https://github.com/viamin/agent-harness/issues/134)) ([c16a6f8](https://github.com/viamin/agent-harness/commit/c16a6f8de312e137e5c3431f32d42b8c65126e0e))
+* **providers:** add test_command_overrides and parse_test_error methods ([#129](https://github.com/viamin/agent-harness/issues/129)) ([a18102d](https://github.com/viamin/agent-harness/commit/a18102d1ee333a1db37f711c65e16dc20d4a0a11)), closes [#125](https://github.com/viamin/agent-harness/issues/125)
+* **providers:** add token_usage_from_api_response to provider classes ([#130](https://github.com/viamin/agent-harness/issues/130)) ([f2c095d](https://github.com/viamin/agent-harness/commit/f2c095dcc0ae0d7822da90f98704597f08e4ed04)), closes [#126](https://github.com/viamin/agent-harness/issues/126)
+
 ## [0.7.4](https://github.com/viamin/agent-harness/compare/agent-harness/v0.7.3...agent-harness/v0.7.4) (2026-04-18)
 
 

data/lib/agent_harness/providers/adapter.rb

@@ -781,6 +781,50 @@ module AgentHarness
         {}
       end
 
+      # Error classification patterns for downstream consumers
+      #
+      # Returns patterns grouped by classification category. These patterns
+      # encode provider-specific knowledge about how each CLI reports errors
+      # and are intended for use by callers outside agent-harness.
+      #
+      # @return [Hash<Symbol, Array<Regexp>>] patterns by category
+      #   (:auth_expired, :abort, :authentication, :quota)
+      def error_classification_patterns
+        {
+          auth_expired: [],
+          abort: [],
+          authentication: [],
+          quota: [
+            /requires more credits/i,
+            /insufficient credits/i,
+            /credit.*exceeded/i,
+            /spend limit.*reached/i,
+            /billing.*limit/i
+          ]
+        }
+      end
+
+      # Patterns matching noisy/non-actionable error output
+      #
+      # Downstream consumers can use these to filter out log noise
+      # from provider stderr/stdout that is not meaningful for users.
+      #
+      # @return [Array<Regexp>] noisy error patterns
+      def noisy_error_patterns
+        []
+      end
+
+      # Translate a raw error message into a user-friendly string
+      #
+      # Providers override this to map CLI-specific error output
+      # into concise, actionable messages.
+      #
+      # @param message [String] raw error message
+      # @return [String] translated message (or the original if no match)
+      def translate_error(message)
+        message
+      end
+
       # Authentication type for this provider
       #
       # @return [Symbol] :oauth for token-based auth that can expire,
@@ -902,6 +946,17 @@ module AgentHarness
         []
       end
 
+      # Extract token usage from an API response body
+      #
+      # Parses the provider-specific API response shape and returns
+      # normalized token counts.
+      #
+      # @param body [Hash] the parsed JSON response body from the provider API
+      # @return [Hash] with :input_tokens and :output_tokens keys, or empty hash
+      def token_usage_from_api_response(body)
+        {}
+      end
+
       # Whether this provider can extract token usage from CLI output
       #
       # @return [Boolean] true if the provider returns token counts
@@ -1020,6 +1075,35 @@ module AgentHarness
         nil
       end
 
+      # Generate provider-specific config file content.
+      #
+      # Providers that require a config file written before CLI execution
+      # (e.g. Codex TOML, Kilocode JSON) should override this method.
+      #
+      # @param options [Hash] provider-specific options for config generation
+      # @return [String, nil] config file content, or nil when no config is needed
+      def config_file_content(options = {})
+        nil
+      end
+
+      # Generate provider-specific notification hook content.
+      #
+      # Providers that support notification hooks appended to their config
+      # file should override this method.
+      #
+      # @return [String, nil] notify hook content, or nil when not applicable
+      def notify_hook_content
+        nil
+      end
+
+      # Auth lock configuration for providers that need file-based lock
+      # serialization (e.g. OAuth refresh token coordination).
+      #
+      # @return [Hash, nil] lock config with :path and :timeout keys, or nil
+      def auth_lock_config
+        nil
+      end
+
       private
 
       def classify_smoke_test_message(message)

data/lib/agent_harness/providers/aider.rb

@@ -170,6 +170,10 @@ module AgentHarness
         }
       end
 
+      def api_key_env_var_names = ["ANTHROPIC_API_KEY"]
+
+      def api_key_unset_vars = ["ANTHROPIC_BASE_URL", "ANTHROPIC_HEADER_X_AGENT_RUN_ID", "ANTHROPIC_HEADER_X_PROXY_TOKEN"]
+
       def error_patterns
         COMMON_ERROR_PATTERNS.merge(
           auth_expired: COMMON_ERROR_PATTERNS[:auth_expired] + [/incorrect.*api.*key/i],

data/lib/agent_harness/providers/anthropic.rb

@@ -14,6 +14,8 @@ module AgentHarness
     #   provider = AgentHarness::Providers::Anthropic.new
     #   response = provider.send_message(prompt: "Hello!")
     class Anthropic < Base
+      include RateLimitResetParsing
+
       # Model name pattern for Anthropic Claude models
       MODEL_PATTERN = /^claude-[\d.-]+-(?:opus|sonnet|haiku)(?:-\d{8})?$/i
       SUPPORTED_CLI_VERSION = "2.1.92"
@@ -379,6 +381,12 @@ module AgentHarness
         cleanup_mcp_tempfiles!
       end
 
+      def api_key_env_var_names = ["ANTHROPIC_API_KEY"]
+
+      def api_key_unset_vars = ["ANTHROPIC_BASE_URL", "ANTHROPIC_HEADER_X_AGENT_RUN_ID", "ANTHROPIC_HEADER_X_PROXY_TOKEN"]
+
+      def subscription_unset_vars = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL"] + api_key_unset_vars
+
       def supports_mcp?
         true
       end
@@ -414,6 +422,16 @@ module AgentHarness
         true
       end
 
+      def token_usage_from_api_response(body)
+        usage = body&.dig("usage")
+        return {} unless usage
+
+        {
+          input_tokens: usage["input_tokens"].to_i,
+          output_tokens: usage["output_tokens"].to_i
+        }
+      end
+
       def execution_semantics
         {
           prompt_delivery: :arg,
@@ -474,6 +492,15 @@ module AgentHarness
         }
       end
 
+      def error_classification_patterns
+        super.merge(
+          abort: [
+            /free tier limit reached/i,
+            /please upgrade to a paid plan/i
+          ]
+        )
+      end
+
       def fetch_mcp_servers
         return [] unless self.class.available?
 

data/lib/agent_harness/providers/base.rb

@@ -205,6 +205,62 @@ module AgentHarness
         @executor.is_a?(DockerCommandExecutor)
       end
 
+      # Environment variable names that the provider's CLI reads for API key authentication.
+      #
+      # @return [Array<String>] env var names (empty by default)
+      def api_key_env_var_names = []
+
+      # Environment variable names to unset when the caller supplies its own API key,
+      # preventing the CLI from reading stale or conflicting proxy/header variables.
+      #
+      # @return [Array<String>] env var names (empty by default)
+      def api_key_unset_vars = []
+
+      # Environment variable names to unset when the caller uses subscription-based auth,
+      # ensuring the CLI does not pick up API-key or proxy variables that would conflict.
+      #
+      # @return [Array<String>] env var names (empty by default)
+      def subscription_unset_vars = []
+
+      # Provider-specific environment variable overrides that the caller should set
+      # when invoking the CLI (e.g. feature flags or sandbox controls).
+      #
+      # @return [Hash{String => String}] env var name => value (empty by default)
+      def cli_env_overrides = {}
+
+      # Additional CLI flags for health-check/test invocations
+      #
+      # Providers override this to supply flags that should be appended
+      # when the CLI is invoked in a test or smoke-test context.
+      #
+      # @return [Array<String>] extra CLI flags (empty by default)
+      def test_command_overrides
+        []
+      end
+
+      # Parse provider-specific error information from test output
+      #
+      # Providers override this to extract structured error details from
+      # CLI output or sidecar files produced during a test invocation.
+      #
+      # @param output [String] the CLI stdout/stderr output
+      # @param files [Hash] mapping of logical names to file paths
+      # @return [Hash, nil] structured error hash or nil if no error detected
+      def parse_test_error(output:, files: {})
+        nil
+      end
+
+      # Parse rate-limit reset time from provider error output.
+      #
+      # Providers that emit rate-limit reset times should override this
+      # method (or include RateLimitResetParsing for the common format).
+      #
+      # @param text [String, nil] error output text
+      # @return [Time, nil] UTC reset time, or nil if not parseable
+      def parse_rate_limit_reset(text)
+        nil
+      end
+
       protected
 
       # Build CLI command - override in subclasses

data/lib/agent_harness/providers/codex.rb

@@ -8,6 +8,8 @@ module AgentHarness
     #
     # Provides integration with the OpenAI Codex CLI tool.
     class Codex < Base
+      include RateLimitResetParsing
+
       SUPPORTED_CLI_VERSION = "0.116.0"
       SUPPORTED_CLI_REQUIREMENT = Gem::Requirement.new(">= #{SUPPORTED_CLI_VERSION}", "< 0.117.0").freeze
       OAUTH_REFRESH_FAILURE_PATTERNS = [
@@ -167,10 +169,32 @@ module AgentHarness
         }
       end
 
+      def api_key_env_var_names = ["OPENAI_API_KEY"]
+
+      def api_key_unset_vars = ["OPENAI_BASE_URL", "OPENAI_HEADER_X_AGENT_RUN_ID", "OPENAI_HEADER_X_PROXY_TOKEN"]
+
+      def subscription_unset_vars = ["OPENAI_API_KEY", "OPENAI_BASE_URL"] + api_key_unset_vars
+
+      def cli_env_overrides = {"PAID_CODEX_SUBSCRIPTION_AUTH" => "1"}
+
+      def test_command_overrides
+        ["--skip-git-repo-check", "--output-last-message"]
+      end
+
       def dangerous_mode_flags
         ["--full-auto"]
       end
 
+      def token_usage_from_api_response(body)
+        usage = body&.dig("usage")
+        return {} unless usage
+
+        {
+          input_tokens: usage["prompt_tokens"].to_i,
+          output_tokens: usage["completion_tokens"].to_i
+        }
+      end
+
       def execution_semantics
         {
           prompt_delivery: :arg,
@@ -216,6 +240,31 @@ module AgentHarness
         }
       end
 
+      def error_classification_patterns
+        super.merge(
+          auth_expired: [
+            /refresh_token_reused/i,
+            /refresh token has already been used/i,
+            /Please log out and sign in again/i,
+            /authentication_error/i,
+            /invalid_grant/i,
+            /Token is expired or invalid/i
+          ],
+          abort: [
+            /free tier limit reached/i,
+            /please upgrade to a paid plan/i
+          ]
+        )
+      end
+
+      def translate_error(message)
+        case message
+        when /refresh_token_reused/i then "Codex authentication expired. Please re-authenticate."
+        when /free tier limit/i then "Codex free tier limit reached."
+        else message
+        end
+      end
+
       def auth_status
         api_key = ENV["OPENAI_API_KEY"]
         if api_key && !api_key.strip.empty?
@@ -272,6 +321,28 @@ module AgentHarness
         {valid: errors.empty?, errors: errors}
       end
 
+      def config_file_content(options = {})
+        <<~TOML
+          [chatgpt]
+          model_provider = "#{escape_toml_string(options[:model_provider])}"
+          base_url = "#{escape_toml_string(options[:base_url])}"
+          env_key = "#{escape_toml_string(options[:env_key])}"
+          wire_api = "#{escape_toml_string(options[:wire_api])}"
+        TOML
+      end
+
+      def notify_hook_content
+        <<~TOML
+
+          [notify]
+          # Paid notification hook
+        TOML
+      end
+
+      def auth_lock_config
+        {path: "/tmp/codex-auth.lock", timeout: 30}
+      end
+
       protected
 
       def parse_response(result, duration:)
@@ -387,6 +458,10 @@ module AgentHarness
 
       private
 
+      def escape_toml_string(val)
+        val.to_s.gsub("\\") { "\\\\" }.gsub('"') { "\\\"" }.gsub("\n") { "\\n" }
+      end
+
       def parse_jsonl_output(raw_output)
         return nil if raw_output.nil? || raw_output.strip.empty?
 

data/lib/agent_harness/providers/cursor.rb

@@ -12,6 +12,8 @@ module AgentHarness
     #   provider = AgentHarness::Providers::Cursor.new
     #   response = provider.send_message(prompt: "Hello!")
     class Cursor < Base
+      include RateLimitResetParsing
+
       INSTALL_SCRIPT_URL = "https://cursor.com/install"
       INSTALL_TARGET_LATEST = "latest"
       INSTALL_BUILD = "2026.03.30-a5d3e17"
@@ -206,6 +208,10 @@ module AgentHarness
         fetch_mcp_servers_cli || fetch_mcp_servers_config
       end
 
+      def api_key_env_var_names = ["ANTHROPIC_API_KEY"]
+
+      def api_key_unset_vars = ["ANTHROPIC_BASE_URL", "ANTHROPIC_HEADER_X_AGENT_RUN_ID", "ANTHROPIC_HEADER_X_PROXY_TOKEN"]
+
       def auth_type
         :oauth
       end

data/lib/agent_harness/providers/gemini.rb

@@ -163,10 +163,47 @@ module AgentHarness
         }
       end
 
+      def parse_test_error(output:, files: {})
+        error_file = files.values.find { |path| path.match?(/gemini-client-error-.*\.json/) }
+        return nil unless error_file
+
+        error_data = begin
+          JSON.parse(File.read(error_file))
+        rescue JSON::ParserError, Errno::ENOENT
+          nil
+        end
+        return nil unless error_data
+
+        {message: error_data.dig("error", "message") || output, type: :configuration}
+      end
+
+      def api_key_env_var_names = ["GEMINI_API_KEY", "GOOGLE_API_KEY"]
+
+      def api_key_unset_vars = ["GOOGLE_GEMINI_BASE_URL", "GOOGLE_GENAI_BASE_URL", "GEMINI_CLI_CUSTOM_HEADERS"]
+
+      def subscription_unset_vars = ["GEMINI_API_KEY", "GOOGLE_GEMINI_BASE_URL", "GOOGLE_GENAI_BASE_URL", "GEMINI_CLI_CUSTOM_HEADERS"]
+
+      def cli_env_overrides
+        {
+          "GEMINI_SANDBOX" => "false",
+          "GEMINI_CLI_DISABLE_RETRIES" => "true"
+        }
+      end
+
       def auth_type
         :oauth
       end
 
+      def token_usage_from_api_response(body)
+        usage = body&.dig("usageMetadata")
+        return {} unless usage
+
+        {
+          input_tokens: usage["promptTokenCount"].to_i,
+          output_tokens: usage["candidatesTokenCount"].to_i
+        }
+      end
+
       def execution_semantics
         {
           prompt_delivery: :flag,
@@ -204,6 +241,35 @@ module AgentHarness
         }
       end
 
+      def error_classification_patterns
+        super.merge(
+          authentication: [
+            /GEMINI_API_KEY/i,
+            /GOOGLE_GENAI_USE_VERTEXAI/i,
+            /GOOGLE_GENAI_USE_GCA/i,
+            /ValidationRequiredError/i,
+            /API key not configured for google/i,
+            /API key not valid/i
+          ]
+        )
+      end
+
+      def noisy_error_patterns
+        [
+          /Error when talking to Gemini API/i,
+          /service=.*status/i,
+          /loading\.\.\./i,
+          /subscribing/i
+        ]
+      end
+
+      def translate_error(message)
+        case message
+        when /API key not configured/i then "Gemini API key not set. Run: export GEMINI_API_KEY=..."
+        else message
+        end
+      end
+
       def auth_status
         api_key = [ENV["GEMINI_API_KEY"], ENV["GOOGLE_API_KEY"]].find { |key| key && !key.strip.empty? }
         if api_key

data/lib/agent_harness/providers/github_copilot.rb

@@ -191,6 +191,13 @@ module AgentHarness
         }
       end
 
+      def translate_error(message)
+        case message
+        when /github-copilot-cli.*not found/i then "GitHub Copilot CLI not installed."
+        else message
+        end
+      end
+
       def supports_token_counting?
         supports_json_output_format?
       end

data/lib/agent_harness/providers/kilocode.rb

@@ -113,6 +113,10 @@ module AgentHarness
         "Kilocode CLI"
       end
 
+      def test_command_overrides
+        ["--auto", "--print-logs"]
+      end
+
       def capabilities
         {
           streaming: false,
@@ -125,6 +129,13 @@ module AgentHarness
         }
       end
 
+      def config_file_content(options = {})
+        {
+          provider: options[:api_provider],
+          model: options[:model_id]
+        }.to_json
+      end
+
       def error_patterns
         COMMON_ERROR_PATTERNS
       end

data/lib/agent_harness/providers/rate_limit_reset_parsing.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module AgentHarness
+  module Providers
+    # Shared rate-limit reset time parsing for providers whose CLIs emit
+    # standard reset-time formats in error output.
+    #
+    # Include this module in any provider that uses the common format:
+    #   - "retry after 60s"        (seconds)
+    #   - "reset at 1234567890"    (unix timestamp)
+    #   - "resets 5am (UTC)"       (time today/tomorrow)
+    #   - "resets 5:00am (UTC)"    (time with minutes)
+    #   - "resets Jan 15, 5pm (UTC)" (date + time)
+    #
+    # Providers with a different format should override
+    # +parse_rate_limit_reset+ directly instead of including this module.
+    module RateLimitResetParsing
+      # Parse rate-limit reset time from provider error output.
+      #
+      # @param text [String, nil] error output text
+      # @return [Time, nil] UTC reset time, or nil if not parseable
+      def parse_rate_limit_reset(text)
+        return nil unless text
+
+        parse_retry_after(text) ||
+          parse_reset_at(text) ||
+          parse_resets_time(text) ||
+          parse_resets_date_time(text)
+      end
+
+      private
+
+      # "retry after 60s"
+      def parse_retry_after(text)
+        match = text.match(/retry\s+after\s+(\d+)\s*s/i)
+        return unless match
+
+        Time.now.utc + match[1].to_i
+      end
+
+      # "reset at 1234567890" (unix timestamp)
+      def parse_reset_at(text)
+        match = text.match(/reset\s+at\s+(\d{10})/i)
+        return unless match
+
+        Time.at(match[1].to_i).utc
+      end
+
+      # "resets 5am (UTC)" / "resets 5:00am (UTC)"
+      def parse_resets_time(text)
+        match = text.match(/resets\s+(\d{1,2})(?::(\d{2}))?\s*(am|pm)\s*\(UTC\)/i)
+        return unless match
+
+        hour = match[1].to_i
+        minute = match[2]&.to_i || 0
+        meridiem = match[3].downcase
+        hour = to_24h(hour, meridiem)
+
+        now = Time.now.utc
+        reset_time = Time.utc(now.year, now.month, now.day, hour, minute)
+        reset_time += 86_400 if reset_time <= now
+        reset_time
+      end
+
+      # "resets Jan 15, 5pm (UTC)"
+      def parse_resets_date_time(text)
+        match = text.match(/resets\s+(\w{3})\s+(\d{1,2}),?\s+(\d{1,2})(?::(\d{2}))?\s*(am|pm)\s*\(UTC\)/i)
+        return unless match
+
+        month = Date::ABBR_MONTHNAMES.index(match[1].capitalize)
+        return unless month
+
+        day = match[2].to_i
+        hour = match[3].to_i
+        minute = match[4]&.to_i || 0
+        meridiem = match[5].downcase
+        hour = to_24h(hour, meridiem)
+
+        year = Time.now.utc.year
+        year += 1 if month < Time.now.utc.month
+
+        Time.utc(year, month, day, hour, minute)
+      end
+
+      def to_24h(hour, meridiem)
+        hour += 12 if meridiem == "pm" && hour != 12
+        hour = 0 if meridiem == "am" && hour == 12
+        hour
+      end
+    end
+  end
+end

data/lib/agent_harness/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AgentHarness
-  VERSION = "0.7.4"
+  VERSION = "0.8.0"
 end

data/lib/agent_harness.rb

@@ -254,6 +254,7 @@ require_relative "agent_harness/providers/registry"
 require_relative "agent_harness/providers/adapter"
 require_relative "agent_harness/providers/base"
 require_relative "agent_harness/providers/token_usage_parsing"
+require_relative "agent_harness/providers/rate_limit_reset_parsing"
 require_relative "agent_harness/providers/anthropic"
 require_relative "agent_harness/providers/aider"
 require_relative "agent_harness/providers/codex"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: agent-harness
 version: !ruby/object:Gem::Version
-  version: 0.7.4
+  version: 0.8.0
 platform: ruby
 authors:
 - Bart Agapinan
@@ -126,6 +126,7 @@ files:
 - lib/agent_harness/providers/kilocode.rb
 - lib/agent_harness/providers/mistral_vibe.rb
 - lib/agent_harness/providers/opencode.rb
+- lib/agent_harness/providers/rate_limit_reset_parsing.rb
 - lib/agent_harness/providers/registry.rb
 - lib/agent_harness/providers/token_usage_parsing.rb
 - lib/agent_harness/response.rb