agent-harness 0.14.1 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 972e7e3144da1a59c0a25dbf4668766d07693870dde63c74dece4842c960dfe5
-  data.tar.gz: 3988ed1d19d61ce9144224302c0246c2c32442abdfa1d1c9bc4616abcb4e11c3
+  metadata.gz: 34ea90d66e03bff4f53fe144d888ab33909d80e9a198a9a53f7dac00c0ed5d52
+  data.tar.gz: 4daf550bb940f4176b06ce186658803e090a90ed2e5811ae7f22d61dc95391b8
 SHA512:
-  metadata.gz: 97d598d30445ef7617c172b692d43f3c8d8c896b9c12f28f7f0e56f822128e298312129bdae67adbbfc81eb407686587db677787bda70a2292c3a1f07aea9a60
-  data.tar.gz: 863739d9ace22d47b37799e44b36c19cacf221a4042947a627525f542838ccaafb933e03c3e0c10141c0d30971283046791b44d4ca8cc7c84c4a0e50184010a8
+  metadata.gz: 9273ea29e29a8380e9f64a926019bac4adc160b2205aecceec2d5ba8e0837d16b58ab9106f05e61832153de95c03c9c7a58b1458588a4f4552a0218c210a4c4a
+  data.tar.gz: 0ccefa077c32912d9e6dd205d20420d5ba0170fcfa763df22ca65902ea7fcb9ba600c439e91c38c398fc221dc3b41b9662ce5b693e418078dfd1116027378743

data/.release-please-manifest.json

@@ -1,3 +1,3 @@
 {
-  ".": "0.14.1"
+  ".": "0.15.0"
 }

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 ## [Unreleased]
 
+## [0.15.0](https://github.com/viamin/agent-harness/compare/agent-harness/v0.14.1...agent-harness/v0.15.0) (2026-05-03)
+
+
+### Features
+
+* pre-flight connectivity check API for provider health verification ([#185](https://github.com/viamin/agent-harness/issues/185)) ([3ad6a2f](https://github.com/viamin/agent-harness/commit/3ad6a2ffbfe84b2271e4de968fa096276724e63c))
+
 ## [0.14.1](https://github.com/viamin/agent-harness/compare/agent-harness/v0.14.0...agent-harness/v0.14.1) (2026-05-03)
 
 

data/lib/agent_harness/provider_health_check.rb

@@ -186,6 +186,7 @@ module AgentHarness
         klass = registry.get(provider_name)
         provider_instance = build_provider(provider_name, klass, executor: executor)
         host_preflight_allowed = host_preflight_allowed?(executor: executor, provider_runtime: provider_runtime)
+        provider_preflight_allowed = provider_preflight_allowed?(executor: executor)
 
         auth_degraded = false
         if host_preflight_allowed
@@ -273,6 +274,26 @@ module AgentHarness
           )
         end
 
+        # Only run the provider preflight in host contexts. The preflight
+        # hook (e.g. Codex's Net::HTTP probe) executes in the Ruby host
+        # process, so its network view may not match a containerised or
+        # remote executor. Skipping it avoids marking a provider unhealthy
+        # when only the host cannot reach the endpoint.
+        if provider_preflight_allowed
+          preflight_env = build_preflight_env(provider_instance, provider_runtime)
+          preflight = provider_instance.preflight_check(env: preflight_env, timeout: timeout)
+          unless preflight[:healthy]
+            return build_result(
+              name: provider_name,
+              status: "error",
+              message: preflight[:reason] || "Preflight check failed",
+              start_time: start_time,
+              error_category: normalize_preflight_error_category(preflight[:error_category]),
+              check: :preflight
+            )
+          end
+        end
+
         smoke_contract = provider_instance.smoke_test_contract
         # Explicitly handle missing smoke-test contract when no custom smoke_test implementation
         if smoke_contract.nil? && !provider_overrides_method?(provider_instance, :smoke_test)
@@ -363,15 +384,23 @@ module AgentHarness
 
       def host_preflight_allowed?(executor:, provider_runtime: nil)
         effective_executor = executor || AgentHarness.configuration.command_executor
-        # Skip host preflight only when provider runtime has environment/config overrides
-        # that could conflict with host-level checks (env, base_url, api_provider, unset_env)
         if provider_runtime
           runtime = ProviderRuntime.wrap(provider_runtime)
-          return false if runtime && (!runtime.env.empty? || !runtime.unset_env.empty? || runtime.base_url || runtime.api_provider)
+          return false if runtime_sensitive_host_overrides?(runtime)
         end
+
+        provider_preflight_allowed?(executor: effective_executor)
+      end
+
+      def provider_preflight_allowed?(executor:)
+        effective_executor = executor || AgentHarness.configuration.command_executor
         effective_executor.is_a?(CommandExecutor) && !effective_executor.is_a?(DockerCommandExecutor)
       end
 
+      def runtime_sensitive_host_overrides?(runtime)
+        runtime && (!runtime.env.empty? || !runtime.unset_env.empty? || runtime.base_url || runtime.api_provider)
+      end
+
       def effective_check_timeout(provider_name, base_timeout)
         registry = Providers::Registry.instance
         return base_timeout unless registry.registered?(provider_name)
@@ -410,6 +439,25 @@ module AgentHarness
         end
       end
 
+      def normalize_preflight_error_category(category)
+        case category&.to_sym
+        when :installation
+          :installation
+        when :auth_expired, :authentication
+          :authentication
+        when :rate_limited, :rate_limit
+          :rate_limit
+        when :quota_exceeded, :quota
+          :quota
+        when :timeout
+          :timeout
+        when :configuration
+          :configuration
+        else
+          :transient
+        end
+      end
+
       def installation_failure_message?(message)
         message.to_s.match?(/(not found in PATH|command not found|No such file or directory|is not installed)/i)
       end
@@ -453,6 +501,15 @@ module AgentHarness
         provider
       end
 
+      def build_preflight_env(provider_instance, provider_runtime)
+        return {} unless provider_instance.respond_to?(:build_env, true)
+
+        runtime = ProviderRuntime.wrap(provider_runtime)
+        provider_instance.send(:build_env, {provider_runtime: runtime})
+      rescue ArgumentError, NoMethodError
+        {}
+      end
+
       def monotonic_now
         Process.clock_gettime(Process::CLOCK_MONOTONIC)
       end

data/lib/agent_harness/providers/adapter.rb

@@ -1042,6 +1042,15 @@ module AgentHarness
         {healthy: true, message: "OK"}
       end
 
+      # Lightweight provider-owned preflight check executed before smoke tests.
+      #
+      # @param env [Hash] request-scoped environment overrides
+      # @param timeout [Numeric] time budget in seconds
+      # @return [Hash] with :healthy and optional :reason keys
+      def preflight_check(env:, timeout: 10)
+        {healthy: true}
+      end
+
       # Canonical smoke-test contract for this provider instance.
       #
       # @return [Hash, nil] smoke-test metadata

data/lib/agent_harness/providers/base.rb

@@ -368,6 +368,19 @@ module AgentHarness
         nil
       end
 
+      # Run a lightweight provider-owned preflight check before committing to a
+      # full prompt execution.
+      #
+      # Providers can override this to validate request-scoped connectivity,
+      # credentials, CLI version, or other fast-fail prerequisites.
+      #
+      # @param env [Hash] request-scoped environment overrides
+      # @param timeout [Numeric] time budget in seconds
+      # @return [Hash] with :healthy and optional :reason keys
+      def preflight_check(env:, timeout: 10)
+        {healthy: true}
+      end
+
       protected
 
       # Build CLI command - override in subclasses

data/lib/agent_harness/providers/codex.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require "json"
+require "net/http"
+require "uri"
 
 module AgentHarness
   module Providers
@@ -537,30 +539,7 @@ module AgentHarness
       end
 
       def auth_status
-        api_key = ENV["OPENAI_API_KEY"]
-        if api_key && !api_key.strip.empty?
-          if api_key.strip.start_with?("sk-")
-            return {valid: true, expires_at: nil, error: nil, auth_method: :api_key}
-          else
-            return {valid: false, expires_at: nil, error: "OPENAI_API_KEY is set but does not appear to be a valid OpenAI API key", auth_method: nil}
-          end
-        end
-
-        credentials = read_codex_credentials
-        if credentials
-          key = credentials["api_key"] || credentials["apiKey"] || credentials["OPENAI_API_KEY"]
-          if key.is_a?(String) && !key.strip.empty?
-            if key.strip.start_with?("sk-")
-              return {valid: true, expires_at: nil, error: nil, auth_method: :config_file}
-            else
-              return {valid: false, expires_at: nil, error: "Config file API key is set but does not appear to be a valid OpenAI API key", auth_method: nil}
-            end
-          end
-        end
-
-        {valid: false, expires_at: nil, error: "No OpenAI API key found. Set OPENAI_API_KEY or configure in #{codex_config_path}", auth_method: nil}
-      rescue IOError, JSON::ParserError => e
-        {valid: false, expires_at: nil, error: e.message, auth_method: nil}
+        auth_status_for_env({})
       end
 
       def health_status
@@ -576,6 +555,32 @@ module AgentHarness
         {healthy: true, message: "Codex CLI available and authenticated"}
       end
 
+      def preflight_check(env:, timeout: 10)
+        auth = auth_status_for_env(env)
+        return {healthy: false, reason: auth[:error], error_category: :authentication} unless auth[:valid]
+
+        version = codex_cli_version(env: env, timeout: timeout)
+        unless version
+          return {
+            healthy: false,
+            reason: "Codex CLI version check failed. Ensure 'codex' is installed and available in PATH.",
+            error_category: :installation
+          }
+        end
+
+        unless SUPPORTED_CLI_REQUIREMENT.satisfied_by?(version)
+          return {
+            healthy: false,
+            reason: "Unsupported Codex CLI version #{version}. Expected #{SUPPORTED_CLI_REQUIREMENT}.",
+            error_category: :installation
+          }
+        end
+
+        check_base_url_reachability(env: env, timeout: timeout)
+      rescue => e
+        {healthy: false, reason: "Codex preflight failed: #{e.message}"}
+      end
+
       def validate_config
         errors = []
 
@@ -734,6 +739,150 @@ module AgentHarness
 
       private
 
+      def auth_status_for_env(env)
+        api_key = env_fetch(env, "OPENAI_API_KEY")
+        # Fall back to process ENV when the provided env hash does not override auth keys
+        if api_key.nil? && !env.key?("OPENAI_API_KEY") && !env.key?(:OPENAI_API_KEY)
+          api_key = ENV["OPENAI_API_KEY"]
+        end
+
+        if api_key.nil? || api_key.strip.empty?
+          credentials = read_codex_credentials_for_env(env)
+          if credentials
+            key = credentials["api_key"] || credentials["apiKey"] || credentials["OPENAI_API_KEY"]
+            if key.is_a?(String) && !key.strip.empty?
+              if key.strip.start_with?("sk-")
+                return {valid: true, expires_at: nil, error: nil, auth_method: :config_file}
+              end
+
+              return {
+                valid: false,
+                expires_at: nil,
+                error: "Config file API key is set but does not appear to be a valid OpenAI API key",
+                auth_method: nil
+              }
+            end
+          end
+
+          return {
+            valid: false,
+            expires_at: nil,
+            error: "No OpenAI API key found. Set OPENAI_API_KEY or configure in #{codex_config_path_for_env(env)}",
+            auth_method: nil
+          }
+        end
+
+        if api_key.strip.start_with?("sk-")
+          {valid: true, expires_at: nil, error: nil, auth_method: :api_key}
+        else
+          {
+            valid: false,
+            expires_at: nil,
+            error: "OPENAI_API_KEY is set but does not appear to be a valid OpenAI API key",
+            auth_method: nil
+          }
+        end
+      rescue IOError, JSON::ParserError => e
+        {valid: false, expires_at: nil, error: e.message, auth_method: nil}
+      end
+
+      def codex_cli_version(env:, timeout:)
+        result = @executor.execute([self.class.binary_name, "--version"], timeout: timeout, env: env)
+        version_string = [result.stdout, result.stderr].join("\n")[/(\d+\.\d+\.\d+)/, 1]
+        return nil unless version_string
+
+        Gem::Version.new(version_string)
+      rescue # rubocop prefers bare rescue; in Ruby this catches StandardError, not Exception/SignalException
+        nil
+      end
+
+      def check_base_url_reachability(env:, timeout:)
+        uri = codex_base_url_uri(env)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = timeout
+        http.read_timeout = timeout
+        http.write_timeout = timeout if http.respond_to?(:write_timeout=)
+
+        response = http.start do |client|
+          head_response = client.request(Net::HTTP::Head.new(uri))
+
+          if http_success_or_redirect?(head_response) || http_auth_rejection?(head_response)
+            head_response
+          else
+            client.request(Net::HTTP::Get.new(uri))
+          end
+        end
+
+        return {healthy: true} if http_success_or_redirect?(response)
+
+        response_code = response.code.to_i
+        # 401/403 confirm the endpoint exists and is reachable; auth is
+        # validated separately by auth_status_for_env.
+        return {healthy: true} if http_auth_rejection?(response)
+        if invalid_base_url_response_code?(response_code)
+          return {
+            healthy: false,
+            reason: "Codex API base URL #{uri} returned HTTP #{response.code}. Check OPENAI_BASE_URL; the configured URL appears to point at an invalid API path.",
+            error_category: :configuration
+          }
+        end
+
+        {
+          healthy: false,
+          reason: "Codex API base URL #{uri} returned HTTP #{response.code}. Check OPENAI_BASE_URL, proxy configuration, and network policy.",
+          error_category: (response_code >= 500) ? :transient : :configuration
+        }
+      rescue URI::InvalidURIError => e
+        {
+          healthy: false,
+          reason: e.message.start_with?("OPENAI_BASE_URL") ? e.message : "OPENAI_BASE_URL is invalid. Check the configured URL format.",
+          error_category: :configuration
+        }
+      rescue SocketError, SystemCallError, IOError, Timeout::Error, OpenSSL::SSL::SSLError => e
+        {
+          healthy: false,
+          reason: "Codex API base URL #{env_fetch(env, "OPENAI_BASE_URL") || "https://api.openai.com"} is unreachable: #{e.message}. Check DNS, proxy settings, and network policy.",
+          error_category: :transient
+        }
+      end
+
+      def codex_base_url_uri(env)
+        raw_url = env_fetch(env, "OPENAI_BASE_URL")
+        # Only fall back to the default URL; do not read process ENV here, as the
+        # caller may have intentionally omitted OPENAI_BASE_URL to use the default.
+        raw_url = "https://api.openai.com" if raw_url.nil? || raw_url.empty?
+
+        uri = URI.parse(raw_url)
+
+        unless uri.is_a?(URI::HTTP) && uri.host && !uri.host.empty?
+          raise URI::InvalidURIError,
+            "OPENAI_BASE_URL must be an absolute HTTP or HTTPS URL (got #{raw_url.inspect})"
+        end
+
+        uri.path = "/" if uri.path.nil? || uri.path.empty?
+        uri
+      end
+
+      def env_fetch(env, key)
+        return env[key] if env.key?(key)
+        return env[key.to_sym] if env.key?(key.to_sym)
+
+        nil
+      end
+
+      def http_success_or_redirect?(response)
+        response.is_a?(Net::HTTPSuccess) || response.is_a?(Net::HTTPRedirection)
+      end
+
+      def http_auth_rejection?(response)
+        [401, 403].include?(response.code.to_i)
+      end
+
+      def invalid_base_url_response_code?(response_code)
+        [404, 410].include?(response_code)
+      end
+
       def build_streaming_event(event)
         raw_event, payload, dispatch_type = unwrap_streaming_event(event)
         return unless payload.is_a?(Hash)
@@ -1644,7 +1793,11 @@ module AgentHarness
       end
 
       def read_codex_credentials
-        path = codex_config_path
+        read_codex_credentials_for_env({})
+      end
+
+      def read_codex_credentials_for_env(env)
+        path = codex_config_path_for_env(env)
         return nil unless File.exist?(path)
 
         parsed = JSON.parse(File.read(path))
@@ -1660,7 +1813,13 @@ module AgentHarness
       end
 
       def codex_config_path
-        config_dir = ENV["CODEX_CONFIG_DIR"] || File.expand_path("~/.codex")
+        codex_config_path_for_env({})
+      end
+
+      def codex_config_path_for_env(env)
+        config_dir = env_fetch(env, "CODEX_CONFIG_DIR")
+        config_dir = ENV["CODEX_CONFIG_DIR"] if config_dir.nil? || config_dir.empty?
+        config_dir = File.expand_path("~/.codex") if config_dir.nil? || config_dir.empty?
         File.join(config_dir, "config.json")
       end
 

data/lib/agent_harness/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AgentHarness
-  VERSION = "0.14.1"
+  VERSION = "0.15.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: agent-harness
 version: !ruby/object:Gem::Version
-  version: 0.14.1
+  version: 0.15.0
 platform: ruby
 authors:
 - Bart Agapinan