agent-harness 0.5.9 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f476f6224ed25cc79c7bc9c8fb239c1df9f787cd59ff279b8c509b7d515d2727
-  data.tar.gz: c0faee9c6d5c139db019707b9174d2d25a4a2fc4a85684ce55eb7baede9fd3be
+  metadata.gz: 11c53accd50a5842f5f67a3ef1adb97e2d25539966e08cf5ed1a2659b047ce0b
+  data.tar.gz: 562f0baad4bdc24dda2dcb7d65ef93d7657be1e64e2460400afc28e7ac419afa
 SHA512:
-  metadata.gz: 113c0d8afbf5e77c6abcb1f78f0793646b72137e48a9776bfd951ee98c5412d3606628ea19ce4b105a6d8c05d1143a12a5607e1bad82462c629f6bdf3907985b
-  data.tar.gz: 4cc2f012c75314a2d096147e4bae9afbed565181878a907339410f7cfdb3ea0cff03e96704a6d3d93cd4f3825eb9f21a97d5351b852d72ec4e09b21a3359df68
+  metadata.gz: c9e7fb58eb6298e79f193b9de47c5fff995f92be7a9aff396277cf64489022d92030464b5cc08da506cc4496a4872dfc1fea1ca880f3d4cb82ad5133eaa65622
+  data.tar.gz: 40ca98102aedafdefb12d00794b721b0eb04f376eb7695034ca840ce17582abfb0741cd689f5c1a27b516acffa85399f9b58550fb516fe8e9e5652c3bb01d8d0

data/.release-please-manifest.json

@@ -1,3 +1,3 @@
 {
-  ".": "0.5.9"
+  ".": "0.6.0"
 }

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 ## [Unreleased]
 
+## [0.6.0](https://github.com/viamin/agent-harness/compare/agent-harness/v0.5.9...agent-harness/v0.6.0) (2026-04-12)
+
+
+### Features
+
+* **aider:** extract token usage via --llm-history-file ([0fff343](https://github.com/viamin/agent-harness/commit/0fff343f943d93899d0222b16ffa9832611289ff)), closes [#100](https://github.com/viamin/agent-harness/issues/100)
+
 ## [0.5.9](https://github.com/viamin/agent-harness/compare/agent-harness/v0.5.8...agent-harness/v0.5.9) (2026-04-12)
 
 

data/lib/agent_harness/providers/aider.rb

@@ -1,5 +1,9 @@
 # frozen_string_literal: true
 
+require "securerandom"
+require "shellwords"
+require "tmpdir"
+
 module AgentHarness
   module Providers
     # Aider AI coding assistant provider
@@ -192,29 +196,382 @@ module AgentHarness
         ["--restore-chat-history", session_id]
       end
 
+      def send_message(prompt:, **options)
+        log_debug("send_message_start", prompt_length: prompt.length, options: options.keys)
+
+        options = normalize_provider_runtime(options)
+        runtime = options[:provider_runtime]
+
+        options = normalize_mcp_servers(options)
+        validate_mcp_servers!(options[:mcp_servers]) if options[:mcp_servers]&.any?
+
+        llm_history_path = generate_llm_history_path
+        command = build_command(prompt, options.merge(llm_history_path: llm_history_path))
+        preparation = build_execution_preparation(options)
+        timeout = options[:timeout] || @config.timeout || default_timeout
+
+        start_time = Time.now
+        result = execute_with_timeout(
+          command,
+          timeout: timeout,
+          env: build_env(options),
+          preparation: preparation,
+          **command_execution_options(options)
+        )
+        duration = Time.now - start_time
+
+        response = parse_response(result, duration: duration, llm_history_path: llm_history_path)
+        if runtime&.model
+          response = Response.new(
+            output: response.output,
+            exit_code: response.exit_code,
+            duration: response.duration,
+            provider: response.provider,
+            model: runtime.model,
+            tokens: response.tokens,
+            metadata: response.metadata,
+            error: response.error
+          )
+        end
+
+        track_tokens(response) if response.tokens
+
+        log_debug("send_message_complete", duration: duration, tokens: response.tokens)
+
+        response
+      rescue McpConfigurationError, McpUnsupportedError, McpTransportUnsupportedError
+        raise
+      rescue => e
+        handle_error(e, prompt: prompt, options: options)
+      ensure
+        cleanup_llm_history_file!(llm_history_path)
+      end
+
       protected
 
       def build_command(prompt, options)
         cmd = [self.class.binary_name]
+        runtime = options[:provider_runtime]
 
-        # Run in non-interactive mode
         cmd << "--yes"
 
-        if @config.model && !@config.model.empty?
-          cmd += ["--model", @config.model]
+        if options[:llm_history_path]
+          cmd += ["--llm-history-file", options[:llm_history_path]]
+        end
+
+        model = runtime&.model || @config.model
+        if model && !model.empty?
+          cmd += ["--model", model]
         end
 
         if options[:session]
           cmd += session_flags(options[:session])
         end
 
+        if runtime&.flags&.any?
+          validate_runtime_flags!(runtime.flags)
+          cmd += runtime.flags
+        end
+
         cmd += ["--message", prompt]
 
         cmd
       end
 
+      def parse_response(result, duration:, llm_history_path: nil)
+        response = super(result, duration: duration)
+        tokens = parse_token_usage(result, llm_history_path: llm_history_path)
+
+        return response unless tokens
+
+        Response.new(
+          output: response.output,
+          exit_code: response.exit_code,
+          duration: response.duration,
+          provider: response.provider,
+          model: response.model,
+          tokens: tokens,
+          metadata: response.metadata,
+          error: response.error
+        )
+      end
+
       def default_timeout
-        600 # Aider can take longer
+        600
+      end
+
+      private
+
+      TOKEN_COUNT_PATTERN = /\d[\d,]*(?:\.\d+)?[kmb]?/i
+
+      TOKEN_USAGE_PATTERN =
+        /^\s*Tokens:\s*(?<input>#{TOKEN_COUNT_PATTERN})\s+sent(?:,\s*#{TOKEN_COUNT_PATTERN}\s+cache\s+\w+)*,\s*(?<output>#{TOKEN_COUNT_PATTERN})\s+received\.?(?:\s+Cost:\s+.+)?\s*$/i
+      FOOTER_COST_PATTERN = /^\s*Cost:\s+.+\s*$/i
+      RUN_SHELL_COMMAND_PATTERN = /^\s*Run shell command\?.*$/i
+      OUTPUT_STATUS_PATTERN =
+        /^\s*(?:Applied edit to|Commit\b|Committing\b|You can use \/undo\b|Added .+ to the chat\.|Removed .+ from the chat\.|Use \/help\b|Create new file\?|Allow edits to\b|Edit the files\?|Run shell command\?).*$/i
+      OUTPUT_PATH_PATTERN = /\A(?:\.\.?\/|\/|~\/)[\w.\-\/]+\z/
+      OUTPUT_DOTFILE_PATTERN = /\A\.[\w.-]+\z/
+      OUTPUT_FILENAME_PATTERN = /\A[\w.-]+\.[A-Za-z][\w.-]*\z/
+      COMMON_SHELL_COMMAND_PATTERN =
+        /\A(?:git|bundle|ruby|python\d*(?:\.\d+)?|uv|npm|yarn|pnpm|node|bash|sh|zsh|make|rake|rspec|rails|go|pytest|bin\/[\w.-]+|sed|rg|grep|find|ls|cat|cp|mv|rm|mkdir|touch|chmod|chown|docker|kubectl)\z/
+      EXECUTOR_LLM_HISTORY_TIMEOUT = 10
+
+      def generate_llm_history_path
+        return "/tmp/aider_llm_history_#{Process.pid}_#{SecureRandom.hex(8)}" if sandboxed_environment?
+
+        File.join(Dir.tmpdir, "aider_llm_history_#{Process.pid}_#{SecureRandom.hex(8)}")
+      end
+
+      def parse_token_usage(result, llm_history_path:)
+        # Aider 0.86.x writes --llm-history-file as conversation text, not JSONL.
+        # Prefer the request-local history file when it includes a token report,
+        # but fall back to captured command output because the usage summary is
+        # printed there during normal runs.
+        parse_token_usage_text(safe_read_llm_history(llm_history_path), source: :history) ||
+          parse_token_usage_text(result.stdout, source: :output) ||
+          parse_token_usage_text(result.stderr, source: :output)
+      end
+
+      def read_llm_history(path)
+        return read_executor_llm_history(path) if sandboxed_environment?
+        return nil unless path && File.exist?(path) && !File.zero?(path)
+
+        content = File.read(path)
+        return nil if content.strip.empty?
+
+        content
+      end
+
+      def safe_read_llm_history(path)
+        read_llm_history(path)
+      rescue => e
+        log_debug("llm_history_parse_error", error: e.message)
+        nil
+      end
+
+      def parse_token_usage_text(content, source: :output)
+        return nil if content.nil? || content.strip.empty?
+
+        match = if source == :history
+          extract_history_token_usage_match(content)
+        else
+          extract_output_token_usage_match(content)
+        end
+        return nil unless match
+
+        input = parse_token_count(match[:input])
+        output = parse_token_count(match[:output])
+
+        {input: input, output: output, total: input + output}
+      end
+
+      def extract_history_token_usage_match(content)
+        lines = content.lines
+
+        lines.each_index.reverse_each do |index|
+          match = TOKEN_USAGE_PATTERN.match(lines[index])
+          next unless match
+          next unless history_token_usage_footer_line?(lines, index)
+
+          return match
+        end
+
+        nil
+      end
+
+      def extract_output_token_usage_match(content)
+        lines = content.lines
+
+        lines.each_index.reverse_each do |index|
+          match = TOKEN_USAGE_PATTERN.match(lines[index])
+          next unless match
+          next unless output_token_usage_footer_line?(lines, index)
+
+          return match
+        end
+
+        nil
+      end
+
+      def history_token_usage_footer_line?(lines, index)
+        footer_prefix?(lines, index) && footer_suffix?(lines, index)
+      end
+
+      def output_token_usage_footer_line?(lines, index)
+        footer_prefix?(lines, index) && output_footer_suffix?(lines, index)
+      end
+
+      def footer_prefix?(lines, index)
+        block_start = index
+        while block_start.positive? && TOKEN_USAGE_PATTERN.match?(lines[block_start - 1])
+          block_start -= 1
+        end
+
+        return false if block_start.zero?
+
+        lines[block_start - 1].strip.empty?
+      end
+
+      def footer_suffix?(lines, index)
+        lines[(index + 1)..].to_a.all? do |line|
+          stripped = line.strip
+          stripped.empty? || TOKEN_USAGE_PATTERN.match?(line) || FOOTER_COST_PATTERN.match?(line)
+        end
+      end
+
+      def output_footer_suffix?(lines, index)
+        suffix_lines = lines[(index + 1)..].to_a
+        shell_prompt_index = suffix_lines.index { |line| RUN_SHELL_COMMAND_PATTERN.match?(line) }
+
+        suffix_lines.each_with_index.all? do |line, line_index|
+          stripped = line.strip
+          stripped.empty? ||
+            TOKEN_USAGE_PATTERN.match?(line) ||
+            FOOTER_COST_PATTERN.match?(line) ||
+            OUTPUT_STATUS_PATTERN.match?(line) ||
+            output_path_footer_line?(stripped) ||
+            output_command_footer_line?(line, line_index, shell_prompt_index)
+        end
+      end
+
+      def output_path_footer_line?(line)
+        OUTPUT_PATH_PATTERN.match?(line) ||
+          OUTPUT_DOTFILE_PATTERN.match?(line) ||
+          OUTPUT_FILENAME_PATTERN.match?(line) ||
+          (line.include?("/") && line.match?(/\A[\w.\-\/]+\z/))
+      end
+
+      def output_command_footer_line?(line, line_index, shell_prompt_index)
+        return false unless shell_prompt_index && line_index < shell_prompt_index
+
+        stripped = line.strip
+        return false if stripped.end_with?(".", "?", "!")
+        return false if stripped.empty?
+
+        tokens = shell_command_footer_tokens(stripped)
+        return false if tokens.empty?
+        command = tokens.first
+        return false unless command_invocation_token?(command)
+        return single_token_command_footer?(command) if tokens.length == 1
+        return false unless command_line_token?(command, tokens[1..])
+
+        tokens[1..].all? { |token| command_argument_token?(token) }
+      end
+
+      def shell_command_footer_tokens(line)
+        Shellwords.shellsplit(line.sub(/\A[$>#]\s*/, ""))
+      rescue ArgumentError
+        []
+      end
+
+      def command_token?(token)
+        token.match?(/\A[a-z0-9_][\w.\/~:-]*\z/) && token.match?(/[a-z]/)
+      end
+
+      def command_invocation_token?(token)
+        command_token?(token) || executable_path_token?(token)
+      end
+
+      def executable_path_token?(token)
+        token.match?(%r{\A(?:\.\.?/|/|~/)[\w.+%:@=-][\w./+%:@~=-]*\z})
+      end
+
+      def command_line_token?(token, arguments)
+        command_invocation_token?(token) &&
+          (COMMON_SHELL_COMMAND_PATTERN.match?(token) ||
+            executable_path_token?(token) ||
+            command_footer_shell_like_arguments?(arguments))
+      end
+
+      def single_token_command_footer?(token)
+        COMMON_SHELL_COMMAND_PATTERN.match?(token) || executable_path_token?(token)
+      end
+
+      def command_footer_shell_like_arguments?(arguments)
+        arguments.any? do |argument|
+          argument.match?(%r{\A(?:&&|\|\|?|\||[<>]|>>|&>|2>)\z}) ||
+            argument.start_with?("-", "./", "../", "/", "~/") ||
+            argument.include?("/")
+        end
+      end
+
+      def command_argument_token?(token)
+        !token.empty? && !token.match?(/[[:cntrl:]]/)
+      end
+
+      def parse_token_count(value)
+        normalized = value.delete(",").downcase
+        multiplier = case normalized[-1]
+        when "k" then 1_000
+        when "m" then 1_000_000
+        when "b" then 1_000_000_000
+        else 1
+        end
+        normalized = normalized[0...-1] if multiplier > 1
+
+        (normalized.to_f * multiplier).round
+      end
+
+      def cleanup_llm_history_file!(path)
+        return unless path
+
+        return cleanup_executor_llm_history_file!(path) if sandboxed_environment?
+
+        File.delete(path) if File.exist?(path)
+      rescue => e
+        log_debug("llm_history_cleanup_error", error: e.message)
+        nil
+      end
+
+      def validate_runtime_flags!(flags)
+        invalid_flags = reserved_runtime_flags(flags)
+        return if invalid_flags.empty?
+
+        raise ArgumentError,
+          "Aider provider_runtime.flags cannot override provider-managed flags: " \
+          "#{invalid_flags.join(", ")}"
+      end
+
+      def reserved_runtime_flags(flags)
+        flags.each_with_index.filter_map do |flag, index|
+          next unless reserved_runtime_flag?(flag)
+
+          if flag == "--llm-history-file" && flags[index + 1]
+            "#{flag} #{flags[index + 1]}"
+          else
+            flag
+          end
+        end.uniq
+      end
+
+      def reserved_runtime_flag?(flag)
+        flag == "--llm-history-file" || flag.start_with?("--llm-history-file=")
+      end
+
+      def read_executor_llm_history(path)
+        return nil unless path
+
+        result = @executor.execute(
+          ["sh", "-lc", "if [ -s #{Shellwords.escape(path)} ]; then cat #{Shellwords.escape(path)}; fi"],
+          timeout: EXECUTOR_LLM_HISTORY_TIMEOUT
+        )
+        return nil unless result.success?
+
+        content = result.stdout
+        return nil if content.to_s.strip.empty?
+
+        content
+      end
+
+      def cleanup_executor_llm_history_file!(path)
+        @executor.execute(
+          ["sh", "-lc", "rm -f -- #{Shellwords.escape(path)}"],
+          timeout: EXECUTOR_LLM_HISTORY_TIMEOUT
+        )
+      rescue => e
+        log_debug("llm_history_cleanup_error", error: e.message)
+        nil
       end
     end
   end

data/lib/agent_harness/providers/codex.rb

@@ -10,6 +10,29 @@ module AgentHarness
     class Codex < Base
       SUPPORTED_CLI_VERSION = "0.116.0"
       SUPPORTED_CLI_REQUIREMENT = Gem::Requirement.new(">= #{SUPPORTED_CLI_VERSION}", "< 0.117.0").freeze
+      OAUTH_REFRESH_FAILURE_PATTERNS = [
+        /refresh_token_reused/i,
+        /failed to refresh token\b.*\b401\b/im,
+        /failed to refresh token\b.*unauthorized/im,
+        /failed to refresh token\b.*\binvalid_client\b/im,
+        /failed to refresh token\b.*\binvalid_grant\b/im,
+        /failed to refresh token\b.*invalid.*refresh.*token/im,
+        /failed to refresh token\b.*refresh.*token.*invalid/im,
+        /your access token could not be refreshed because\b.*\b401\b/im,
+        /your access token could not be refreshed because\b.*unauthorized/im,
+        /your access token could not be refreshed because\b.*\binvalid_client\b/im,
+        /your access token could not be refreshed because\b.*\binvalid_grant\b/im,
+        /your access token could not be refreshed because\b.*invalid.*refresh.*token/im,
+        /your access token could not be refreshed because\b.*refresh.*token.*invalid/im,
+        /your access token could not be refreshed because\s+your refresh token .*already (?:been )?used/im,
+        /refresh token .*already (?:been )?used/im
+      ].freeze
+      OAUTH_REFRESH_TRANSIENT_PATTERNS = [
+        /your access token could not be refreshed because\s+(?:the\s+)?auth(?:entication)? service(?:\s+(?:is|was))?\s+(?:temporarily\s+)?unavailable/im,
+        /your access token could not be refreshed because .*connection.*error/im,
+        /failed to refresh token\b.*connection.*error/im,
+        /failed to refresh token\b.*service(?:\s+(?:is|was))?\s+(?:temporarily\s+)?unavailable/im
+      ].freeze
 
       class << self
         def provider_name
@@ -171,15 +194,26 @@ module AgentHarness
       end
 
       def error_patterns
-        COMMON_ERROR_PATTERNS.merge(
-          auth_expired: COMMON_ERROR_PATTERNS[:auth_expired] + [/\b401\b/, /incorrect.*api.*key/i],
-          transient: COMMON_ERROR_PATTERNS[:transient] + [/connection.*reset/i],
+        {
+          rate_limited: COMMON_ERROR_PATTERNS[:rate_limited],
+          timeout: [
+            /your access token could not be refreshed.*(?:timeout|timed.?out)/im,
+            /failed to refresh token\b.*(?:timeout|timed.?out)/im
+          ],
+          transient: COMMON_ERROR_PATTERNS[:transient] + [
+            /connection.*reset/i
+          ] + OAUTH_REFRESH_TRANSIENT_PATTERNS,
+          auth_expired: COMMON_ERROR_PATTERNS[:auth_expired] + [
+            /\b401\b/,
+            /incorrect.*api.*key/i
+          ] + OAUTH_REFRESH_FAILURE_PATTERNS,
+          quota_exceeded: COMMON_ERROR_PATTERNS[:quota_exceeded],
           sandbox_failure: [
             /bwrap.*no permissions/i,
             /no permissions to create a new namespace/i,
             /unprivileged.*namespace/i
           ]
-        )
+        }
       end
 
       def auth_status

data/lib/agent_harness/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AgentHarness
-  VERSION = "0.5.9"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: agent-harness
 version: !ruby/object:Gem::Version
-  version: 0.5.9
+  version: 0.6.0
 platform: ruby
 authors:
 - Bart Agapinan