ask-agent 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4812b86203eb5a12eb03138de1e50fe8f0a80532f545bb683dcf051f2729c088
-  data.tar.gz: ba9e999b2aedae3e9c0f55067a35e3d3f29f828717a8f7129c0e0037bd518a33
+  metadata.gz: 3d59df39cdd8e4f3aef8b71f9c63fee0c571735d03f409667d3c3b9fed964e59
+  data.tar.gz: e6546221a9c0d609d6b0bcaa4295d6c285ebf6e9e2750e2ee69715cda257017c
 SHA512:
-  metadata.gz: a5b049b05acf1a82334a2169a708d60a6511db2d790ec5db3f30b9b4775a561eb204b8fdb209b60d095edeba43bd27f1572a909d78118eba9f295943625da370
-  data.tar.gz: 54ba99bbacd6152c52196f276f28559d2a13e09b048338c4c5198e9ab01c8f2cc9e5eb57c2a957b1ba6dea4410726c3eba644104b66d767729e908a10c366133
+  metadata.gz: f0569cd462c103a755d1a05606150ef229504cc090a8b0dfbbd1bfb79dba8f67cac82d1a1aaa1039ce75294b3aaf55d9f68478df5de1f878b197f5bc6d12335a
+  data.tar.gz: 970cabb7492914f0874a76c04ed72f04d9882020a900db3a8d07ca043097432a4e25bfa2c46e3678d7db67cd98f8d25a03d97c64dab1157a2278f97bb23132cd

data/CHANGELOG.md

@@ -1,3 +1,43 @@
+## [0.4.0] — 2026-07-17
+
+### Added
+
+- **Agent testing framework** — `Ask::Agent::Test` provides deterministic agent behavior tests without calling real LLMs. Stub tool calls and text responses, assert which tools were called, in what order, and verify the final response. No flaky tests, no API keys, no cost.
+
+  ```ruby
+  require "ask/agent/test"
+
+  class MyAgentTest < Minitest::Test
+    include Ask::Agent::Test::Assertions
+
+    def setup
+      @session = Ask::Agent::Session.new(model: "gpt-4o", tools: [my_tool])
+      @session.test_mode
+    end
+
+    def test_calls_search_tool
+      @session.stub_tool_call("search", query: "weather")
+      @session.stub_text("Sunny")
+      @session.run("What's the weather?")
+      assert_called_tool "search"
+      assert_final_response /Sunny/
+      assert_no_unused_stubs
+    end
+  end
+  ```
+
+  Assertions: `assert_called_tool`, `refute_called_tool`, `assert_tool_order`, `assert_final_response`, `assert_no_unused_stubs`.
+
+## [0.3.1] — 2026-07-17
+
+### Added
+
+- **Rate-limit aware retry in Chat** — `Chat#ask` retries up to 3 times on `RateLimitError`, using `retry_after` from the error when available, otherwise exponential backoff with jitter.
+
+### Fixed
+
+- **`retryable_error_name?` in ToolExecutor** — fixed duplicate `Ask::RateLimitError` and non-existent `Ask::ServiceUnavailableError`. Now uses class hierarchy matching so subclasses are also retried. (Backport from LiteLLM error classification.)
+
 ## [0.3.0] — 2026-07-17
 
 ### Added

data/lib/ask/agent/chat.rb

@@ -49,29 +49,9 @@ module Ask
         provider_schema = @schema&.respond_to?(:to_json_schema) ? @schema.to_json_schema : @schema
         provider_params = @extra_params || {}
 
-        result = provider.chat(
-          @messages.map(&:to_h),
-          model: provider_model,
-          tools: provider_tools,
-          temperature: provider_temp,
-          stream: stream,
-          schema: provider_schema,
-          **provider_params
-        ) do |raw_chunk|
-          next unless block_given?
-
-          accumulate_tool_calls(raw_chunk, calls_acc)
-
-          yield ChatChunk.new(
-            content: raw_chunk.content,
-            tool_calls: build_current_tool_calls(calls_acc),
-            thinking: raw_chunk.respond_to?(:thinking) ? raw_chunk.thinking : nil,
-            input_tokens: nil,
-            output_tokens: nil
-          )
-        end
+        result = chat_with_retry(stream, calls_acc, &block)
 
-        response_msg = if stream
+        response_msg = if result.respond_to?(:chunks)
           build_stream_response(result, calls_acc)
         else
           build_response(result)
@@ -124,10 +104,12 @@ module Ask
         @messages.clear
       end
 
+      attr_writer :test_provider
+
       private
 
       def provider
-        @provider ||= build_provider
+        @test_provider || @provider ||= build_provider
       end
 
       def build_provider
@@ -234,6 +216,41 @@ module Ask
         nil
       end
 
+      MAX_CHAT_RETRIES = 3
+
+      def chat_with_retry(stream, calls_acc, &block)
+        MAX_CHAT_RETRIES.times do |attempt|
+          begin
+            return provider.chat(
+              @messages.map(&:to_h),
+              model: @model_id,
+              tools: @tools.map { |t| Ask::ToolDef.from_tool(t) },
+              temperature: @temperature,
+              stream: stream,
+              schema: @schema&.respond_to?(:to_json_schema) ? @schema.to_json_schema : @schema,
+              **(@extra_params || {})
+            ) do |raw_chunk|
+              next unless block
+
+              accumulate_tool_calls(raw_chunk, calls_acc)
+
+              block.call(ChatChunk.new(
+                content: raw_chunk.content,
+                tool_calls: build_current_tool_calls(calls_acc),
+                thinking: raw_chunk.respond_to?(:thinking) ? raw_chunk.thinking : nil,
+                input_tokens: nil,
+                output_tokens: nil
+              ))
+            end
+          rescue Ask::RateLimitError => e
+            raise if attempt >= MAX_CHAT_RETRIES - 1
+
+            delay = e.retry_after || ((2 ** attempt) + rand(0.0..1.0))
+            sleep(delay)
+          end
+        end
+      end
+
       def emit_instrumentation(stream, response_msg)
         return unless defined?(Ask::Instrumentation)
 

data/lib/ask/agent/test.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module Ask
+  module Agent
+    module Test
+      # Holds stubs, tracks called tools and final response for a test session.
+      class Mode
+        attr_reader :stubs, :called_tools, :final_response
+
+        def initialize
+          @stubs = []
+          @called_tools = []
+          @final_response = nil
+        end
+
+        def stub_tool_call(name, arguments = {})
+          @stubs << { type: :tool_call, name: name.to_s, arguments: arguments }
+        end
+
+        def stub_text(content)
+          @stubs << { type: :text, content: content }
+        end
+
+        def record_tool_call(name)
+          @called_tools << name
+        end
+
+        def record_final_response(response)
+          @final_response = response
+        end
+
+        def unused_stubs
+          @stubs
+        end
+      end
+
+      # Stub provider that returns canned responses instead of calling an LLM.
+      class StubProvider
+        def initialize(mode)
+          @mode = mode
+        end
+
+        def chat(messages, model: nil, tools: nil, temperature: nil, stream: nil, schema: nil, **params)
+          stub = @mode.stubs.shift or raise "No more stubs available. Unconsumed messages: #{messages.length}"
+
+          case stub[:type]
+          when :tool_call
+            Ask::Message.new(
+              role: :assistant,
+              content: nil,
+              tool_calls: [{
+                id: "call_test_#{SecureRandom.hex(4)}",
+                type: "function",
+                name: stub[:name],
+                arguments: JSON.generate(stub[:arguments])
+              }],
+              metadata: { input_tokens: 10, output_tokens: 5 }
+            )
+          when :text
+            Ask::Message.new(
+              role: :assistant,
+              content: stub[:content],
+              metadata: { input_tokens: 10, output_tokens: 5 }
+            )
+          else
+            raise "Unknown stub type: #{stub[:type].inspect}"
+          end
+        end
+
+        def headers
+          {}
+        end
+      end
+
+      # Methods added to Session when in test mode.
+      module SessionOverride
+        def test_mode
+          return @test_mode if @test_mode
+
+          @test_mode = Mode.new
+          @chat&.test_provider = StubProvider.new(@test_mode) if @chat
+          @test_mode
+        end
+
+        def stub_tool_call(name, arguments = {})
+          test_mode.stub_tool_call(name, arguments)
+        end
+
+        def stub_text(content)
+          test_mode.stub_text(content)
+        end
+
+        def called_tool?(name)
+          test_mode.called_tools.include?(name.to_s)
+        end
+
+        private
+
+        def build_chat(model, system_prompt, tools, **chat_options)
+          if @test_mode
+            chat = Ask::Agent::Chat.new(model: model, tools: tools, **chat_options)
+            chat.with_instructions(system_prompt) if system_prompt
+            chat.test_provider = StubProvider.new(@test_mode)
+            chat
+          else
+            super
+          end
+        end
+
+        def emit(event)
+          super
+          return unless @test_mode
+
+          case event
+          when Ask::Agent::Events::ToolExecutionStart
+            @test_mode.record_tool_call(event.name)
+          when Ask::Agent::Events::SessionEnd
+            @test_mode.record_final_response(event.result)
+          end
+        end
+
+        public :emit
+      end
+
+      # Test assertions to include in Minitest test classes.
+      module Assertions
+        def session
+          @session || raise("@session not set — define it in setup")
+        end
+
+        def assert_called_tool(name, msg = nil)
+          assert session.called_tool?(name),
+                 msg || "Expected tool #{name.inspect} to be called. Called: #{session.test_mode.called_tools.inspect}"
+        end
+
+        def refute_called_tool(name, msg = nil)
+          refute session.called_tool?(name),
+                 msg || "Expected tool #{name.inspect} not to be called. Called: #{session.test_mode.called_tools.inspect}"
+        end
+
+        def assert_tool_order(names, msg = nil)
+          expected = names.map(&:to_s)
+          called = session.test_mode.called_tools
+          matched = called.first(expected.length)
+          assert_equal expected, matched,
+                       msg || "Tool call order mismatch. Expected: #{expected.inspect}, Got: #{matched.inspect}"
+        end
+
+        def assert_final_response(match, msg = nil)
+          assert_match match, session.test_mode.final_response.to_s, msg
+        end
+
+        def assert_no_unused_stubs(msg = nil)
+          assert_empty session.test_mode.unused_stubs,
+                       msg || "Unused stubs: #{session.test_mode.unused_stubs.inspect}"
+        end
+      end
+
+      Ask::Agent::Session.prepend(SessionOverride)
+    end
+  end
+end

data/lib/ask/agent/tool_executor.rb

@@ -182,10 +182,15 @@ module Ask
       end
 
       def retryable_error_name?(error_name)
-        retryable = %w[Timeout::Error Errno::ETIMEDOUT
-                       Ask::RateLimitError Ask::ServerError
-                       Ask::RateLimitError Ask::ServiceUnavailableError]
-        retryable.include?(error_name)
+        return false unless error_name
+
+        klass = Object.const_get(error_name) rescue nil
+        return false unless klass
+
+        klass <= Ask::RateLimitError ||
+        klass <= Ask::ServerError ||
+        klass <= Ask::ServiceUnavailable ||
+        %w[Timeout::Error Errno::ETIMEDOUT].include?(error_name)
       end
 
       def critical_error?(error_class_name)

data/lib/ask/agent/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module Agent
-    VERSION = "0.3.0"
+    VERSION = "0.4.0"
   end
 end

data/lib/ask/agent.rb

@@ -56,3 +56,6 @@ require_relative "agent/configuration"
 require_relative "agent/meta_agent"
 require_relative "agent/persistence/base"
 require_relative "agent/persistence/in_memory"
+
+# Test helpers (loaded on demand)
+autoload :Test, "ask/agent/test"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-agent
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Kaka Ruto
@@ -149,6 +149,7 @@ files:
 - lib/ask/agent/reflector.rb
 - lib/ask/agent/session.rb
 - lib/ask/agent/telemetry.rb
+- lib/ask/agent/test.rb
 - lib/ask/agent/tool_abort_controller.rb
 - lib/ask/agent/tool_executor.rb
 - lib/ask/agent/version.rb