ask-agent 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 26c124febbf2652616754dd539cc8df5d24afe5fbe8c03381f760598578a051f
-  data.tar.gz: 50d87794d60901a85da426f50a1a619807faf96bbe1fedb58145dd2d5d7fbeb1
+  metadata.gz: 3d59df39cdd8e4f3aef8b71f9c63fee0c571735d03f409667d3c3b9fed964e59
+  data.tar.gz: e6546221a9c0d609d6b0bcaa4295d6c285ebf6e9e2750e2ee69715cda257017c
 SHA512:
-  metadata.gz: 5927098dcbd1fe84faa0cf2a7cddbc63e544a7113e47c61f972044e598053758417edce0e63d9a55d7922fbbc65ad9f552ef5c3fb1c07ac71ba1b718d1cbb815
-  data.tar.gz: 0436ac54f1dcd57ac06706d7e6a8ad5030574341170e0c7e99ca174d3dd94341651142c78bf5d138be1f99a978d71eb21f03a57df1ef4b6cb038ebb22778a637
+  metadata.gz: f0569cd462c103a755d1a05606150ef229504cc090a8b0dfbbd1bfb79dba8f67cac82d1a1aaa1039ce75294b3aaf55d9f68478df5de1f878b197f5bc6d12335a
+  data.tar.gz: 970cabb7492914f0874a76c04ed72f04d9882020a900db3a8d07ca043097432a4e25bfa2c46e3678d7db67cd98f8d25a03d97c64dab1157a2278f97bb23132cd

data/CHANGELOG.md

@@ -1,3 +1,33 @@
+## [0.4.0] — 2026-07-17
+
+### Added
+
+- **Agent testing framework** — `Ask::Agent::Test` provides deterministic agent behavior tests without calling real LLMs. Stub tool calls and text responses, assert which tools were called, in what order, and verify the final response. No flaky tests, no API keys, no cost.
+
+  ```ruby
+  require "ask/agent/test"
+
+  class MyAgentTest < Minitest::Test
+    include Ask::Agent::Test::Assertions
+
+    def setup
+      @session = Ask::Agent::Session.new(model: "gpt-4o", tools: [my_tool])
+      @session.test_mode
+    end
+
+    def test_calls_search_tool
+      @session.stub_tool_call("search", query: "weather")
+      @session.stub_text("Sunny")
+      @session.run("What's the weather?")
+      assert_called_tool "search"
+      assert_final_response /Sunny/
+      assert_no_unused_stubs
+    end
+  end
+  ```
+
+  Assertions: `assert_called_tool`, `refute_called_tool`, `assert_tool_order`, `assert_final_response`, `assert_no_unused_stubs`.
+
 ## [0.3.1] — 2026-07-17
 
 ### Added

data/lib/ask/agent/chat.rb

@@ -51,7 +51,7 @@ module Ask
 
         result = chat_with_retry(stream, calls_acc, &block)
 
-        response_msg = if stream
+        response_msg = if result.respond_to?(:chunks)
           build_stream_response(result, calls_acc)
         else
           build_response(result)
@@ -104,10 +104,12 @@ module Ask
         @messages.clear
       end
 
+      attr_writer :test_provider
+
       private
 
       def provider
-        @provider ||= build_provider
+        @test_provider || @provider ||= build_provider
       end
 
       def build_provider

data/lib/ask/agent/test.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module Ask
+  module Agent
+    module Test
+      # Holds stubs, tracks called tools and final response for a test session.
+      class Mode
+        attr_reader :stubs, :called_tools, :final_response
+
+        def initialize
+          @stubs = []
+          @called_tools = []
+          @final_response = nil
+        end
+
+        def stub_tool_call(name, arguments = {})
+          @stubs << { type: :tool_call, name: name.to_s, arguments: arguments }
+        end
+
+        def stub_text(content)
+          @stubs << { type: :text, content: content }
+        end
+
+        def record_tool_call(name)
+          @called_tools << name
+        end
+
+        def record_final_response(response)
+          @final_response = response
+        end
+
+        def unused_stubs
+          @stubs
+        end
+      end
+
+      # Stub provider that returns canned responses instead of calling an LLM.
+      class StubProvider
+        def initialize(mode)
+          @mode = mode
+        end
+
+        def chat(messages, model: nil, tools: nil, temperature: nil, stream: nil, schema: nil, **params)
+          stub = @mode.stubs.shift or raise "No more stubs available. Unconsumed messages: #{messages.length}"
+
+          case stub[:type]
+          when :tool_call
+            Ask::Message.new(
+              role: :assistant,
+              content: nil,
+              tool_calls: [{
+                id: "call_test_#{SecureRandom.hex(4)}",
+                type: "function",
+                name: stub[:name],
+                arguments: JSON.generate(stub[:arguments])
+              }],
+              metadata: { input_tokens: 10, output_tokens: 5 }
+            )
+          when :text
+            Ask::Message.new(
+              role: :assistant,
+              content: stub[:content],
+              metadata: { input_tokens: 10, output_tokens: 5 }
+            )
+          else
+            raise "Unknown stub type: #{stub[:type].inspect}"
+          end
+        end
+
+        def headers
+          {}
+        end
+      end
+
+      # Methods added to Session when in test mode.
+      module SessionOverride
+        def test_mode
+          return @test_mode if @test_mode
+
+          @test_mode = Mode.new
+          @chat&.test_provider = StubProvider.new(@test_mode) if @chat
+          @test_mode
+        end
+
+        def stub_tool_call(name, arguments = {})
+          test_mode.stub_tool_call(name, arguments)
+        end
+
+        def stub_text(content)
+          test_mode.stub_text(content)
+        end
+
+        def called_tool?(name)
+          test_mode.called_tools.include?(name.to_s)
+        end
+
+        private
+
+        def build_chat(model, system_prompt, tools, **chat_options)
+          if @test_mode
+            chat = Ask::Agent::Chat.new(model: model, tools: tools, **chat_options)
+            chat.with_instructions(system_prompt) if system_prompt
+            chat.test_provider = StubProvider.new(@test_mode)
+            chat
+          else
+            super
+          end
+        end
+
+        def emit(event)
+          super
+          return unless @test_mode
+
+          case event
+          when Ask::Agent::Events::ToolExecutionStart
+            @test_mode.record_tool_call(event.name)
+          when Ask::Agent::Events::SessionEnd
+            @test_mode.record_final_response(event.result)
+          end
+        end
+
+        public :emit
+      end
+
+      # Test assertions to include in Minitest test classes.
+      module Assertions
+        def session
+          @session || raise("@session not set — define it in setup")
+        end
+
+        def assert_called_tool(name, msg = nil)
+          assert session.called_tool?(name),
+                 msg || "Expected tool #{name.inspect} to be called. Called: #{session.test_mode.called_tools.inspect}"
+        end
+
+        def refute_called_tool(name, msg = nil)
+          refute session.called_tool?(name),
+                 msg || "Expected tool #{name.inspect} not to be called. Called: #{session.test_mode.called_tools.inspect}"
+        end
+
+        def assert_tool_order(names, msg = nil)
+          expected = names.map(&:to_s)
+          called = session.test_mode.called_tools
+          matched = called.first(expected.length)
+          assert_equal expected, matched,
+                       msg || "Tool call order mismatch. Expected: #{expected.inspect}, Got: #{matched.inspect}"
+        end
+
+        def assert_final_response(match, msg = nil)
+          assert_match match, session.test_mode.final_response.to_s, msg
+        end
+
+        def assert_no_unused_stubs(msg = nil)
+          assert_empty session.test_mode.unused_stubs,
+                       msg || "Unused stubs: #{session.test_mode.unused_stubs.inspect}"
+        end
+      end
+
+      Ask::Agent::Session.prepend(SessionOverride)
+    end
+  end
+end

data/lib/ask/agent/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module Agent
-    VERSION = "0.3.1"
+    VERSION = "0.4.0"
   end
 end

data/lib/ask/agent.rb

@@ -56,3 +56,6 @@ require_relative "agent/configuration"
 require_relative "agent/meta_agent"
 require_relative "agent/persistence/base"
 require_relative "agent/persistence/in_memory"
+
+# Test helpers (loaded on demand)
+autoload :Test, "ask/agent/test"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-agent
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Kaka Ruto
@@ -149,6 +149,7 @@ files:
 - lib/ask/agent/reflector.rb
 - lib/ask/agent/session.rb
 - lib/ask/agent/telemetry.rb
+- lib/ask/agent/test.rb
 - lib/ask/agent/tool_abort_controller.rb
 - lib/ask/agent/tool_executor.rb
 - lib/ask/agent/version.rb