ask-agent 0.14.0 → 0.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c80f27cbdec5b3808533dba4861762e19e4ef81c084be099a4a7eb0b2169170e
-  data.tar.gz: 34e5388035a8f216df468039bf8c7e304296d7661832e8109df4d76fbb527193
+  metadata.gz: 44a7d741bb255935ebfa4790a7da3d36dc4b4200ab8ebd1e061f8d31b194c798
+  data.tar.gz: ad33f40cffc85328e4cc753291db6fa5730fd7edffc52fcf4f96b34eade7a863
 SHA512:
-  metadata.gz: bb1bf1b6aac7a61a650291f9dcabd781c0a664cb09d97c7f2cc2fcdcfaa035b71adef2c423ebacfaae0de04d0524b4dff649f503921006837e0644d9a9f86615
-  data.tar.gz: d718885adebbaf33f2b9a2b95704968494d14e3080746a58da37c9518af475ac09103357014e5bea98339d56150ce63a162c6ac9a9c8ae4cffa07f13716b8793
+  metadata.gz: 3e8c63b1465e8090d050e92b03773ec1a38caf9b2c4b22abbb08f5b19007e50317d5befa058343ea6c08e11450ad0a5c5c2f99609ef64d5f00ef9855893f5086
+  data.tar.gz: 02e39e5fdb5ecdcd91dc57572a0594d90155534ed7dc282b1029a4041fe7943b94e45c5066e1c86c75b1cb0efb7824a5eb522dcd650c81ad41cfd5177dfafc98

data/CHANGELOG.md

@@ -1,3 +1,132 @@
+## [0.19.0] — 2026-07-26
+
+### Added
+
+- **`Ask::Agent::SubAgent.new("definition_name")` — create sub-agents from
+  filesystem definitions**. Passing a string looks up an agent definition
+  by name (same convention as `Ask::Agent.new("name")`), reading model,
+  tools, instructions, and other settings from the definition files.
+
+  ```ruby
+  # agents/web_search/agent.rb defines model, tools, instructions
+  search = Ask::Agent::SubAgent.new("web_search")
+
+  coordinator = Ask::Agent::Session.new(
+    model: "gpt-4o",
+    tools: [search, Ask::Tools::Shell::Bash]
+  )
+  ```
+
+- **VCR-based integration tests** for SubAgent. Real API calls are recorded
+  and replayed via VCR cassettes. Run with `OPENAI_API_KEY` set to record,
+  or without to replay existing cassettes.
+
+### Changed
+
+- `Ask::Agent::SubAgent.new(name:, ...)` now supports `provider:` parameter
+  for provider-specific sub-agents.
+
+## [0.18.0] — 2026-07-26
+
+### Added
+
+- **`Ask::Agent::SubAgent` — delegate tasks to a specialized sub-agent tool**.
+  A self-contained tool class that satisfies the tool duck type (`name`,
+  `description`, `params_schema`, `call`). When the coordinator LLM calls it,
+  a fresh sub-agent session runs independently with its own model, tools,
+  and instructions.
+
+  ```ruby
+  search = Ask::Agent::SubAgent.new(
+    name: "web_search",
+    description: "Search the web for current information",
+    model: "gpt-4o-mini",
+    tools: [MyApp::Tools::WebSearch],
+    system_prompt: "You are a research assistant."
+  )
+
+  coordinator = Ask::Agent::Session.new(
+    model: "gpt-4o",
+    tools: [search, Ask::Tools::Shell::Bash]
+  )
+
+  coordinator.run("What's the latest Rails release and how stable is it?")
+  ```
+
+### Removed
+
+- **`Ask::Agent.sub_agent_tool`** factory method — replaced by the
+  `Ask::Agent::SubAgent` class directly. The class IS the tool, no
+  factory or wrapper needed.
+
+## [0.17.0] — 2026-07-26
+
+### Added
+
+- Bump ask-tools dependency for `Ask::Tools::SubAgent` support
+
+### Added
+
+- **Independent Evaluator — `Ask::Agent::Evaluator`** — Generator/evaluator separation.
+  A separate model (configured independently from the session's model) judges the
+  agent's output against a structured rubric before delivery. This prevents the
+  anti-pattern of a model grading its own work.
+
+  ```ruby
+  # Evaluate with a different model — the recommended approach
+  session = Ask::Agent::Session.new(
+    model: "gpt-4o",
+    evaluator: { model: "claude-sonnet-4", goal: "Write an email validator" }
+  )
+  session.run("Write email validation")
+  ```
+
+  Three verdicts:
+  - **`:accept`** — output meets the goal, passes through to reflection
+  - **`:revise`** — evaluator provides actionable feedback; session runs another
+    turn with the feedback injected into system context
+  - **`:block`** — output is fundamentally wrong; session returns blocked message
+    and emits `Events::EvaluationBlocked`
+
+  Rubric dimensions (each scored 0-2):
+  - correctness (3× weight), completeness (2×), verification (2×), scope (1×), clarity (1×)
+
+  Custom rubrics supported:
+  ```ruby
+  evaluator = Ask::Agent::Evaluator.new(
+    model: "claude-sonnet-4",
+    rubric: [
+      Ask::Agent::Evaluator::Dimension.new(name: "performance", description: "Is it fast?", weight: 2)
+    ]
+  )
+  ```
+
+- **`evaluator:` option on `Session`** — accepts `true`, `false`/`nil`, or a Hash:
+  - `evaluator: true` — uses `config.default_evaluator_model` (falls back to the
+    session's model, though using a different model is strongly recommended)
+  - `evaluator: { model: "claude-sonnet-4", goal: "Custom goal" }` — explicit config
+  - `evaluator: false` (default) — no evaluation, backward compatible
+
+- **`default_evaluator_model` config option** — set a global default:
+  ```ruby
+  Ask::Agent.configure do |c|
+    c.default_evaluator_model = "claude-sonnet-4"
+  end
+  ```
+
+- **New event types** for streaming evaluation:
+  - `Events::EvaluationStart` — emitted when evaluation begins (includes dimension list)
+  - `Events::EvaluationDelta` — streamed evaluation text from the evaluator model
+  - `Events::EvaluationEnd` — emitted with decision, feedback, scores, and evidence
+  - `Events::EvaluationBlocked` — emitted when evaluator returns `:block`
+
+- **17 unit tests** for Evaluator — construction, rubric, all three verdicts, event
+  emission, custom rubrics, JSON parsing, and malformed response fallback.
+
+- **7 integration tests** for Session with evaluator — config (true/hash),
+  revise triggers improvement, revise skips reflector, block returns blocked
+  message, block emits event, evaluator-not-configured skips evaluation.
+
 ## [0.14.0] — 2026-07-23
 
 ### Added

data/README.md

@@ -37,7 +37,88 @@ puts response
 | `Ask::Agent::Telemetry` | telemetry.rb | File-backed telemetry for error tracking |
 | `Ask::Agent::Reflector` | reflector.rb | Assistant response self-evaluation |
 | `Ask::Agent::MetaAgent` | meta_agent.rb | LLM-powered self-improvement from telemetry |
-| `Ask::Agent::Configuration` | configuration.rb | Global config: model, turns, concurrency |
+| `Ask::Agent::Evaluator` | evaluator.rb | Independent response evaluation with structured rubric — different model, isolated context |
+| `Ask::Agent::Configuration` | configuration.rb | Global config: model, turns, concurrency, evaluator |
+
+## Evaluator
+
+Independent response evaluation with generator/evaluator separation. The
+evaluator uses a **separate model** (different from the session's model) and an
+**isolated context** to judge the agent's output — preventing the anti-pattern
+of a model grading its own work.
+
+### Quick start
+
+```ruby
+session = Ask::Agent::Session.new(
+  model: "gpt-4o",
+  evaluator: { model: "claude-sonnet-4", goal: "Write an email validator" }
+)
+session.run("Write email validation")
+```
+
+### Verdicts
+
+| Verdict | Behavior |
+|---------|----------|
+| `:accept` | Output passes — falls through to reflection |
+| `:revise` | Evaluator provides feedback; session runs another turn with it injected |
+| `:block` | Output is fundamentally wrong — returns blocked message, emits `EvaluationBlocked` |
+
+### Configuration
+
+```ruby
+# Set a global default evaluator model
+Ask::Agent.configure do |c|
+  c.default_evaluator_model = "claude-sonnet-4"
+end
+
+# Then use evaluator: true to enable with the default
+session = Ask::Agent::Session.new(model: "gpt-4o", evaluator: true)
+```
+
+### Custom rubric
+
+```ruby
+evaluator = Ask::Agent::Evaluator.new(
+  model: "claude-sonnet-4",
+  rubric: [
+    Ask::Agent::Evaluator::Dimension.new(
+      name: "performance",
+      description: "Is the implementation efficient?",
+      weight: 2
+    )
+  ]
+)
+
+result = evaluator.evaluate(
+  goal: "Write an email validator",
+  response: agent_output
+)
+result.accept?  # => true/false
+result.scores   # => { performance: 2 }
+result.feedback # => "Add edge case for unicode characters"
+```
+
+### Events
+
+The evaluator emits its own events during evaluation:
+
+```ruby
+session.on_event do |event|
+  case event
+  when Ask::Agent::Events::EvaluationStart
+    puts "Evaluating against: #{event.dimensions.join(', ')}"
+  when Ask::Agent::Events::EvaluationDelta
+    print event.content
+  when Ask::Agent::Events::EvaluationEnd
+    puts "Decision: #{event.decision}"
+    puts "Scores: #{event.scores}"
+  when Ask::Agent::Events::EvaluationBlocked
+    puts "Blocked: #{event.feedback}"
+  end
+end
+```
 
 ## Events
 

data/lib/ask/agent/configuration.rb

@@ -5,7 +5,7 @@ module Ask
     class Configuration
       attr_accessor :default_model, :default_max_turns, :compactor_enabled,
                     :compactor_threshold, :parallel_tool_execution, :max_tool_retries,
-                    :prompt_caching
+                    :prompt_caching, :default_evaluator_model
 
       # @return [Middleware::Pipeline] the middleware pipeline for provider calls
       attr_reader :middleware

data/lib/ask/agent/evaluator.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module Ask
+  module Agent
+    class Evaluator
+      # Structured result from an evaluation.
+      # - decision :accept — response meets the goal
+      #           :revise — response needs improvement (feedback provided)
+      #           :block  — response is fundamentally wrong (hard stop)
+      # - feedback actionable text the generator can use to improve
+      # - scores   hash of dimension name => score (0, 1, or 2)
+      # - evidence array of specific evidence strings
+      Result = Data.define(:decision, :feedback, :scores, :evidence) do
+        def accept? = decision == :accept
+        def revise? = decision == :revise
+        def block?  = decision == :block
+      end
+
+      # A single dimension in the evaluation rubric.
+      Dimension = Data.define(:name, :description, :weight) do
+        def initialize(name:, description:, weight: 1)
+          super(name: name, description: description, weight: weight)
+        end
+      end
+
+      # Default rubric borrowed from the course's evaluator-rubric template.
+      DEFAULT_DIMENSIONS = [
+        Dimension.new(name: "correctness",  description: "Does the output match the requested goal?",                               weight: 3),
+        Dimension.new(name: "completeness", description: "Are all aspects of the goal addressed?",                                  weight: 2),
+        Dimension.new(name: "verification", description: "Is there evidence that the output actually works?",                        weight: 2),
+        Dimension.new(name: "scope",        description: "Did it stay within the defined boundaries without overreaching?",          weight: 1),
+        Dimension.new(name: "clarity",      description: "Is the output clear, well-structured, and maintainable?",                  weight: 1),
+      ].freeze
+
+      # How many times the evaluator may retry on a malformed response.
+      MAX_EVAL_RETRIES = 2
+
+      attr_reader :model, :rubric
+
+      # @param model [String] the model id to use for evaluation (should differ from the generator's model)
+      # @param rubric [Array<Dimension>] the rubric dimensions to evaluate against
+      def initialize(model:, rubric: DEFAULT_DIMENSIONS)
+        @model = model
+        @rubric = rubric
+      end
+
+      # Evaluate a response against a goal.
+      #
+      # @param goal [String] what the generator was asked to do
+      # @param response [String] what the generator produced
+      # @param event_emitter [#emit, nil] optional event emitter for streaming evaluation
+      # @return [Result] structured evaluation result
+      def evaluate(goal:, response:, event_emitter: nil)
+        event_emitter&.emit(Events::EvaluationStart.new(dimensions: @rubric.map(&:name)))
+
+        chat = build_chat
+        chat.with_instructions(evaluation_prompt(goal))
+
+        accumulated = +""
+        chat.ask(response.to_s) do |chunk|
+          if chunk.content.to_s.strip.length > 0
+            accumulated << chunk.content.to_s
+            event_emitter&.emit(Events::EvaluationDelta.new(content: chunk.content.to_s))
+          end
+        end
+
+        result = parse_result(accumulated)
+        event_emitter&.emit(Events::EvaluationEnd.new(
+          decision: result.decision,
+          feedback: result.feedback,
+          scores: result.scores,
+          evidence: result.evidence
+        ))
+
+        result
+      end
+
+      private
+
+      def build_chat
+        Chat.new(model: @model)
+      end
+
+      def evaluation_prompt(goal)
+        dimensions_text = @rubric.each_with_index.map { |d, i|
+          weight_label = d.weight > 1 ? " (weight: #{d.weight}x)" : ""
+          "#{i + 1}. **#{d.name}**#{weight_label} — #{d.description}"
+        }.join("\n")
+
+        <<~PROMPT
+          You are an independent evaluator. Your job is to assess whether a response
+          successfully achieves the given goal. You are NOT the agent that produced
+          this response — you are a neutral, objective judge.
+
+          ## Goal
+
+          #{goal}
+
+          ## Rubric
+
+          Evaluate the response against these dimensions:
+
+          #{dimensions_text}
+
+          For each dimension, assign a score:
+          - **0** = fails completely
+          - **1** = partially meets
+          - **2** = fully meets
+
+          Then provide:
+          - A final **decision**: "accept" (response meets the goal), "revise" (needs specific improvements), or "block" (fundamentally wrong — cannot be fixed with revisions)
+          - **Actionable feedback** the generator can use to improve (if decision is revise or block)
+          - **Concrete evidence** for your scores
+
+          Return valid JSON only — no other text:
+          {
+            "scores": { "correctness": 2, "completeness": 1, ... },
+            "decision": "accept",
+            "feedback": "Specific feedback here (or empty string if accepted)",
+            "evidence": ["Evidence point 1", "Evidence point 2"]
+          }
+        PROMPT
+      end
+
+      def parse_result(text)
+        json = extract_json(text)
+        return default_fallback unless json
+
+        scores = json["scores"] || {}
+
+        decision = case json["decision"].to_s.strip.downcase
+                   when "revise" then :revise
+                   when "block"  then :block
+                   else :accept
+                   end
+
+        Result.new(
+          decision: decision,
+          feedback: json["feedback"].to_s.strip,
+          scores: scores.transform_keys(&:to_sym),
+          evidence: Array(json["evidence"])
+        )
+      end
+
+      def extract_json(text)
+        # Try direct parse first
+        JSON.parse(text.strip)
+      rescue JSON::ParserError
+        # Fall back to extracting the first JSON object
+        match = text.match(/\{.*\}/m)
+        match ? JSON.parse(match[0]) : nil
+      end
+
+      def default_fallback
+        Result.new(
+          decision: :accept,
+          feedback: "",
+          scores: {},
+          evidence: []
+        )
+      end
+    end
+  end
+end

data/lib/ask/agent/events.rb

@@ -11,6 +11,7 @@ module Ask
 
       MessageStart = Data.define
       TextDelta = Data.define(:content)
+      ThinkingDelta = Data.define(:content)
       ToolCallDelta = Data.define(:name, :arguments, :id)
       MessageEnd = Data.define(:tool_calls)
 
@@ -28,6 +29,11 @@ module Ask
       ReflectionDelta = Data.define(:content)
       ReflectionEnd = Data.define(:decision, :feedback)
 
+      EvaluationStart = Data.define(:dimensions)
+      EvaluationDelta = Data.define(:content)
+      EvaluationEnd = Data.define(:decision, :feedback, :scores, :evidence)
+      EvaluationBlocked = Data.define(:feedback, :scores, :evidence)
+
       MetaAgentAnalysis = Data.define(:results, :count)
 
       Error = Data.define(:error, :recoverable)

data/lib/ask/agent/loop.rb

@@ -27,6 +27,10 @@ module Ask
             event_emitter.emit(Events::TextDelta.new(content: chunk.content))
           end
 
+          if chunk.respond_to?(:thinking) && chunk.thinking.to_s.strip.length > 0
+            event_emitter.emit(Events::ThinkingDelta.new(content: chunk.thinking))
+          end
+
           if chunk.tool_call?
             chunk.tool_calls.each do |id, tc|
               event_emitter.emit(Events::ToolCallDelta.new(

data/lib/ask/agent/session.rb

@@ -21,7 +21,7 @@ module Ask
                      compactor: nil, hooks: {}, state: nil, persistence: nil,
                      id: nil, system_prompt: nil, parallel_tools: true,
                      reflector: nil, telemetry: true, meta_agent: nil,
-                     agent_dir: nil, **chat_options)
+                     agent_dir: nil, evaluator: nil, **chat_options)
         @id = id || SecureRandom.uuid
         @agent_dir = agent_dir
         @max_turns = max_turns
@@ -65,6 +65,21 @@ module Ask
         @meta_agent_results = nil
 
         @compactor&.chat = @chat
+
+        # Parse evaluator configuration
+        @evaluator = nil
+        @evaluator_config = {}
+
+        if evaluator
+          eval_model = if evaluator.is_a?(Hash)
+                         @evaluator_config = evaluator
+                         evaluator[:model] || Ask::Agent.configuration.default_evaluator_model || model_id_from(@chat)
+                       else
+                         Ask::Agent.configuration.default_evaluator_model || model_id_from(@chat)
+                       end
+
+          @evaluator = Evaluator.new(model: eval_model)
+        end
       end
 
       def run(message, tools: nil)
@@ -127,7 +142,62 @@ module Ask
 
         @tool_calls_made = @tool_executor.total_executions
 
-        if @reflector && @reflector.reflect?(@tool_calls_made) && !@abort_requested
+        # Independent evaluator step (generator/evaluator separation).
+        # Runs BEFORE self-reflection so the evaluator gets a fresh, unbiased look
+        # at the generator's output using a separate model and isolated context.
+        @skip_reflector = false
+
+        if @evaluator && !@abort_requested
+          goal = @evaluator_config[:goal] || message
+
+          eval_result = @evaluator.evaluate(
+            goal: goal.to_s,
+            response: response,
+            event_emitter: self
+          )
+
+          @telemetry.log(:evaluation_end, session_id: @id,
+                         decision: eval_result.decision,
+                         feedback: eval_result.feedback,
+                         scores: eval_result.scores)
+
+          case eval_result.decision
+          when :revise
+            @chat.add_message(
+              role: :system,
+              content: "An independent evaluator has requested revisions:\n\n#{eval_result.feedback}"
+            )
+
+            response = @loop.run_turn(
+              chat: @chat,
+              message: "",
+              tools: active_tools,
+              tool_executor: @tool_executor,
+              compactor: @compactor,
+              hooks: @hooks,
+              event_emitter: self,
+              session_id: @id
+            )
+
+            @total_input_tokens += @loop.last_input_tokens.to_i
+            @total_output_tokens += @loop.last_output_tokens.to_i
+            @total_cost += @loop.last_cost.to_f
+
+            # Skip reflector — we already iterated based on evaluator feedback
+            @skip_reflector = true
+          when :block
+            emit(Events::EvaluationBlocked.new(
+              feedback: eval_result.feedback,
+              scores: eval_result.scores,
+              evidence: eval_result.evidence
+            ))
+            response = "This response was blocked by the evaluator: #{eval_result.feedback}"
+          when :accept
+            # Fall through to reflector for backward compatibility
+          end
+        end
+
+        if @reflector && !@skip_reflector && @reflector.reflect?(@tool_calls_made) && !@abort_requested
           eval_result = @reflector.evaluate(response: response, event_emitter: self)
           @telemetry.log(:reflection_end, session_id: @id, decision: eval_result[:decision], feedback: eval_result[:feedback])
 

data/lib/ask/agent/streaming.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Ask
+  module Agent
+    # Framework-agnostic SSE streaming for agent sessions.
+    #
+    # Returns a Rack-compatible Enumerator that yields SSE-formatted strings
+    # as the agent runs. Works with any Rack server (Puma, Falcon, etc.)
+    # without requiring ActionController::Live or Rails.
+    #
+    # @example In a Rails controller (with ActionController::Live::SSE)
+    #   def create
+    #     response.headers["Content-Type"] = "text/event-stream"
+    #     sse = SSE.new(response.stream)
+    #
+    #     Ask::Agent::Streaming.run(session, prompt) do |type, data|
+    #       sse.write(data, event: type)
+    #     end
+    #   ensure
+    #     sse&.close
+    #   end
+    #
+    # @example In a Rack app (raw Enumerator)
+    #   stream = Ask::Agent::Streaming.run(session, prompt)
+    #   [200, { "Content-Type" => "text/event-stream" }, stream]
+    #
+    # @example With custom event mapping
+    #   stream = Ask::Agent::Streaming.run(session, prompt) do |event|
+    #     case event
+    #     when Events::TextDelta
+    #       { type: "delta", data: { content: event.content } }
+    #     when Events::ToolExecutionStart
+    #       { type: "tool_start", data: { name: event.name, id: event.id } }
+    #     else
+    #       nil # skip unhandled events
+    #     end
+    #   end
+    module Streaming
+      DEFAULT_EVENT_MAP = {
+        Events::TextDelta => "delta",
+        Events::ThinkingDelta => "thinking",
+        Events::ToolCallDelta => "tool_call_delta",
+        Events::ToolExecutionStart => "tool_start",
+        Events::ToolExecutionUpdate => "tool_update",
+        Events::ToolExecutionEnd => "tool_end",
+        Events::SessionEnd => "done",
+        Events::Error => "error"
+      }.freeze
+
+      class << self
+        # Run an agent session and stream events as SSE-formatted strings.
+        #
+        # Two modes:
+        #
+        # 1. **No block** — returns a Rack-compatible Enumerator that yields
+        #    raw SSE strings: "data: {\"type\":\"delta\",\"content\":\"...\"}\n\n"
+        #
+        # 2. **With block** — calls the block for each event with
+        #    `(event_type_string, data_hash)`. The block is responsible for
+        #    writing/handling the data. This mode is designed for use with
+        #    Rails' `ActionController::Live::SSE#write`.
+        #
+        # @param session [Session] the agent session to run
+        # @param prompt [String] the user's message
+        # @param event_map [Hash<Class, String>] optional custom event-to-type mapping
+        # @yield [type, data] called for each event (only in block mode)
+        # @yieldparam type [String] the SSE event type name
+        # @yieldparam data [Hash] the event data payload
+        # @return [Enumerator, nil] Enumerator in no-block mode, nil in block mode
+        def run(session, prompt, event_map: {}, &block)
+          mapping = DEFAULT_EVENT_MAP.merge(event_map)
+
+          if block
+            run_with_block(session, prompt, mapping, &block)
+            nil
+          else
+            run_with_enumerator(session, prompt, mapping)
+          end
+        end
+
+        private
+
+        def run_with_block(session, prompt, mapping)
+          errors = []
+
+          session.on_event do |event|
+            type = event_type(event, mapping)
+            data = event_data(event)
+            next unless type
+
+            yield(type, data)
+          end
+
+          # Emit start event
+          yield("start", { session_id: session.id })
+
+          session.run(prompt)
+
+          # If errors accumulated during tool execution, emit them
+          errors.each { |err| yield("error", { message: err }) }
+        rescue => e
+          yield("error", { message: e.message })
+        end
+
+        def run_with_enumerator(session, prompt, mapping)
+          Enumerator.new do |yielder|
+            errors = []
+
+            session.on_event do |event|
+              type = event_type(event, mapping)
+              data = event_data(event)
+              next unless type
+
+              yielder << sse_line(type, data)
+            end
+
+            # Emit start event
+            yielder << sse_line("start", { session_id: session.id })
+
+            session.run(prompt)
+
+            errors.each { |err| yielder << sse_line("error", { message: err }) }
+
+            yielder << sse_line("close", {})
+          rescue => e
+            yielder << sse_line("error", { message: e.message })
+          ensure
+            yielder << sse_line("close", {})
+          end
+        end
+
+        def event_type(event, mapping)
+          # Check for a direct class match
+          return mapping[event.class] if mapping.key?(event.class)
+
+          # Check for a superclass match (e.g. ToolExecutionStart is a kind of event)
+          event.class.ancestors.each do |ancestor|
+            return mapping[ancestor] if mapping.key?(ancestor)
+          end
+
+          nil
+        end
+
+        def event_data(event)
+          case event
+          when Events::TextDelta
+            { content: event.content }
+          when Events::ThinkingDelta
+            { content: event.content }
+          when Events::ToolCallDelta
+            { name: event.name, arguments: event.arguments, id: event.id }
+          when Events::ToolExecutionStart
+            { name: event.name, id: event.id, args: safe_args(event.arguments) }
+          when Events::ToolExecutionUpdate
+            { id: event.id, partial_result: event.partial_result.to_s.truncate(200) }
+          when Events::ToolExecutionEnd
+            { name: event.name, id: event.id, duration_ms: event.duration_ms, is_error: event.is_error }
+          when Events::SessionEnd
+            { turn_count: event.turn_count, tool_calls_made: event.tool_calls_made,
+              input_tokens: event.input_tokens, output_tokens: event.output_tokens,
+              cost: event.cost }
+          when Events::SessionStart
+            {}
+          when Events::Error
+            { message: event.error, recoverable: event.recoverable }
+          else
+            {}
+          end
+        end
+
+        def sse_line(type, data)
+          payload = data.merge(type: type)
+          "data: #{JSON.generate(payload)}\n\n"
+        end
+
+        def safe_args(args)
+          return {} unless args.is_a?(Hash)
+
+          safe = args.dup
+          %w[password secret token api_key key auth_token access_token sql command].each do |sensitive|
+            safe[sensitive] = "[REDACTED]" if safe.key?(sensitive)
+          end
+          safe
+        end
+      end
+    end
+  end
+end

data/lib/ask/agent/sub_agent.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+module Ask
+  module Agent
+    # A tool that delegates a task to a specialized sub-agent.
+    #
+    # The coordinator agent sees this as a regular tool — when the LLM calls it,
+    # a fresh sub-agent session runs independently with its own model, tools,
+    # and instructions, and returns the result.
+    #
+    # Satisfies the tool duck type (name, description, params_schema, call)
+    # so it can be passed directly in the tools array.
+    #
+    # @example From a filesystem definition
+    #   # agents/web_search/agent.rb defines a WebSearch agent.
+    #   # Use it as a sub-agent by passing its name:
+    #
+    #   search = Ask::Agent::SubAgent.new("web_search")
+    #
+    #   coordinator = Ask::Agent::Session.new(
+    #     model: "gpt-4o",
+    #     tools: [search, Ask::Tools::Shell::Bash]
+    #   )
+    #
+    # @example Inline configuration
+    #   search = Ask::Agent::SubAgent.new(
+    #     name: "web_search",
+    #     description: "Search the web for current information",
+    #     model: "gpt-4o-mini",
+    #     tools: [Ask::Tools::WebSearch],
+    #     system_prompt: "You are a research assistant."
+    #   )
+    #
+    # @example Using with a different provider
+    #   review = Ask::Agent::SubAgent.new(
+    #     name: "code_review",
+    #     model: "claude-sonnet-4",
+    #     provider: :anthropic,
+    #     tools: [Ask::Tools::Shell::Read, Ask::Tools::Shell::Grep],
+    #     system_prompt: "You are a senior code reviewer."
+    #   )
+    class SubAgent
+      # @return [String] tool name visible to the LLM
+      attr_reader :name
+
+      # @return [String] tool description visible to the LLM
+      attr_reader :description
+
+      # Create a new SubAgent tool.
+      #
+      # When given a String, looks up a filesystem agent definition by name
+      # (matching the convention used by {Ask::Agent.new}). Model, tools,
+      # instructions, and other settings are read from the definition files.
+      #
+      # When given keyword arguments, configures the sub-agent inline.
+      #
+      # @overload initialize(definition_name)
+      #   @param definition_name [String] Name of a filesystem agent definition.
+      #   @raise [Ask::Agent::UnknownAgent] If no definition is found.
+      #
+      # @overload initialize(name:, description: nil, model:, tools: [],
+      #           system_prompt: nil, provider: nil, max_turns: 10, **session_opts)
+      #   @param name [String] Tool name (e.g. "web_search").
+      #   @param description [String, nil] Tool description. Auto-generated
+      #     from the model and tools count if not provided.
+      #   @param model [String] Model identifier for the sub-agent session.
+      #   @param tools [Array<Class, Object>] Tools available to the sub-agent.
+      #   @param system_prompt [String, nil] Instructions for the sub-agent.
+      #   @param provider [Symbol, nil] Provider override.
+      #   @param max_turns [Integer] Max conversation turns for the sub-agent.
+      #   @param session_opts [Hash] Additional options forwarded to Session.new.
+      def initialize(definition_name = nil, name: nil, description: nil, model: nil,
+                     tools: [], system_prompt: nil, provider: nil,
+                     max_turns: 10, **session_opts)
+        if definition_name
+          from_definition(definition_name, **session_opts)
+        else
+          @name = name
+          @description = description || default_description(model, tools)
+          @model = model
+          @tools = tools.map { |t| t.is_a?(Class) ? t.new : t }
+          @system_prompt = system_prompt
+          @provider = provider
+          @max_turns = max_turns
+          @session_opts = session_opts
+        end
+      end
+
+      # JSON Schema for the tool's parameter.
+      #
+      # @return [Hash]
+      def params_schema
+        {
+          type: "object",
+          properties: {
+            "task" => {
+              type: "string",
+              description: "The task to delegate to the sub-agent"
+            }
+          },
+          required: ["task"],
+          additionalProperties: false
+        }
+      end
+
+      # Provider-specific parameters (none by default).
+      #
+      # @return [Hash]
+      def provider_params
+        {}
+      end
+
+      # Execute the sub-agent with the given task.
+      #
+      # Creates a fresh session for each call, runs the task, and returns the
+      # result. If the sub-agent fails, returns an error result — the
+      # coordinator can decide how to proceed.
+      #
+      # @param args [Hash, String] Arguments from the LLM.
+      # @param abort_controller [Object, nil] Optional abort controller.
+      # @return [Ask::Result]
+      def call(args = {}, abort_controller = nil)
+        task = extract_task(args)
+
+        session_opts = {
+          model: @model,
+          tools: @tools.map(&:class),
+          max_turns: @max_turns
+        }
+        session_opts[:provider] = @provider if @provider
+        session_opts[:system_prompt] = @system_prompt if @system_prompt
+        session_opts.merge!(@session_opts)
+
+        session = Session.new(**session_opts)
+        result = session.run(task.to_s)
+        Ask::Result.ok(data: result.to_s)
+      rescue StandardError => e
+        Ask::Result.failure("SubAgent '#{@name}' error: #{e.message}")
+      end
+
+      # Human-readable representation.
+      #
+      # @return [String]
+      def inspect
+        "#<Ask::Agent::SubAgent name=#{@name.inspect}>"
+      end
+
+      private
+
+      def from_definition(definition_name, **session_opts)
+        Ask::Agent.rediscover!
+        entry = Ask::Agent.definitions[definition_name.to_s]
+        raise UnknownAgent, "Unknown agent: #{definition_name.inspect}" unless entry
+
+        klass, dir = entry
+        config = klass._config
+
+        @name = definition_name
+        @model = config[:model]
+        @description = "Delegate to #{definition_name} sub-agent (#{@model})"
+        @provider = config[:provider]
+
+        # Resolve tools from definition
+        resolved_tools = Ask::Agent.__send__(:resolve_definition_tools, config[:tools] || [], dir)
+        @tools = resolved_tools.map { |t| t.is_a?(Class) ? t.new : t }
+
+        # Load instructions from definition
+        prompt = klass.instructions_content
+        @system_prompt = prompt
+
+        @max_turns = config[:max_turns] || 10
+        # Merge any options from the definition config
+        @session_opts = (config[:options] || {}).merge(session_opts)
+      end
+
+      def extract_task(args)
+        case args
+        when Hash then (args["task"] || args[:task] || args.to_s).to_s
+        else args.to_s
+        end
+      end
+
+      def default_description(model, tools)
+        desc = "Delegate to a sub-agent (#{model}"
+        desc += " with #{tools.size} tool(s)" if tools.any?
+        desc + ")"
+      end
+    end
+  end
+end

data/lib/ask/agent/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module Agent
-    VERSION = "0.14.0"
+    VERSION = "0.20.0"
   end
 end

data/lib/ask/agent.rb

@@ -22,7 +22,6 @@ module Ask
 
     module Extensions
       autoload :Permissions, "ask/agent/extensions/permissions"
-      autoload :PermissionGate, "ask/agent/extensions/permission_gate"
       autoload :RateLimiter, "ask/agent/extensions/rate_limiter"
       autoload :AuditLog, "ask/agent/extensions/audit_log"
     end
@@ -233,6 +232,7 @@ require_relative "agent/tool_abort_controller"
 require_relative "agent/session"
 require_relative "agent/loop"
 require_relative "agent/reflector"
+require_relative "agent/evaluator"
 require_relative "agent/tool_executor"
 require_relative "agent/compactor"
 require_relative "agent/hooks"
@@ -244,6 +244,29 @@ require_relative "agent/skills/load_skill_tool"
 require_relative "agent/scheduler"
 require_relative "agent/definition"
 require_relative "agent/cli"
+require_relative "agent/streaming"
+require_relative "agent/sub_agent"
 
 # Test helpers (loaded on demand)
 autoload :Test, "ask/agent/test"
+
+# Convenience method on the top-level Ask module.
+# Provides a quick one-shot chat without instantiating a Session directly.
+#
+#   Ask.chat("Hello")
+#   Ask.chat("Tell me about X", model: "gpt-4o")
+#   Ask.chat("Stream this") { |chunk| puts chunk.content }
+#
+module Ask
+  def self.chat(message, model: nil, system_prompt: nil, &block)
+    session = Agent::Session.new(
+      model: model || Agent.configuration.default_model,
+      system_prompt: system_prompt
+    )
+    if block
+      session.run(message, &block)
+    else
+      session.run(message)
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-agent
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.20.0
 platform: ruby
 authors:
 - Kaka Ruto
@@ -23,6 +23,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: ask-state-providers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: ask-llm-providers
   requirement: !ruby/object:Gem::Requirement
@@ -157,9 +171,9 @@ files:
 - lib/ask/agent/context_source.rb
 - lib/ask/agent/context_sources.rb
 - lib/ask/agent/definition.rb
+- lib/ask/agent/evaluator.rb
 - lib/ask/agent/events.rb
 - lib/ask/agent/extensions/audit_log.rb
-- lib/ask/agent/extensions/permission_gate.rb
 - lib/ask/agent/extensions/permissions.rb
 - lib/ask/agent/extensions/rate_limiter.rb
 - lib/ask/agent/hooks.rb
@@ -182,6 +196,8 @@ files:
 - lib/ask/agent/stream_transforms/pipeline.rb
 - lib/ask/agent/stream_transforms/text_buffer.rb
 - lib/ask/agent/stream_transforms/thinking_separator.rb
+- lib/ask/agent/streaming.rb
+- lib/ask/agent/sub_agent.rb
 - lib/ask/agent/system_context.rb
 - lib/ask/agent/telemetry.rb
 - lib/ask/agent/test.rb

data/lib/ask/agent/extensions/permission_gate.rb

@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-# Legacy alias — PermissionGate has been renamed to Permissions.
-# This file will be removed in the next major version.
-require_relative "permissions"
-
-module Ask
-  module Agent
-    module Extensions
-      PermissionGate = Permissions
-    end
-  end
-end