ask-agent 0.14.0 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c80f27cbdec5b3808533dba4861762e19e4ef81c084be099a4a7eb0b2169170e
-  data.tar.gz: 34e5388035a8f216df468039bf8c7e304296d7661832e8109df4d76fbb527193
+  metadata.gz: 816f21ed178b31227a58afeeed65f2428dbe9cce7d4a6ddab2c6fda7a6a60a89
+  data.tar.gz: 4dec09ce3cc94c6abc0e94bd5dee9f6ac9bde083dc7995af601a6650e692fd37
 SHA512:
-  metadata.gz: bb1bf1b6aac7a61a650291f9dcabd781c0a664cb09d97c7f2cc2fcdcfaa035b71adef2c423ebacfaae0de04d0524b4dff649f503921006837e0644d9a9f86615
-  data.tar.gz: d718885adebbaf33f2b9a2b95704968494d14e3080746a58da37c9518af475ac09103357014e5bea98339d56150ce63a162c6ac9a9c8ae4cffa07f13716b8793
+  metadata.gz: 8de0c033834105f7c147115bd84b97306fc91b787690fc9e82563835f53b8fad69ff90dfabeeada12be9a8651e9ecfffd960d8e2fa6e84ffff4709f30519a5de
+  data.tar.gz: b5d46db9e66b242ea35fafaf86c67401ad27ee8ef0099ad4811a3d8442e63bd54aa2b16be5e4a7b2aa0f7305603cf39d1b5497e66648f59a9d701489bbdc6c8c

data/CHANGELOG.md

@@ -1,3 +1,67 @@
+## [0.15.0] — 2026-07-24
+
+### Added
+
+- **Independent Evaluator — `Ask::Agent::Evaluator`** — Generator/evaluator separation.
+  A separate model (configured independently from the session's model) judges the
+  agent's output against a structured rubric before delivery. This prevents the
+  anti-pattern of a model grading its own work.
+
+  ```ruby
+  # Evaluate with a different model — the recommended approach
+  session = Ask::Agent::Session.new(
+    model: "gpt-4o",
+    evaluator: { model: "claude-sonnet-4", goal: "Write an email validator" }
+  )
+  session.run("Write email validation")
+  ```
+
+  Three verdicts:
+  - **`:accept`** — output meets the goal, passes through to reflection
+  - **`:revise`** — evaluator provides actionable feedback; session runs another
+    turn with the feedback injected into system context
+  - **`:block`** — output is fundamentally wrong; session returns blocked message
+    and emits `Events::EvaluationBlocked`
+
+  Rubric dimensions (each scored 0-2):
+  - correctness (3× weight), completeness (2×), verification (2×), scope (1×), clarity (1×)
+
+  Custom rubrics supported:
+  ```ruby
+  evaluator = Ask::Agent::Evaluator.new(
+    model: "claude-sonnet-4",
+    rubric: [
+      Ask::Agent::Evaluator::Dimension.new(name: "performance", description: "Is it fast?", weight: 2)
+    ]
+  )
+  ```
+
+- **`evaluator:` option on `Session`** — accepts `true`, `false`/`nil`, or a Hash:
+  - `evaluator: true` — uses `config.default_evaluator_model` (falls back to the
+    session's model, though using a different model is strongly recommended)
+  - `evaluator: { model: "claude-sonnet-4", goal: "Custom goal" }` — explicit config
+  - `evaluator: false` (default) — no evaluation, backward compatible
+
+- **`default_evaluator_model` config option** — set a global default:
+  ```ruby
+  Ask::Agent.configure do |c|
+    c.default_evaluator_model = "claude-sonnet-4"
+  end
+  ```
+
+- **New event types** for streaming evaluation:
+  - `Events::EvaluationStart` — emitted when evaluation begins (includes dimension list)
+  - `Events::EvaluationDelta` — streamed evaluation text from the evaluator model
+  - `Events::EvaluationEnd` — emitted with decision, feedback, scores, and evidence
+  - `Events::EvaluationBlocked` — emitted when evaluator returns `:block`
+
+- **17 unit tests** for Evaluator — construction, rubric, all three verdicts, event
+  emission, custom rubrics, JSON parsing, and malformed response fallback.
+
+- **7 integration tests** for Session with evaluator — config (true/hash),
+  revise triggers improvement, revise skips reflector, block returns blocked
+  message, block emits event, evaluator-not-configured skips evaluation.
+
 ## [0.14.0] — 2026-07-23
 
 ### Added

data/README.md

@@ -37,7 +37,88 @@ puts response
 | `Ask::Agent::Telemetry` | telemetry.rb | File-backed telemetry for error tracking |
 | `Ask::Agent::Reflector` | reflector.rb | Assistant response self-evaluation |
 | `Ask::Agent::MetaAgent` | meta_agent.rb | LLM-powered self-improvement from telemetry |
-| `Ask::Agent::Configuration` | configuration.rb | Global config: model, turns, concurrency |
+| `Ask::Agent::Evaluator` | evaluator.rb | Independent response evaluation with structured rubric — different model, isolated context |
+| `Ask::Agent::Configuration` | configuration.rb | Global config: model, turns, concurrency, evaluator |
+
+## Evaluator
+
+Independent response evaluation with generator/evaluator separation. The
+evaluator uses a **separate model** (different from the session's model) and an
+**isolated context** to judge the agent's output — preventing the anti-pattern
+of a model grading its own work.
+
+### Quick start
+
+```ruby
+session = Ask::Agent::Session.new(
+  model: "gpt-4o",
+  evaluator: { model: "claude-sonnet-4", goal: "Write an email validator" }
+)
+session.run("Write email validation")
+```
+
+### Verdicts
+
+| Verdict | Behavior |
+|---------|----------|
+| `:accept` | Output passes — falls through to reflection |
+| `:revise` | Evaluator provides feedback; session runs another turn with it injected |
+| `:block` | Output is fundamentally wrong — returns blocked message, emits `EvaluationBlocked` |
+
+### Configuration
+
+```ruby
+# Set a global default evaluator model
+Ask::Agent.configure do |c|
+  c.default_evaluator_model = "claude-sonnet-4"
+end
+
+# Then use evaluator: true to enable with the default
+session = Ask::Agent::Session.new(model: "gpt-4o", evaluator: true)
+```
+
+### Custom rubric
+
+```ruby
+evaluator = Ask::Agent::Evaluator.new(
+  model: "claude-sonnet-4",
+  rubric: [
+    Ask::Agent::Evaluator::Dimension.new(
+      name: "performance",
+      description: "Is the implementation efficient?",
+      weight: 2
+    )
+  ]
+)
+
+result = evaluator.evaluate(
+  goal: "Write an email validator",
+  response: agent_output
+)
+result.accept?  # => true/false
+result.scores   # => { performance: 2 }
+result.feedback # => "Add edge case for unicode characters"
+```
+
+### Events
+
+The evaluator emits its own events during evaluation:
+
+```ruby
+session.on_event do |event|
+  case event
+  when Ask::Agent::Events::EvaluationStart
+    puts "Evaluating against: #{event.dimensions.join(', ')}"
+  when Ask::Agent::Events::EvaluationDelta
+    print event.content
+  when Ask::Agent::Events::EvaluationEnd
+    puts "Decision: #{event.decision}"
+    puts "Scores: #{event.scores}"
+  when Ask::Agent::Events::EvaluationBlocked
+    puts "Blocked: #{event.feedback}"
+  end
+end
+```
 
 ## Events
 

data/lib/ask/agent/configuration.rb

@@ -5,7 +5,7 @@ module Ask
     class Configuration
       attr_accessor :default_model, :default_max_turns, :compactor_enabled,
                     :compactor_threshold, :parallel_tool_execution, :max_tool_retries,
-                    :prompt_caching
+                    :prompt_caching, :default_evaluator_model
 
       # @return [Middleware::Pipeline] the middleware pipeline for provider calls
       attr_reader :middleware

data/lib/ask/agent/evaluator.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module Ask
+  module Agent
+    class Evaluator
+      # Structured result from an evaluation.
+      # - decision :accept — response meets the goal
+      #           :revise — response needs improvement (feedback provided)
+      #           :block  — response is fundamentally wrong (hard stop)
+      # - feedback actionable text the generator can use to improve
+      # - scores   hash of dimension name => score (0, 1, or 2)
+      # - evidence array of specific evidence strings
+      Result = Data.define(:decision, :feedback, :scores, :evidence) do
+        def accept? = decision == :accept
+        def revise? = decision == :revise
+        def block?  = decision == :block
+      end
+
+      # A single dimension in the evaluation rubric.
+      Dimension = Data.define(:name, :description, :weight) do
+        def initialize(name:, description:, weight: 1)
+          super(name: name, description: description, weight: weight)
+        end
+      end
+
+      # Default rubric borrowed from the course's evaluator-rubric template.
+      DEFAULT_DIMENSIONS = [
+        Dimension.new(name: "correctness",  description: "Does the output match the requested goal?",                               weight: 3),
+        Dimension.new(name: "completeness", description: "Are all aspects of the goal addressed?",                                  weight: 2),
+        Dimension.new(name: "verification", description: "Is there evidence that the output actually works?",                        weight: 2),
+        Dimension.new(name: "scope",        description: "Did it stay within the defined boundaries without overreaching?",          weight: 1),
+        Dimension.new(name: "clarity",      description: "Is the output clear, well-structured, and maintainable?",                  weight: 1),
+      ].freeze
+
+      # How many times the evaluator may retry on a malformed response.
+      MAX_EVAL_RETRIES = 2
+
+      attr_reader :model, :rubric
+
+      # @param model [String] the model id to use for evaluation (should differ from the generator's model)
+      # @param rubric [Array<Dimension>] the rubric dimensions to evaluate against
+      def initialize(model:, rubric: DEFAULT_DIMENSIONS)
+        @model = model
+        @rubric = rubric
+      end
+
+      # Evaluate a response against a goal.
+      #
+      # @param goal [String] what the generator was asked to do
+      # @param response [String] what the generator produced
+      # @param event_emitter [#emit, nil] optional event emitter for streaming evaluation
+      # @return [Result] structured evaluation result
+      def evaluate(goal:, response:, event_emitter: nil)
+        event_emitter&.emit(Events::EvaluationStart.new(dimensions: @rubric.map(&:name)))
+
+        chat = build_chat
+        chat.with_instructions(evaluation_prompt(goal))
+
+        accumulated = +""
+        chat.ask(response.to_s) do |chunk|
+          if chunk.content.to_s.strip.length > 0
+            accumulated << chunk.content.to_s
+            event_emitter&.emit(Events::EvaluationDelta.new(content: chunk.content.to_s))
+          end
+        end
+
+        result = parse_result(accumulated)
+        event_emitter&.emit(Events::EvaluationEnd.new(
+          decision: result.decision,
+          feedback: result.feedback,
+          scores: result.scores,
+          evidence: result.evidence
+        ))
+
+        result
+      end
+
+      private
+
+      def build_chat
+        Chat.new(model: @model)
+      end
+
+      def evaluation_prompt(goal)
+        dimensions_text = @rubric.each_with_index.map { |d, i|
+          weight_label = d.weight > 1 ? " (weight: #{d.weight}x)" : ""
+          "#{i + 1}. **#{d.name}**#{weight_label} — #{d.description}"
+        }.join("\n")
+
+        <<~PROMPT
+          You are an independent evaluator. Your job is to assess whether a response
+          successfully achieves the given goal. You are NOT the agent that produced
+          this response — you are a neutral, objective judge.
+
+          ## Goal
+
+          #{goal}
+
+          ## Rubric
+
+          Evaluate the response against these dimensions:
+
+          #{dimensions_text}
+
+          For each dimension, assign a score:
+          - **0** = fails completely
+          - **1** = partially meets
+          - **2** = fully meets
+
+          Then provide:
+          - A final **decision**: "accept" (response meets the goal), "revise" (needs specific improvements), or "block" (fundamentally wrong — cannot be fixed with revisions)
+          - **Actionable feedback** the generator can use to improve (if decision is revise or block)
+          - **Concrete evidence** for your scores
+
+          Return valid JSON only — no other text:
+          {
+            "scores": { "correctness": 2, "completeness": 1, ... },
+            "decision": "accept",
+            "feedback": "Specific feedback here (or empty string if accepted)",
+            "evidence": ["Evidence point 1", "Evidence point 2"]
+          }
+        PROMPT
+      end
+
+      def parse_result(text)
+        json = extract_json(text)
+        return default_fallback unless json
+
+        scores = json["scores"] || {}
+
+        decision = case json["decision"].to_s.strip.downcase
+                   when "revise" then :revise
+                   when "block"  then :block
+                   else :accept
+                   end
+
+        Result.new(
+          decision: decision,
+          feedback: json["feedback"].to_s.strip,
+          scores: scores.transform_keys(&:to_sym),
+          evidence: Array(json["evidence"])
+        )
+      end
+
+      def extract_json(text)
+        # Try direct parse first
+        JSON.parse(text.strip)
+      rescue JSON::ParserError
+        # Fall back to extracting the first JSON object
+        match = text.match(/\{.*\}/m)
+        match ? JSON.parse(match[0]) : nil
+      end
+
+      def default_fallback
+        Result.new(
+          decision: :accept,
+          feedback: "",
+          scores: {},
+          evidence: []
+        )
+      end
+    end
+  end
+end

data/lib/ask/agent/events.rb

@@ -28,6 +28,11 @@ module Ask
       ReflectionDelta = Data.define(:content)
       ReflectionEnd = Data.define(:decision, :feedback)
 
+      EvaluationStart = Data.define(:dimensions)
+      EvaluationDelta = Data.define(:content)
+      EvaluationEnd = Data.define(:decision, :feedback, :scores, :evidence)
+      EvaluationBlocked = Data.define(:feedback, :scores, :evidence)
+
       MetaAgentAnalysis = Data.define(:results, :count)
 
       Error = Data.define(:error, :recoverable)

data/lib/ask/agent/session.rb

@@ -21,7 +21,7 @@ module Ask
                      compactor: nil, hooks: {}, state: nil, persistence: nil,
                      id: nil, system_prompt: nil, parallel_tools: true,
                      reflector: nil, telemetry: true, meta_agent: nil,
-                     agent_dir: nil, **chat_options)
+                     agent_dir: nil, evaluator: nil, **chat_options)
         @id = id || SecureRandom.uuid
         @agent_dir = agent_dir
         @max_turns = max_turns
@@ -65,6 +65,21 @@ module Ask
         @meta_agent_results = nil
 
         @compactor&.chat = @chat
+
+        # Parse evaluator configuration
+        @evaluator = nil
+        @evaluator_config = {}
+
+        if evaluator
+          eval_model = if evaluator.is_a?(Hash)
+                         @evaluator_config = evaluator
+                         evaluator[:model] || Ask::Agent.configuration.default_evaluator_model || model_id_from(@chat)
+                       else
+                         Ask::Agent.configuration.default_evaluator_model || model_id_from(@chat)
+                       end
+
+          @evaluator = Evaluator.new(model: eval_model)
+        end
       end
 
       def run(message, tools: nil)
@@ -127,7 +142,62 @@ module Ask
 
         @tool_calls_made = @tool_executor.total_executions
 
-        if @reflector && @reflector.reflect?(@tool_calls_made) && !@abort_requested
+        # Independent evaluator step (generator/evaluator separation).
+        # Runs BEFORE self-reflection so the evaluator gets a fresh, unbiased look
+        # at the generator's output using a separate model and isolated context.
+        @skip_reflector = false
+
+        if @evaluator && !@abort_requested
+          goal = @evaluator_config[:goal] || message
+
+          eval_result = @evaluator.evaluate(
+            goal: goal.to_s,
+            response: response,
+            event_emitter: self
+          )
+
+          @telemetry.log(:evaluation_end, session_id: @id,
+                         decision: eval_result.decision,
+                         feedback: eval_result.feedback,
+                         scores: eval_result.scores)
+
+          case eval_result.decision
+          when :revise
+            @chat.add_message(
+              role: :system,
+              content: "An independent evaluator has requested revisions:\n\n#{eval_result.feedback}"
+            )
+
+            response = @loop.run_turn(
+              chat: @chat,
+              message: "",
+              tools: active_tools,
+              tool_executor: @tool_executor,
+              compactor: @compactor,
+              hooks: @hooks,
+              event_emitter: self,
+              session_id: @id
+            )
+
+            @total_input_tokens += @loop.last_input_tokens.to_i
+            @total_output_tokens += @loop.last_output_tokens.to_i
+            @total_cost += @loop.last_cost.to_f
+
+            # Skip reflector — we already iterated based on evaluator feedback
+            @skip_reflector = true
+          when :block
+            emit(Events::EvaluationBlocked.new(
+              feedback: eval_result.feedback,
+              scores: eval_result.scores,
+              evidence: eval_result.evidence
+            ))
+            response = "This response was blocked by the evaluator: #{eval_result.feedback}"
+          when :accept
+            # Fall through to reflector for backward compatibility
+          end
+        end
+
+        if @reflector && !@skip_reflector && @reflector.reflect?(@tool_calls_made) && !@abort_requested
           eval_result = @reflector.evaluate(response: response, event_emitter: self)
           @telemetry.log(:reflection_end, session_id: @id, decision: eval_result[:decision], feedback: eval_result[:feedback])
 

data/lib/ask/agent/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module Agent
-    VERSION = "0.14.0"
+    VERSION = "0.15.0"
   end
 end

data/lib/ask/agent.rb

@@ -22,7 +22,6 @@ module Ask
 
     module Extensions
       autoload :Permissions, "ask/agent/extensions/permissions"
-      autoload :PermissionGate, "ask/agent/extensions/permission_gate"
       autoload :RateLimiter, "ask/agent/extensions/rate_limiter"
       autoload :AuditLog, "ask/agent/extensions/audit_log"
     end
@@ -233,6 +232,7 @@ require_relative "agent/tool_abort_controller"
 require_relative "agent/session"
 require_relative "agent/loop"
 require_relative "agent/reflector"
+require_relative "agent/evaluator"
 require_relative "agent/tool_executor"
 require_relative "agent/compactor"
 require_relative "agent/hooks"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-agent
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.15.0
 platform: ruby
 authors:
 - Kaka Ruto
@@ -23,6 +23,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: ask-state-providers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: ask-llm-providers
   requirement: !ruby/object:Gem::Requirement
@@ -157,9 +171,9 @@ files:
 - lib/ask/agent/context_source.rb
 - lib/ask/agent/context_sources.rb
 - lib/ask/agent/definition.rb
+- lib/ask/agent/evaluator.rb
 - lib/ask/agent/events.rb
 - lib/ask/agent/extensions/audit_log.rb
-- lib/ask/agent/extensions/permission_gate.rb
 - lib/ask/agent/extensions/permissions.rb
 - lib/ask/agent/extensions/rate_limiter.rb
 - lib/ask/agent/hooks.rb

data/lib/ask/agent/extensions/permission_gate.rb

@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-# Legacy alias — PermissionGate has been renamed to Permissions.
-# This file will be removed in the next major version.
-require_relative "permissions"
-
-module Ask
-  module Agent
-    module Extensions
-      PermissionGate = Permissions
-    end
-  end
-end