ask-decisions 0.1.1

data/lib/ask/decisions/quality_judge.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # Decision-based quality judge — replaces the LLM-as-judge evaluator
+    # with calibrated Jev scores. Uses composite scoring across rubric
+    # dimensions plus a verdict choice.
+    #
+    #   judge = Ask::Decisions::QualityJudge.new(provider)
+    #   verdict = judge.evaluate(
+    #     request: "What's the weather in Seattle?",
+    #     response: "It's 72°F and sunny in Seattle today.",
+    #     rubric: { accuracy: "Is the answer factually correct?", completeness: "Does it fully address the request?" }
+    #   )
+    #   verdict.accepted?  # => true
+    #   verdict.scores     # => { accuracy: 4.0, completeness: 3.5 }
+    #
+    class QualityJudge
+      # Default rubric dimensions.
+      DEFAULT_RUBRIC = {
+        accuracy: "Is the answer factually correct and free of hallucinations?",
+        completeness: "Does the answer fully address the user's request?",
+        clarity: "Is the answer clear, well-organized, and easy to understand?"
+      }.freeze
+
+      # @param provider [Ask::DecisionProvider]
+      def initialize(provider)
+        @provider = provider
+      end
+
+      # Evaluate a response against a rubric.
+      #
+      # @param request [String] the original user request
+      # @param response [String] the assistant's response
+      # @param rubric [Hash{Symbol => String}] dimension → instruction
+      # @param threshold [Float] minimum weighted score to accept
+      # @return [QualityVerdict]
+      def evaluate(request:, response:, rubric: DEFAULT_RUBRIC, threshold: 2.5)
+        questions = {}
+        rubric.each do |dim, instruction|
+          questions[dim.to_s] = Ask::Decision::Score.new(
+            instructions: instruction,
+            criteria: [
+              "Poor — factually wrong, incomplete, or unclear",
+              "Fair — partially correct but missing key aspects",
+              "Good — correct and complete, minor issues only",
+              "Excellent — thorough, accurate, and well-presented"
+            ]
+          )
+        end
+
+        questions["verdict"] = Ask::Decision::Choice.new(
+          instructions: "Should this response be accepted, revised, or blocked?",
+          criteria: {
+            "accept" => "The response is good enough to deliver to the user",
+            "revise" => "The response has issues that can be fixed with minor edits",
+            "block" => "The response is fundamentally wrong or harmful and must be regenerated"
+          }
+        )
+
+        state = { request: request, response: response }
+        result = @provider.evaluate(state: state, decisions: questions)
+
+        QualityVerdict.new(result, rubric.keys, threshold)
+      end
+
+      # Verdict from quality evaluation.
+      class QualityVerdict
+        attr_reader :result, :scores, :verdict, :threshold
+
+        def initialize(result, dimensions, threshold)
+          @result = result
+          @threshold = threshold
+          @scores = {}
+
+          dimensions.each do |dim|
+            answer = result[dim.to_s]
+            @scores[dim] = answer&.score || 0.0
+          end
+
+          verdict_answer = result["verdict"]
+          @verdict = verdict_answer&.choice || "accept"
+        end
+
+        def accepted?  = @verdict == "accept"
+        def revised?   = @verdict == "revise"
+        def blocked?   = @verdict == "block"
+
+        # Weighted average score across dimensions.
+        def average_score
+          return 0.0 if @scores.empty?
+          @scores.values.sum / @scores.size
+        end
+
+        # Should the response be accepted based on the score threshold?
+        def score_accepts?
+          average_score >= @threshold
+        end
+
+        def to_s
+          scores_str = @scores.map { |k, v| "#{k}: #{('%.1f' % v)}" }.join(", ")
+          "#{@verdict} (#{scores_str}, avg: #{('%.1f' % average_score)})"
+        end
+      end
+    end
+  end
+end

data/lib/ask/decisions/reader.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # One request, several answers, about one piece of state.
+    #
+    # The questions in a decision call are independent and run in parallel, so
+    # asking three costs no more latency than asking one. That makes "ask
+    # everything this call might need" the right instinct rather than a false
+    # economy — and makes this the shared shape for doing it: a Choice over a
+    # described set of options, with anything else the caller needs riding
+    # along in the same request.
+    #
+    # The caller owns *what* to ask — which options, in what words. This owns
+    # asking it once and handing back the answers. `ToolRouter` and `Triage`
+    # are both façades over it.
+    #
+    #   reader = Ask::Decisions::Reader.new(
+    #     provider,
+    #     id: "lane",
+    #     instructions: "Which of these best describes what the person wants?",
+    #     options: {"knowledge" => "Asks about the business", "booking" => "Wants to book"},
+    #     also: {"urgent" => Ask::Decision::Noul.new(instructions: "Is this urgent?")}
+    #   )
+    #   batch = reader.read(state: {message: "What time do you close?"})
+    #   reader.choice(batch).choice      # => "knowledge"
+    #   batch["urgent"].noul             # => 0.12
+    #
+    class Reader
+      # How much of an option's description to keep. A description is the
+      # separator between one option and the others, and a long one buries the
+      # part that separates. What is dropped is said out loud, because a
+      # silently clipped option reads as a whole one.
+      DEFAULT_LIMIT = 160
+
+      # The number of characters the elision marker itself needs.
+      ELISION_ROOM = 20
+
+      attr_reader :id, :instructions, :options
+
+      # @param provider [Ask::DecisionProvider]
+      # @param id [String, Symbol] the question id the Choice is asked under.
+      #   Ids are for the caller's code and are never sent to the model.
+      # @param instructions [String] the question the options answer
+      # @param options [Hash{String => String}] option => what belongs in it
+      # @param also [Hash{String => Decision::Choice,Decision::Score,Decision::Noul}]
+      #   further questions asked in the same request
+      # @param limit [Integer] characters kept per option description
+      def initialize(provider, id:, instructions:, options:, also: {}, limit: DEFAULT_LIMIT)
+        @provider = provider
+        @id = id.to_s
+        @instructions = instructions
+        @options = options
+        @also = also
+        @limit = limit
+
+        if @also.key?(@id)
+          raise ArgumentError, "#{@id.inspect} is asked twice: once as the choice, once in also"
+        end
+      end
+
+      # Ask everything about +state+ in one request.
+      #
+      # @param state [String, Hash, Array] the whole context, sent once
+      # @param model [String, nil] model override
+      # @return [Ask::DecisionResult::Batch]
+      def read(state:, model: nil)
+        @provider.evaluate(state: state, decisions: questions, model: model)
+      end
+
+      # The Choice answer, or nil when nothing came back for it.
+      def choice(batch) = batch[@id]
+
+      private
+
+      def questions
+        {id => choice_question}.merge(@also)
+      end
+
+      def choice_question
+        Ask::Decision::Choice.new(instructions: instructions, criteria: described_options)
+      end
+
+      def described_options
+        options.transform_values { |description| truncate(description) }
+      end
+
+      def truncate(value)
+        text = value.to_s
+        return text if text.length <= @limit
+
+        "#{text[0, @limit - ELISION_ROOM]}…[#{text.length - @limit + ELISION_ROOM} chars elided]"
+      end
+    end
+  end
+end

data/lib/ask/decisions/reflection_judge.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # Decision-based reflection — replaces the LLM self-critique with a
+    # simple Noul: "is there a concrete, actionable improvement?"
+    #
+    #   judge = Ask::Decisions::ReflectionJudge.new(provider)
+    #   verdict = judge.reflect(
+    #     request: "Fix the bug in login.rb",
+    #     response: "I found the issue...",
+    #     attempt: 2
+    #   )
+    #   verdict.improve?  # => true
+    #   verdict.done?     # => false
+    #
+    class ReflectionJudge
+      QUESTIONS = {
+        has_improvement: Ask::Decision::Noul.new(
+          instructions: "Is there a concrete, actionable improvement that would make this response better for the user?"
+        ),
+        quality: Ask::Decision::Score.new(
+          instructions: "How close is this response to being fully satisfactory?",
+          criteria: [
+            "Poor — major issues remain",
+            "Fair — some improvements possible",
+            "Good — minor polish only",
+            "Excellent — ready to deliver"
+          ]
+        )
+      }.freeze
+
+      # @param provider [Ask::DecisionProvider]
+      # @param max_reflections [Integer] stop after this many improvement rounds
+      def initialize(provider, max_reflections: 3)
+        @provider = provider
+        @max_reflections = max_reflections
+      end
+
+      # Decide whether to improve or stop.
+      #
+      # @param request [String] the original user request
+      # @param response [String] the current response
+      # @param attempt [Integer] which reflection round (1-based)
+      # @return [ReflectionVerdict]
+      def reflect(request:, response:, attempt: 1)
+        state = { request: request, response: response, attempt: attempt }
+        result = @provider.evaluate(state: state, decisions: QUESTIONS)
+        ReflectionVerdict.new(result, attempt, @max_reflections)
+      end
+
+      # Verdict from reflection.
+      class ReflectionVerdict
+        attr_reader :result, :attempt, :max_reflections
+
+        def initialize(result, attempt, max_reflections)
+          @result = result
+          @attempt = attempt
+          @max_reflections = max_reflections
+        end
+
+        def improve?
+          noul = @result["has_improvement"]
+          quality = @result["quality"]
+          # Improve if there's a concrete improvement AND quality isn't excellent
+          (noul&.noul || 0) >= 0.5 && (quality&.score || 0) < 3.5
+        end
+
+        def done?
+          !improve? || attempt >= @max_reflections
+        end
+
+        def quality_score
+          @result["quality"]&.score || 0.0
+        end
+
+        def to_s
+          done? ? "done (quality: #{('%.1f' % quality_score)})" : "improve (quality: #{('%.1f' % quality_score)})"
+        end
+      end
+    end
+  end
+end

data/lib/ask/decisions/reranker.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # Reranks retrieved passages using Jev. One Score question per
+    # query–candidate pair, scored on a relevance rubric.
+    #
+    # This is the ask-rag integration: replace a cosine floor with a
+    # per-passage relevance judgment. TypeSafe's CLERC cookbook reports
+    # top-1 accuracy 5% → 18%, top-10 38% → 62%.
+    #
+    #   reranker = Ask::Decisions::Reranker.new(provider)
+    #   ranked = reranker.rerank(
+    #     query: "How do I reset my password?",
+    #     passages: [
+    #       { id: "doc1", text: "To reset your password, go to Settings..." },
+    #       { id: "doc2", text: "Our pricing plans start at $9/month..." }
+    #     ]
+    #   )
+    #   ranked.first  # => { id: "doc1", score: 3.8, confidence: 0.9 }
+    #
+    class Reranker
+      # @param provider [Ask::DecisionProvider]
+      def initialize(provider)
+        @provider = provider
+      end
+
+      # Rerank passages by relevance to a query.
+      #
+      # @param query [String] the search query
+      # @param passages [Array<Hash>] each with :id and :text
+      # @param model [String, nil] model override
+      # @return [Array<Hash>] sorted by score descending, each with :id, :score, :confidence
+      def rerank(query:, passages:, model: nil)
+        return [] if passages.empty?
+
+        questions = passages.each_with_object({}) do |passage, h|
+          id = passage[:id] || passage["id"] || "passage_#{h.size}"
+          text = passage[:text] || passage["text"] || ""
+
+          h["rel_#{id}"] = Ask::Decision::Score.new(
+            instructions: "How relevant is this passage to answering the query?",
+            criteria: [
+              "Not relevant — unrelated to the query",
+              "Marginally relevant — shares some keywords but doesn't address the query",
+              "Relevant — partially answers the query",
+              "Highly relevant — directly answers the query"
+            ]
+          )
+        end
+
+        state = { query: query, passages: passages.map { |p|
+          { id: p[:id] || p["id"], text: truncate(p[:text] || p["text"] || "", 1000) }
+        }}
+
+        result = @provider.evaluate(state: state, decisions: questions, model: model)
+
+        passages.each_with_index.map do |passage, i|
+          id = passage[:id] || passage["id"] || "passage_#{i}"
+          answer = result["rel_#{id}"]
+          {
+            id: id,
+            text: passage[:text] || passage["text"],
+            score: answer&.score || 0.0,
+            confidence: answer&.confidence || 0.0
+          }
+        end.sort_by { |p| -p[:score] }
+      end
+
+      private
+
+      def truncate(str, limit)
+        return "" if str.nil?
+        str.length > limit ? "#{str[0, limit]}…" : str
+      end
+    end
+  end
+end

data/lib/ask/decisions/static.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # A provider that returns canned answers. For tests and offline fixtures.
+    #
+    #   Ask::Decisions.configure do |c|
+    #     c.default_provider = :static
+    #   end
+    #
+    #   # Or per-call:
+    #   Ask::Decisions.decide(
+    #     state: "...",
+    #     decisions: { "route" => Ask::Decision::Choice.new(instructions: "...", criteria: {...}) },
+    #     provider: :static
+    #   )
+    #
+    class Static < Ask::DecisionProvider
+      attr_reader :answers
+
+      def initialize(answers: {})
+        super()
+        @answers = answers.transform_keys(&:to_s).freeze
+      end
+
+      # @param state [String, Hash, Array] ignored
+      # @param decisions [Hash{String => Decision::Choice|Decision::Score|Decision::Noul}]
+      #   used to produce placeholder answers for any missing fixtures
+      # @return [DecisionResult::Batch]
+      def evaluate(state:, decisions:, model: nil)
+        answers = decisions.each_with_object({}) do |(id, decision), h|
+          h[id.to_s] = @answers[id.to_s] || default_answer(id, decision)
+        end
+
+        Ask::DecisionResult::Batch.new(answers: answers, model: "static")
+      end
+
+      private
+
+      def default_answer(id, decision)
+        case decision
+        when Decision::Choice
+          key = decision.criteria.keys.first
+          Ask::DecisionResult::ChoiceAnswer.new(
+            id: id,
+            choice: key,
+            probabilities: Hash[decision.criteria.keys.map { |k| [k, 1.0 / decision.criteria.size] }],
+            confidence: 1.0
+          )
+        when Decision::Score
+          Ask::DecisionResult::ScoreAnswer.new(
+            id: id,
+            score: (decision.criteria.size / 2.0),
+            legend: Hash[decision.criteria.each_with_index.map { |c, i| [i.to_s, c] }],
+            probabilities: Hash[decision.criteria.each_with_index.map { |_, i| [i.to_s, 1.0 / decision.criteria.size] }],
+            confidence: 1.0
+          )
+        when Decision::Noul
+          Ask::DecisionResult::NoulAnswer.new(id: id, noul: 0.5)
+        else
+          raise ArgumentError, "Unknown decision type: #{decision.class}"
+        end
+      end
+    end
+  end
+end

data/lib/ask/decisions/structured_state_loop.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # A structured-state computer-use loop: extract state from a screen
+    # (OCR or DOM), ask Jev to pick the next action, execute it, repeat.
+    #
+    # This is the ask-computer integration. Instead of a vision-LLM loop
+    # (expensive, slow), we use OCR/DOM → Jev → action at ~$0.0002/step.
+    #
+    #   loop = Ask::Decisions::StructuredStateLoop.new(provider)
+    #   result = loop.run(
+    #     goal: "open Safari and search for Ruby gems",
+    #     state_fn: -> { extract_screen_state() },  # returns Hash
+    #     action_fn: ->(action) { execute_action(action) },  # returns void
+    #     max_steps: 20
+    #   )
+    #   result.completed?  # => true
+    #   result.steps       # => 8
+    #
+    class StructuredStateLoop
+      ACTIONS = {
+        click: "Click on an element identified by its position or label",
+        type: "Type text into the focused input field",
+        scroll: "Scroll the page or window in a direction",
+        key: "Press a keyboard key or combination",
+        wait: "Wait for the screen to update",
+        done: "The goal has been achieved"
+      }.freeze
+
+      # @param provider [Ask::DecisionProvider]
+      def initialize(provider)
+        @provider = provider
+      end
+
+      # Run the decision loop until the goal is achieved or max steps hit.
+      #
+      # @param goal [String] what the user wants to accomplish
+      # @param state_fn [Proc] returns a Hash representing the current screen state
+      # @param action_fn [Proc] executes an action Hash, returns void
+      # @param max_steps [Integer] safety limit
+      # @return [LoopResult]
+      def run(goal:, state_fn:, action_fn:, max_steps: 20)
+        history = []
+        max_steps.times do |step|
+          state = state_fn.call
+          decision = decide_next(goal: goal, state: state, history: history)
+
+          if decision.done?
+            return LoopResult.new(completed: true, steps: step + 1, history: history)
+          end
+
+          action = decision.action
+          action_fn.call(action)
+          history << { step: step, state_summary: summarize_state(state), action: action }
+        end
+
+        LoopResult.new(completed: false, steps: max_steps, history: history)
+      end
+
+      private
+
+      def decide_next(goal:, state:, history:)
+        recent = history.last(5).map { |h| "#{h[:action][:type]}: #{h[:action][:target]}" }
+        state_text = summarize_state(state)
+
+        questions = {
+          action_type: Ask::Decision::Choice.new(
+            instructions: "What is the next action to achieve the goal?",
+            criteria: ACTIONS.transform_keys(&:to_s)
+          ),
+          target: Ask::Decision::Noul.new(
+            instructions: "Is the goal already achieved based on the current state?"
+          )
+        }
+
+        result = @provider.evaluate(
+          state: {
+            goal: goal,
+            current_state: state_text,
+            recent_actions: recent,
+            step: history.size
+          },
+          decisions: questions
+        )
+
+        action_type = result["action_type"]&.choice&.to_sym || :wait
+        goal_met = result["target"]&.noul&.>= 0.8
+
+        StepDecision.new(
+          action_type: action_type,
+          goal_met: goal_met,
+          confidence: result["action_type"]&.confidence
+        )
+      end
+
+      def summarize_state(state)
+        return state.to_s if state.is_a?(String)
+        state.map { |k, v| "#{k}: #{v}" }.join("\n")
+      end
+
+      # A single step decision.
+      class StepDecision
+        attr_reader :action_type, :confidence
+
+        def initialize(action_type:, goal_met:, confidence: nil)
+          @action_type = action_type
+          @goal_met = goal_met
+          @confidence = confidence
+        end
+
+        def done? = @goal_met
+
+        def action
+          { type: @action_type, target: nil }
+        end
+      end
+
+      # Result of the loop.
+      class LoopResult
+        attr_reader :steps, :history
+
+        def initialize(completed:, steps:, history:)
+          @completed = completed
+          @steps = steps
+          @history = history
+        end
+
+        def completed? = @completed
+
+        def to_s
+          @completed ? "completed in #{steps} steps" : "not completed after #{steps} steps"
+        end
+      end
+    end
+  end
+end

data/lib/ask/decisions/tool_repairer.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # Decision-based tool call repair — replaces the LLM repair pass
+    # with a Choice over valid tool names and enum values. This is a
+    # real type check, not a prompt-and-hope.
+    #
+    #   repairer = Ask::Decisions::ToolRepairer.new(provider)
+    #   result = repairer.repair(
+    #     attempted_tool: "bsh",  # typo
+    #     attempted_args: { command: "ls" },
+    #     available_tools: [
+    #       { name: "bash", description: "Run a shell command" },
+    #       { name: "read", description: "Read a file" }
+    #     ]
+    #   )
+    #   result.repaired_tool   # => "bash"
+    #   result.repaired?       # => true
+    #
+    class ToolRepairer
+      # @param provider [Ask::DecisionProvider]
+      def initialize(provider)
+        @provider = provider
+      end
+
+      # Attempt to repair a malformed tool call.
+      #
+      # @param attempted_tool [String] the tool name the LLM tried to use
+      # @param attempted_args [Hash] the arguments it passed
+      # @param available_tools [Array<Hash>] the valid tool roster
+      # @return [RepairResult]
+      def repair(attempted_tool:, attempted_args:, available_tools:)
+        tool_names = available_tools.map { |t| t[:name] || t["name"] }
+
+        # If the tool name is valid, try to repair args only.
+        if tool_names.include?(attempted_tool)
+          return repair_args(
+            tool_name: attempted_tool,
+            args: attempted_args,
+            tools: available_tools
+          )
+        end
+
+        # Tool name is invalid — ask Jev to pick the right one.
+        question = Ask::Decision::Choice.new(
+          instructions: "The assistant tried to call a tool named '#{attempted_tool}' which does not exist. " \
+                        "Which available tool is closest to what the assistant intended?",
+          criteria: tool_names.each_with_object({}) { |n, h|
+            desc = available_tools.find { |t| (t[:name] || t["name"]) == n }
+            h[n] = desc ? (desc[:description] || desc["description"] || n) : n
+          }
+        )
+
+        result = @provider.evaluate(
+          state: { attempted_tool: attempted_tool, attempted_args: attempted_args },
+          decisions: { "corrected_tool" => question }
+        )
+
+        corrected = result["corrected_tool"]&.choice
+        RepairResult.new(
+          repaired_tool: corrected || attempted_tool,
+          repaired_args: attempted_args,
+          repaired: corrected && corrected != attempted_tool,
+          confidence: result["corrected_tool"]&.confidence
+        )
+      end
+
+      private
+
+      def repair_args(tool_name:, args:, tools:)
+        tool_def = tools.find { |t| (t[:name] || t["name"]) == tool_name }
+        schema = tool_def&.dig(:params_schema) || tool_def&.dig("params_schema") || {}
+
+        resolver = ArgumentResolver.new(@provider)
+        resolution = resolver.resolve(
+          tool_name: tool_name,
+          params_schema: schema,
+          user_turn: args.to_json
+        )
+
+        merged = resolution.auto_fill_args.merge(args)
+        RepairResult.new(
+          repaired_tool: tool_name,
+          repaired_args: merged,
+          repaired: resolution.needs_generation?,
+          confidence: resolution.confidence
+        )
+      end
+
+      # Result of tool call repair.
+      class RepairResult
+        attr_reader :repaired_tool, :repaired_args, :confidence
+
+        def initialize(repaired_tool:, repaired_args:, repaired:, confidence: nil)
+          @repaired_tool = repaired_tool
+          @repaired_args = repaired_args
+          @repaired = repaired
+          @confidence = confidence
+        end
+
+        def repaired? = @repaired
+
+        def to_s
+          if repaired?
+            "repaired: #{repaired_tool}(#{repaired_args.keys.join(', ')})"
+          else
+            "unrepairable"
+          end
+        end
+      end
+    end
+  end
+end