ask-decisions 0.1.1

data/lib/ask/decisions/failure_classifier.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # Classifies tool output into failure categories and provides actionable
+    # advice. Wraps the OutputJudge with a retry-oriented interface.
+    #
+    #   classifier = Ask::Decisions::FailureClassifier.new(provider)
+    #   verdict = classifier.classify(
+    #     tool: "bash",
+    #     output: "npm ERR! code ECONNRESET",
+    #     args: { command: "npm test" },
+    #     attempt: 1
+    #   )
+    #   verdict.retryable?     # => true
+    #   verdict.advice         # => "Retry unchanged."
+    #   verdict.should_retry?  # => true (attempt 1 < max_retries)
+    #
+    class FailureClassifier
+      DEFAULT_MAX_RETRIES = 3
+
+      # @param provider [Ask::DecisionProvider]
+      # @param max_retries [Integer] maximum retry attempts before giving up
+      # @param output_limit [Integer] max chars of output to send
+      def initialize(provider, max_retries: DEFAULT_MAX_RETRIES, output_limit: 2000)
+        @judge = OutputJudge.new(provider, output_limit: output_limit)
+        @max_retries = max_retries
+      end
+
+      # Classify a tool result and produce a retry decision.
+      #
+      # @param tool [String] the tool name
+      # @param output [String] the tool's stdout/stderr
+      # @param args [Hash] the original tool arguments
+      # @param attempt [Integer] which attempt this is (1-based)
+      # @return [ClassifyResult]
+      def classify(tool:, output:, args: {}, attempt: 1)
+        judge_result = @judge.judge(tool: tool, output: output, args: args)
+        ClassifyResult.new(judge_result, attempt, @max_retries)
+      end
+
+      # Result of classifying a tool output.
+      class ClassifyResult
+        attr_reader :judge_result, :attempt, :max_retries
+
+        def initialize(judge_result, attempt, max_retries)
+          @judge_result = judge_result
+          @attempt = attempt
+          @max_retries = max_retries
+        end
+
+        def failure_class = judge_result.failure_class
+        def advice = judge_result.advice
+        def leak? = judge_result.leak?
+
+        # Is this a failure that could succeed on retry?
+        def retryable?
+          return false if leak?
+          return false if failure_class == "no_failure"
+          # transient and environment are retryable with backoff
+          %w[transient environment].include?(failure_class)
+        end
+
+        # Should we actually retry right now? (attempt < max_retries)
+        def should_retry?
+          retryable? && attempt < max_retries
+        end
+
+        # Should we stop retrying and report to the user?
+        def give_up?
+          !retryable? || attempt >= max_retries
+        end
+
+        # Human-readable summary of the classification.
+        def to_s
+          if leak?
+            "LEAK detected in output"
+          elsif failure_class == "no_failure"
+            "no failure"
+          else
+            "#{failure_class}: #{advice} (attempt #{attempt}/#{max_retries})"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ask/decisions/gate.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # Pre-tool-call gate: judges intent before a tool executes.
+    # Calibrated thresholds from pi-jev's measured set (0.03–0.99 range on
+    # destructive, 0.10–0.96 on beyond_scope).
+    #
+    #   gate = Ask::Decisions::Gate.new(provider)
+    #   verdict = gate.judge(
+    #     tool: "bash",
+    #     args: { command: "rm -rf src && git push --force origin main" },
+    #     working_dir: "/project",
+    #     user_message: "clean up the old code"
+    #   )
+    #   verdict.passed?    # => false
+    #   verdict.flagged    # => [:destructive, :beyond_scope]
+    #   verdict.to_s       # => "blocked by destructive (0.99) and beyond_scope (0.98)"
+    #
+    class Gate
+      # Default calibrated questions from pi-jev.
+      QUESTIONS = {
+        destructive: Ask::Decision::Noul.new(
+          instructions: "Is this action destructive?"
+        ),
+        exfiltration: Ask::Decision::Noul.new(
+          instructions: "Does this send local data or secrets off-machine?"
+        ),
+        beyond_scope: Ask::Decision::Noul.new(
+          instructions: "Does this affect anything beyond what the user asked for?"
+        ),
+        impact: Ask::Decision::Score.new(
+          instructions: "How much damage if the user did not want it?",
+          criteria: [
+            "No damage — read-only or easily reversible",
+            "Minor — cosmetic or recoverable from version control",
+            "Moderate — data loss or service disruption, reversible with effort",
+            "Severe — permanent data loss, security breach, or system-wide impact"
+          ]
+        )
+      }.freeze
+
+      # Default thresholds from pi-jev calibration.
+      DEFAULT_THRESHOLDS = {
+        destructive: 0.90,
+        exfiltration: 0.70,
+        beyond_scope: 0.85,
+        impact: 2.50  # on the 0–3 scale
+      }.freeze
+
+      # @param provider [Ask::DecisionProvider] the decision provider to use
+      # @param thresholds [Hash] override specific thresholds
+      # @param tools [Array<String>, nil] tools to gate (nil = all)
+      def initialize(provider, thresholds: {}, tools: nil)
+        @provider = provider
+        @thresholds = DEFAULT_THRESHOLDS.merge(thresholds)
+        @tools = tools
+      end
+
+      # Judge a tool call before execution.
+      #
+      # @param tool [String] the tool name
+      # @param args [Hash] the tool arguments
+      # @param working_dir [String, nil] current working directory
+      # @param user_message [String, nil] the latest user message (truncated)
+      # @return [Verdict]
+      def judge(tool:, args:, working_dir: nil, user_message: nil)
+        return Verdict.pass if @tools && !@tools.include?(tool)
+
+        state = build_state(tool: tool, args: args, working_dir: working_dir, user_message: user_message)
+
+        result = @provider.evaluate(
+          state: state,
+          decisions: QUESTIONS
+        )
+
+        Verdict.new(result, @thresholds)
+      end
+
+      private
+
+      def build_state(tool:, args:, working_dir: nil, user_message: nil)
+        {
+          tool: tool,
+          arguments: truncate_values(args, 400),
+          working_directory: working_dir,
+          user_message: truncate_string(user_message, 1200)
+        }.compact
+      end
+
+      def truncate_values(hash, limit)
+        hash.transform_values do |v|
+          v.is_a?(String) && v.length > limit ? "#{v[0, limit]}…[#{v.length - limit} chars elided]" : v
+        end
+      end
+
+      def truncate_string(str, limit)
+        return nil if str.nil?
+        str.length > limit ? "#{str[0, limit]}…" : str
+      end
+
+      # Verdict from a gate judgment.
+      class Verdict
+        attr_reader :result, :flagged, :scores
+
+        def initialize(result, thresholds)
+          @result = result
+          @flagged = []
+          @scores = {}
+
+          thresholds.each do |key, threshold|
+            answer = result[key.to_s]
+            next unless answer
+
+            value = answer.respond_to?(:noul) ? answer.noul : answer.score
+            @scores[key] = value
+
+            if answer.respond_to?(:noul)
+              @flagged << key if value >= threshold
+            else
+              # Score: flagged when the score exceeds the threshold
+              @flagged << key if value && value >= threshold
+            end
+          end
+        end
+
+        def passed? = @flagged.empty?
+        def flagged? = !@flagged.empty?
+
+        def to_s
+          if flagged?
+            "flagged by #{@flagged.map { |k| "#{k} (#{@scores[k]})" }.join(' and ')}"
+          else
+            "passed"
+          end
+        end
+      end
+
+      # Pass-through verdict for ungated tools.
+      class Verdict
+        def self.pass
+          new(Ask::DecisionResult::Batch.new(answers: {}), {})
+        end
+      end
+    end
+  end
+end

data/lib/ask/decisions/lint.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # Lints decision definitions before they hit the API. Warns on patterns
+    # known to reduce Jev's accuracy (from TypeSafe's jaggedness page and
+    # measured failures in pi-jev).
+    #
+    # Run in dev/test as a rake task or at definition time:
+    #
+    #   warnings = Ask::Decisions::Lint.check(decisions)
+    #   warnings.each { |w| puts "WARNING: #{w}" }
+    #
+    module Lint
+      module_function
+
+      # Lint a hash of decisions.
+      #
+      # @param decisions [Hash{String => Ask::Decision::Choice|Score|Noul}]
+      # @return [Array<String>] warnings (empty if clean)
+      def check(decisions)
+        warnings = []
+        decisions.each do |id, decision|
+          warnings.concat(check_one(id, decision))
+        end
+        warnings
+      end
+
+      # @private
+      def check_one(id, decision)
+        w = []
+        instructions = decision.respond_to?(:instructions) ? decision.instructions : ""
+
+        # Reasoning paths in instructions — measured to cause failures
+        # ("cannot be recovered from version control" → 0.77 on destructive instead of 0.99).
+        if instructions.match?(/\b(because|since|unless|therefore|still recoverable|cannot be recovered|which means|this implies)\b/i)
+          w << "#{id}: instructions contain a reasoning path or justification clause. " \
+               "Ask the plain property instead — measured to reduce accuracy."
+        end
+
+        # Math/arithmetic
+        if instructions.match?(/\b(how many|count|total|sum|average|multiply|add up|how much)\b/i)
+          w << "#{id}: instructions ask for counting or arithmetic. " \
+               "Jev is unreliable at math — compute in Ruby."
+        end
+
+        # Date comparison
+        if instructions.match?(/\b(earlier|later|which date|which comes first|how many days between|what day)\b/i)
+          w << "#{id}: instructions ask for date comparison. " \
+               "Extract components as Choices and compare in Ruby."
+        end
+
+        # Large cardinality without warning
+        if decision.respond_to?(:criteria) && decision.criteria.is_a?(Hash) && decision.criteria.size > 255
+          w << "#{id}: #{decision.criteria.size} options exceeds Jev's cardinality limit of 255. " \
+               "Use a two-stage rank → shortlist → rerank."
+        end
+
+        # Score with fewer than 2 levels
+        if decision.is_a?(Decision::Score) && decision.criteria.size < 2
+          w << "#{id}: Score needs at least 2 criteria levels (got #{decision.criteria.size})."
+        end
+
+        # Missing none/other on Choice (soft warning)
+        if decision.is_a?(Decision::Choice) &&
+           !decision.criteria.keys.any? { |k| k.to_s.downcase.match?(/\b(none|other|none of the above|unknown)\b/) }
+          w << "#{id}: Choice without a none/other option. " \
+               "Consider adding one so the model can reject all options."
+        end
+
+        # Derived questions — measured to overlap across phrasings
+        # ("is it safe to run this again unchanged" → 0.73–0.96 vs 0.37–0.66, ambiguous).
+        # Derive the answer from a classification instead.
+        if instructions.match?(/\b(is it safe to|can this be|should this be retried|is it okay to run again)\b/i)
+          w << "#{id}: question asks what code can derive from a classification. " \
+               "Ask the primitive (e.g. failure_class) and derive advice in code."
+        end
+
+        w
+      end
+    end
+  end
+end

data/lib/ask/decisions/loop_detector.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # Detects agent loops using Jev — whether the agent is repeating itself,
+    # making progress, or stuck. Replaces the naive exact-match ×3 approach
+    # with a judgment over the conversation trajectory.
+    #
+    #   detector = Ask::Decisions::LoopDetector.new(provider)
+    #   verdict = detector.check(
+    #     recent_turns: [...],
+    #     current_tool: "bash",
+    #     current_args: { command: "npm test" },
+    #     turn_count: 8
+    #   )
+    #   verdict.stuck?       # => true
+    #   verdict.progressing?  # => false
+    #   verdict.advice        # => "repeat"
+    #
+    class LoopDetector
+      QUESTIONS = {
+        repeating: Ask::Decision::Noul.new(
+          instructions: "Is the assistant repeating the same action or producing the same output as in the recent conversation?"
+        ),
+        stuck: Ask::Decision::Noul.new(
+          instructions: "Is the assistant stuck — unable to make progress on the user's request?"
+        ),
+        progress: Ask::Decision::Score.new(
+          instructions: "How much progress has the assistant made toward completing the user's request?",
+          criteria: [
+            "No progress — same state as before",
+            "Minimal — tried something but it failed or was undone",
+            "Some — partially addressed the request",
+            "Significant — most of the work is done",
+            "Complete — the request appears to be fulfilled"
+          ]
+        )
+      }.freeze
+
+      # Thresholds for loop detection.
+      STUCK_THRESHOLD = 0.7     # noul >= 0.7 means stuck
+      REPEAT_THRESHOLD = 0.7    # noul >= 0.7 means repeating
+      PROGRESS_THRESHOLD = 1.5  # score < 1.5 means little progress
+
+      # @param provider [Ask::DecisionProvider]
+      def initialize(provider)
+        @provider = provider
+      end
+
+      # Check whether the agent is looping or stuck.
+      #
+      # @param recent_turns [Array<Hash>] recent conversation messages
+      # @param current_tool [String, nil] the tool being called now
+      # @param current_args [Hash, nil] its arguments
+      # @param turn_count [Integer] how many turns have happened
+      # @return [LoopVerdict]
+      def check(recent_turns:, current_tool: nil, current_args: nil, turn_count: 0)
+        state = build_state(
+          recent_turns: recent_turns,
+          current_tool: current_tool,
+          current_args: current_args,
+          turn_count: turn_count
+        )
+
+        result = @provider.evaluate(state: state, decisions: QUESTIONS)
+        LoopVerdict.new(result)
+      end
+
+      private
+
+      def build_state(recent_turns:, current_tool:, current_args:, turn_count:)
+        state = {
+          turn_count: turn_count,
+          recent_actions: extract_actions(recent_turns)
+        }
+        if current_tool
+          state[:current_tool] = current_tool
+          state[:current_args] = current_args
+        end
+        state
+      end
+
+      def extract_actions(turns)
+        return [] unless turns
+        turns.last(6).filter_map do |turn|
+          next unless turn[:tool_calls] || turn["tool_calls"]
+          calls = turn[:tool_calls] || turn["tool_calls"]
+          Array(calls).map do |call|
+            name = call[:name] || call["name"]
+            args = call[:arguments] || call["arguments"] || {}
+            "#{name}(#{args.keys.join(', ')})"
+          end
+        end.flatten
+      end
+
+      # Verdict from loop detection.
+      class LoopVerdict
+        attr_reader :result
+
+        def initialize(result)
+          @result = result
+        end
+
+        def repeating?
+          noul = @result["repeating"]
+          noul && noul.noul >= REPEAT_THRESHOLD
+        end
+
+        def stuck?
+          noul = @result["stuck"]
+          noul && noul.noul >= STUCK_THRESHOLD
+        end
+
+        def progress_score
+          score = @result["progress"]
+          score&.score || 0.0
+        end
+
+        def progressing?
+          progress_score >= PROGRESS_THRESHOLD
+        end
+
+        # The recommended action based on the verdict.
+        def advice
+          return "stop" if repeating? && !progressing?
+          return "pivot" if stuck?
+          return "continue" if progressing?
+          "monitor"
+        end
+
+        def to_s
+          parts = []
+          parts << "repeating" if repeating?
+          parts << "stuck" if stuck?
+          parts << "progress: #{('%.1f' % progress_score)}"
+          parts << "→ #{advice}"
+          parts.join(", ")
+        end
+      end
+    end
+  end
+end

data/lib/ask/decisions/mcp_helper.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # Helper for adding the decide tool to an MCP server.
+    #
+    #   require "ask-decisions/mcp_helper"
+    #
+    #   # In your MCP server setup:
+    #   tools = [Ask::Decisions::MCPHelper.tool, ...other_tools...]
+    #   server = Ask::MCP::Adapters::ToolServer.new(tools)
+    #
+    module MCPHelper
+      # Returns an Ask::Tools::Decide instance configured with the
+      # current decision provider. This is the tool to pass to
+      # Ask::MCP::Adapters::ToolServer.
+      def self.tool
+        require "ask/tools/decide"
+        Ask::Tools::Decide.new
+      end
+    end
+  end
+end

data/lib/ask/decisions/output_judge.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # Post-tool-call output judge: screens the result for leaks and classifies
+    # failures. Two questions, one request (~126ms median from pi-jev).
+    #
+    #   judge = Ask::Decisions::OutputJudge.new(provider)
+    #   result = judge.judge(
+    #     tool: "bash",
+    #     output: "npm ERR! code ECONNRESET",
+    #     args: { command: "npm test" }
+    #   )
+    #   result.leak?       # => false
+    #   result.failure_class  # => "transient"
+    #   result.advice      # => "retry unchanged"
+    #
+    class OutputJudge
+      # Calibrated questions from pi-jev.
+      QUESTIONS = {
+        leaks_secret: Ask::Decision::Noul.new(
+          instructions: "Does this output contain a secret, credential, private key, or access token?"
+        ),
+        failure_class: Ask::Decision::Choice.new(
+          instructions: "What kind of failure is this?",
+          criteria: {
+            "no_failure" => "The command succeeded or produced normal output",
+            "transient" => "A temporary error — network timeout, rate limit, DNS hiccup. Safe to retry unchanged.",
+            "environment" => "A missing dependency, wrong version, port in use, or wrong permissions on a file. Fix the environment before retrying.",
+            "code_bug" => "A type error, syntax error, assertion failure, or test failure in the code itself. The code must change.",
+            "permission" => "Access denied, insufficient privileges. Do not retry without elevated permissions.",
+            "user_error" => "Bad invocation, wrong arguments, or incorrect usage by the caller."
+          }
+        )
+      }.freeze
+
+      # Advice table: maps failure class to a one-line instruction.
+      # Derived from pi-jev's CLASS_ADVICE — adding a class is a row.
+      ADVICE = {
+        "no_failure"  => nil,
+        "transient"   => "Retry unchanged.",
+        "environment" => "Fix the environment before retrying (missing dependency, port conflict, wrong version).",
+        "code_bug"    => "The code must change — do not retry the same command.",
+        "permission"  => "Do not retry without elevated permissions.",
+        "user_error"  => "Fix the invocation arguments."
+      }.freeze
+
+      # @param provider [Ask::DecisionProvider]
+      # @param tools [Array<String>, nil] tools to judge (nil = ["bash"])
+      # @param leak_threshold [Float] noul threshold for leak detection
+      # @param failure_threshold [Float] confidence threshold for failure classification
+      # @param output_limit [Integer] max characters of output to send
+      def initialize(provider, tools: nil, leak_threshold: 0.90, failure_threshold: 0.60, output_limit: 2000)
+        @provider = provider
+        @tools = tools || ["bash"]
+        @leak_threshold = leak_threshold
+        @failure_threshold = failure_threshold
+        @output_limit = output_limit
+      end
+
+      # Judge a tool result after execution.
+      #
+      # @param tool [String] the tool name
+      # @param output [String] the tool's stdout/stderr
+      # @param args [Hash] the original tool arguments
+      # @return [OutputResult]
+      def judge(tool:, output:, args: {})
+        return OutputResult.empty unless @tools.include?(tool)
+
+        truncated = truncate(output, @output_limit)
+        state = { output: truncated, tool_arguments: truncate_values(args, 400) }
+
+        result = @provider.evaluate(state: state, decisions: QUESTIONS)
+        OutputResult.new(result, @leak_threshold, @failure_threshold)
+      end
+
+      private
+
+      def truncate(str, limit)
+        return "" if str.nil?
+        str.length > limit ? "#{str[0, limit]}…[#{str.length - limit} chars elided]" : str
+      end
+
+      def truncate_values(hash, limit)
+        hash.transform_values do |v|
+          v.is_a?(String) && v.length > limit ? "#{v[0, limit]}…[#{v.length - limit} chars elided]" : v
+        end
+      end
+
+      # Typed result from the output judge.
+      class OutputResult
+        attr_reader :leak_noul, :failure_class, :failure_confidence, :advice
+
+        def initialize(batch, leak_threshold, failure_threshold)
+          leak_answer = batch["leaks_secret"]
+          failure_answer = batch["failure_class"]
+
+          @leak_noul = leak_answer&.noul || 0.0
+          @leak_threshold = leak_threshold
+
+          @failure_class = failure_answer&.choice || "no_failure"
+          @failure_confidence = failure_answer&.confidence || 0.0
+          @failure_threshold = failure_threshold
+          @advice = ADVICE[@failure_class]
+        end
+
+        def leak?
+          @leak_noul >= @leak_threshold
+        end
+
+        def failure?
+          @failure_class != "no_failure" && @failure_confidence >= @failure_threshold
+        end
+
+        def to_s
+          parts = []
+          parts << "LEAK (#{@leak_noul})" if leak?
+          parts << "#{@failure_class} (#{'%.2f' % @failure_confidence})" if failure?
+          parts.empty? ? "clean" : parts.join(", ")
+        end
+      end
+
+      # Empty result when tool is not judged.
+      def self.empty_result_class
+        OutputResult
+      end
+
+      class OutputResult
+        def self.empty
+          new(Ask::DecisionResult::Batch.new(answers: {}), 0.9, 0.6)
+        end
+      end
+    end
+  end
+end