ask-decisions 0.1.1

data/lib/ask/decisions/batcher.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # Accumulates questions and executes them in a single API call.
+    #
+    # Jev evaluates all questions in one request in parallel, and adding questions
+    # barely changes latency or cost. This batching discipline is the core
+    # ergonomic of the decision layer — send every question the turn might need
+    # in one call, let Ruby ignore the answers it doesn't reach.
+    #
+    #   result = Ask::Decisions.batch(state: "...") do |b|
+    #     b.ask("route",   Ask::Decision::Choice.new(...))
+    #     b.ask("urgent",  Ask::Decision::Noul.new(...))
+    #     b.ask("quality", Ask::Decision::Score.new(...))
+    #   end
+    #   result["route"].choice
+    #
+    class Batcher
+      attr_reader :provider, :state, :model, :questions
+
+      def initialize(provider, state:, model: nil)
+        @provider = provider
+        @state = state
+        @model = model
+        @questions = {}
+      end
+
+      # Add a question to the batch.
+      #
+      # @param id [String] the question id (used to access the answer)
+      # @param decision [Ask::Decision::Choice, Ask::Decision::Score, Ask::Decision::Noul]
+      # @return [self]
+      def ask(id, decision)
+        @questions[id.to_s] = decision
+        self
+      end
+
+      # Execute all accumulated questions in a single API call.
+      #
+      # @return [Ask::DecisionResult::Batch]
+      def execute
+        return empty_result if @questions.empty?
+
+        @provider.evaluate(
+          state: @state,
+          decisions: @questions,
+          model: @model
+        )
+      end
+
+      private
+
+      def empty_result
+        Ask::DecisionResult::Batch.new(answers: {})
+      end
+    end
+  end
+end

data/lib/ask/decisions/cache.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # Caches DecisionProvider results keyed on (state, questions).
+    # Jev is self-consistent — identical inputs return identical outputs —
+    # so a cache is safe and prevents redundant calls when the same decision
+    # is made multiple times in a turn (e.g. guard + rerun).
+    #
+    #   cached = Ask::Decisions::Cache.new(provider, ttl: 120)
+    #   cached.evaluate(state: s, decisions: qs)  # hits the API
+    #   cached.evaluate(state: s, decisions: qs)  # returns the cached result
+    #
+    class Cache
+      attr_reader :provider, :ttl
+
+      def initialize(provider, ttl: 120)
+        @provider = provider
+        @ttl = ttl
+        @store = {}
+      end
+
+      def evaluate(state:, decisions:, model: nil)
+        key = cache_key(state, decisions, model)
+        entry = @store[key]
+
+        if entry && !expired?(entry)
+          return entry[:result]
+        end
+
+        result = @provider.evaluate(state: state, decisions: decisions, model: model)
+        @store[key] = { result: result, timestamp: Time.now.to_f }
+        result
+      end
+
+      def clear
+        @store.clear
+        nil
+      end
+
+      def size
+        @store.size
+      end
+
+      private
+
+      def cache_key(state, decisions, model)
+        digest = ::JSON.generate({
+          state: state,
+          decisions: decisions.transform_values(&:to_h),
+          model: model
+        })
+        [state_hash(digest), model].join(":")
+      end
+
+      def state_hash(data)
+        # Use a simple digest for cache keys — not cryptographic, just dedup.
+        require "digest" unless defined?(::Digest)
+        Digest::MD5.hexdigest(data)
+      end
+
+      def expired?(entry)
+        (Time.now.to_f - entry[:timestamp]) > @ttl
+      end
+    end
+  end
+end

data/lib/ask/decisions/calibration_harness.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # Runs a set of test cases against a decision provider and produces
+    # a calibration report. Used to measure whether Jev's confidence is
+    # trustworthy on YOUR decisions.
+    #
+    #   harness = Ask::Decisions::CalibrationHarness.new(provider)
+    #
+    #   harness.add_case(
+    #     id: "urgent_message",
+    #     state: "Help! My server is down!",
+    #     decisions: { "urgent" => Ask::Decision::Noul.new(instructions: "Is this urgent?") },
+    #     expected: { "urgent" => { noul_above: 0.7 } }
+    #   )
+    #
+    #   report = harness.run
+    #   puts report
+    #
+    class CalibrationHarness
+      def initialize(provider)
+        @provider = provider
+        @cases = []
+      end
+
+      # Add a test case.
+      #
+      # @param id [String] a descriptive id for this case
+      # @param state [String, Hash] the state to evaluate
+      # @param decisions [Hash] the questions to ask
+      # @param expected [Hash] expected outcomes for assertions:
+      #   - { choice_is: "value" } — check choice answer
+      #   - { noul_above: 0.7 } — check noul is above threshold
+      #   - { noul_below: 0.3 } — check noul is below threshold
+      #   - { score_above: 2.0 } — check score is above threshold
+      #   - { confidence_above: 0.7 } — check confidence
+      # @param runs [Integer] how many times to run this case (for variance)
+      def add_case(id:, state:, decisions:, expected: {}, runs: 1)
+        @cases << { id: id, state: state, decisions: decisions, expected: expected, runs: runs }
+      end
+
+      # Run all test cases and produce a calibration report.
+      #
+      # @return [CalibrationReport::Summary]
+      def run
+        report = CalibrationReport.new
+
+        @cases.each do |tc|
+          tc[:runs].times do |run_idx|
+            result = @provider.evaluate(
+              state: tc[:state],
+              decisions: tc[:decisions]
+            )
+
+            tc[:decisions].each do |decision_id, decision|
+              answer = result[decision_id.to_s]
+              next unless answer
+
+              predicted, confidence = extract_prediction(answer)
+              outcome = check_expected(tc[:expected][decision_id], answer)
+              correct = predicted == outcome
+
+              report.record(
+                decision_id: "#{tc[:id]}.#{decision_id}",
+                confidence: confidence || 0.5,
+                predicted: predicted.to_s,
+                outcome: outcome.to_s,
+                metadata: { run: run_idx, case_id: tc[:id] }
+              )
+            end
+          end
+        end
+
+        report.summarize
+      end
+
+      private
+
+      def extract_prediction(answer)
+        case answer
+        when DecisionResult::ChoiceAnswer
+          [answer.choice, answer.confidence]
+        when DecisionResult::ScoreAnswer
+          [answer.score, answer.confidence]
+        when DecisionResult::NoulAnswer
+          [answer.noul, nil]  # noul has no confidence
+        else
+          [nil, nil]
+        end
+      end
+
+      def check_expected(expected, answer)
+        return "pass" unless expected
+
+        if expected[:choice_is]
+          answer.respond_to?(:choice) && answer.choice == expected[:choice_is] ? "pass" : "fail"
+        elsif expected[:noul_above]
+          answer.respond_to?(:noul) && answer.noul >= expected[:noul_above] ? "pass" : "fail"
+        elsif expected[:noul_below]
+          answer.respond_to?(:noul) && answer.noul <= expected[:noul_below] ? "pass" : "fail"
+        elsif expected[:score_above]
+          answer.respond_to?(:score) && answer.score >= expected[:score_above] ? "pass" : "fail"
+        elsif expected[:confidence_above]
+          answer.respond_to?(:confidence) && answer.confidence && answer.confidence >= expected[:confidence_above] ? "pass" : "fail"
+        else
+          "pass"
+        end
+      end
+    end
+  end
+end

data/lib/ask/decisions/calibration_report.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # Records decision outcomes and produces a calibration report —
+    # reliability curves per decision id, variance across runs, and
+    # threshold sweep.
+    #
+    # This is the measurement loop that makes confidence thresholds
+    # trustworthy. Without it, thresholds are guesses.
+    #
+    #   report = Ask::Decisions::CalibrationReport.new
+    #
+    #   # Record a decision and its outcome.
+    #   report.record(
+    #     decision_id: "tool.route",
+    #     confidence: 0.89,
+    #     predicted: "bash",
+    #     outcome: "bash",  # what actually happened
+    #     latency: 0.12
+    #   )
+    #
+    #   # Generate the report.
+    #   summary = report.summarize
+    #   summary.accuracy        # => 0.92
+    #   summary.calibration     # => 0.85 (Brier-like score)
+    #   summary.by_confidence   # => [{ range: "0.9–1.0", count: 15, accuracy: 0.93 }, ...]
+    #
+    class CalibrationReport
+      attr_reader :records
+
+      def initialize
+        @records = []
+      end
+
+      # Record a decision and its outcome.
+      #
+      # @param decision_id [String] the decision id
+      # @param confidence [Float] the model's confidence
+      # @param predicted [String] what the model chose
+      # @param outcome [String] what actually happened (ground truth)
+      # @param latency [Float, nil] response time in seconds
+      # @param metadata [Hash] extra data (tool name, risk level, etc.)
+      def record(decision_id:, confidence:, predicted:, outcome:, latency: nil, metadata: {})
+        @records << {
+          decision_id: decision_id.to_s,
+          confidence: confidence.to_f,
+          predicted: predicted.to_s,
+          outcome: outcome.to_s,
+          latency: latency,
+          metadata: metadata,
+          timestamp: Time.now.to_f
+        }
+      end
+
+      # Generate the calibration summary.
+      #
+      # @return [Summary]
+      def summarize
+        Summary.new(@records)
+      end
+
+      # Reset all records.
+      def clear
+        @records.clear
+      end
+
+      # Summary of recorded decisions.
+      class Summary
+        attr_reader :total, :correct, :accuracy, :calibration, :by_confidence, :by_decision_id
+
+        def initialize(records)
+          @records = records
+          @total = records.size
+          @correct = records.count { |r| r[:predicted] == r[:outcome] }
+          @accuracy = @total > 0 ? @correct.to_f / @total : 0.0
+          @calibration = compute_calibration(records)
+          @by_confidence = bucket_by_confidence(records)
+          @by_decision_id = group_by_decision_id(records)
+        end
+
+        # Overall accuracy.
+        def accuracy_pct
+          ("%.1f%%" % (@accuracy * 100))
+        end
+
+        # Average latency.
+        def avg_latency
+          latencies = @records.filter_map { |r| r[:latency] }
+          return nil if latencies.empty?
+          latencies.sum / latencies.size
+        end
+
+        # Print a human-readable report.
+        def to_s
+          lines = []
+          lines << "=== Calibration Report ==="
+          lines << "Total decisions: #{@total}"
+          lines << "Correct: #{@correct} (#{accuracy_pct})"
+          lines << "Calibration (Brier): #{('%.4f' % @calibration)}" if @calibration
+          lines << "Avg latency: #{('%.0fms' % (avg_latency * 1000))}" if avg_latency
+          lines << ""
+          lines << "By confidence band:"
+          @by_confidence.each do |band|
+            lines << "  #{band[:range]}: #{band[:count]} decisions, #{band[:accuracy_pct]} accuracy"
+          end
+          lines << ""
+          lines << "By decision id:"
+          @by_decision_id.each do |id, data|
+            lines << "  #{id}: #{data[:count]} decisions, #{data[:accuracy_pct]} accuracy"
+          end
+          lines.join("\n")
+        end
+
+        private
+
+        # Brier-like calibration score: mean squared error between
+        # confidence and outcome (1 if correct, 0 if wrong).
+        def compute_calibration(records)
+          return nil if records.empty?
+          records.sum do |r|
+            actual = r[:predicted] == r[:outcome] ? 1.0 : 0.0
+            (r[:confidence] - actual) ** 2
+          end / records.size
+        end
+
+        # Bucket records by confidence ranges.
+        def bucket_by_confidence(records)
+          ranges = [
+            { min: 0.9, max: 1.0, label: "0.9–1.0" },
+            { min: 0.7, max: 0.9, label: "0.7–0.9" },
+            { min: 0.5, max: 0.7, label: "0.5–0.7" },
+            { min: 0.3, max: 0.5, label: "0.3–0.5" },
+            { min: 0.0, max: 0.3, label: "0.0–0.3" }
+          ]
+
+          ranges.filter_map do |range|
+            bucket = records.select { |r| r[:confidence] >= range[:min] && r[:confidence] < range[:max] }
+            next if bucket.empty?
+            correct = bucket.count { |r| r[:predicted] == r[:outcome] }
+            acc = correct.to_f / bucket.size
+            {
+              range: range[:label],
+              count: bucket.size,
+              accuracy: acc,
+              accuracy_pct: "%.1f%%" % (acc * 100),
+              avg_confidence: bucket.sum { |r| r[:confidence] } / bucket.size
+            }
+          end
+        end
+
+        # Group records by decision id.
+        def group_by_decision_id(records)
+          records.group_by { |r| r[:decision_id] }.transform_values do |group|
+            correct = group.count { |r| r[:predicted] == r[:outcome] }
+            acc = correct.to_f / group.size
+            {
+              count: group.size,
+              accuracy: acc,
+              accuracy_pct: "%.1f%%" % (acc * 100),
+              avg_confidence: group.sum { |r| r[:confidence] } / group.size
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ask/decisions/confidence_policy.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # Gates actions on confidence thresholds. Extends the existing
+    # approval system with a confidence axis: high confidence → act,
+    # medium → require approval, low → escalate to LLM/human.
+    #
+    #   policy = Ask::Decisions::ConfidencePolicy.new
+    #   policy.add_rule(:bash, risk: :low, act_threshold: 0.5, review_threshold: 0.3)
+    #   policy.add_rule(:write, risk: :high, act_threshold: 0.9, review_threshold: 0.7)
+    #
+    #   decision = policy.evaluate(tool: "bash", confidence: 0.6)
+    #   decision.action  # => :act
+    #
+    #   decision = policy.evaluate(tool: "write", confidence: 0.8)
+    #   decision.action  # => :review (need approval)
+    #
+    class ConfidencePolicy
+      # Default rules by risk level. Matches the pi-jev calibration:
+      # read-only tools act at 0.5, destructive tools need 0.9.
+      DEFAULT_RULES = {
+        low:    { act_threshold: 0.5, review_threshold: 0.3 },
+        medium: { act_threshold: 0.7, review_threshold: 0.5 },
+        high:   { act_threshold: 0.9, review_threshold: 0.7 }
+      }.freeze
+
+      def initialize
+        @rules = {}        # tool_name → { risk:, act_threshold:, review_threshold: }
+        @default_risk = :medium
+      end
+
+      # Add a rule for a specific tool.
+      #
+      # @param tool [String, Symbol] the tool name
+      # @param risk [Symbol] :low, :medium, or :high
+      # @param act_threshold [Float, nil] override the act threshold
+      # @param review_threshold [Float, nil] override the review threshold
+      def add_rule(tool, risk: :medium, act_threshold: nil, review_threshold: nil)
+        defaults = DEFAULT_RULES[risk] || DEFAULT_RULES[:medium]
+        @rules[tool.to_s] = {
+          risk: risk,
+          act_threshold: act_threshold || defaults[:act_threshold],
+          review_threshold: review_threshold || defaults[:review_threshold]
+        }
+      end
+
+      # Set the default risk level for tools without explicit rules.
+      def default_risk=(risk)
+        @default_risk = risk
+      end
+
+      # Evaluate whether an action should proceed based on confidence.
+      #
+      # @param tool [String] the tool name
+      # @param confidence [Float, nil] the decision's confidence
+      # @return [PolicyDecision]
+      def evaluate(tool:, confidence: nil)
+        rule = @rules[tool.to_s] || DEFAULT_RULES[@default_risk] || DEFAULT_RULES[:medium]
+
+        action = classify(confidence, rule)
+        PolicyDecision.new(
+          tool: tool,
+          confidence: confidence,
+          action: action,
+          risk: rule[:risk],
+          act_threshold: rule[:act_threshold],
+          review_threshold: rule[:review_threshold]
+        )
+      end
+
+      private
+
+      def classify(confidence, rule)
+        return :escalate if confidence.nil?
+
+        if confidence >= rule[:act_threshold]
+          :act
+        elsif confidence >= rule[:review_threshold]
+          :review
+        else
+          :escalate
+        end
+      end
+
+      # Decision from policy evaluation.
+      class PolicyDecision
+        attr_reader :tool, :confidence, :action, :risk, :act_threshold, :review_threshold
+
+        def initialize(tool:, confidence:, action:, risk:, act_threshold:, review_threshold:)
+          @tool = tool
+          @confidence = confidence
+          @action = action
+          @risk = risk
+          @act_threshold = act_threshold
+          @review_threshold = review_threshold
+        end
+
+        def act?       = action == :act
+        def review?    = action == :review
+        def escalate?  = action == :escalate
+
+        def to_s
+          case action
+          when :act      then "act on #{tool} (confidence #{fmt})"
+          when :review   then "review #{tool} (confidence #{fmt}, need approval)"
+          when :escalate then "escalate #{tool} (confidence #{fmt}, below threshold)"
+          end
+        end
+
+        private
+
+        def fmt
+          confidence ? "%.2f" % confidence : "nil"
+        end
+      end
+    end
+  end
+end

data/lib/ask/decisions/decision_state.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # Projects session state into a compact document for Jev. Filters out
+    # irrelevant fields, truncates to a token budget, and produces the
+    # structured state that System One models expect.
+    #
+    # Jev degrades with irrelevant detail ("context rot"), so the
+    # projection layer is the single most important piece of the
+    # integration: it determines accuracy by deciding what the model sees.
+    #
+    #   state = Ask::Decisions::DecisionState.build(
+    #     user_turn: "check the weather in seattle tomorrow",
+    #     recent_turns: [...],
+    #     tools: [...],
+    #     plan: "Investigate the weather API...",
+    #     budget: 8000
+    #   )
+    #
+    class DecisionState
+      # Approximate chars per token (conservative for mixed content).
+      CHARS_PER_TOKEN = 4
+
+      # Build a compact state document from session components.
+      #
+      # @param user_turn [String] the latest user message
+      # @param recent_turns [Array<Hash>, nil] recent conversation messages
+      # @param tools [Array<Hash>, nil] tool roster (name + description)
+      # @param agent_roster [Array<Hash>, nil] available agents
+      # @param plan [String, nil] current plan/task description
+      # @param memory [Array<String>, nil] relevant memory entries
+      # @param todos [Array<Hash>, nil] current todo items
+      # @param budget [Integer] max characters for the state
+      # @return [Hash] the projected state
+      def self.build(
+        user_turn:,
+        recent_turns: nil,
+        tools: nil,
+        agent_roster: nil,
+        plan: nil,
+        memory: nil,
+        todos: nil,
+        budget: 32_000 * CHARS_PER_TOKEN
+      )
+        state = {}
+        remaining = budget
+
+        # Always include the user turn (highest priority).
+        truncated_turn = truncate(user_turn, remaining - 100)
+        state[:user_turn] = truncated_turn
+        remaining -= truncated_turn.length
+
+        # Add recent turns if there's budget.
+        if recent_turns && remaining > 500
+          recent = truncate_messages(recent_turns, remaining / 2)
+          state[:recent_turns] = recent
+          remaining -= recent.to_s.length
+        end
+
+        # Add tools if there's budget.
+        if tools && remaining > 200
+          tool_summary = tools.map do |t|
+            name = t[:name] || t["name"]
+            desc = t[:description] || t["description"] || ""
+            "#{name}: #{truncate(desc, 80)}"
+          end
+          state[:available_tools] = tool_summary
+          remaining -= tool_summary.to_s.length
+        end
+
+        # Add plan if there's budget.
+        if plan && remaining > 100
+          state[:plan] = truncate(plan, remaining / 3)
+          remaining -= state[:plan].length
+        end
+
+        # Add memory if there's budget.
+        if memory && remaining > 100
+          state[:memory] = memory.first(5).map { |m| truncate(m, 200) }
+          remaining -= state[:memory].to_s.length
+        end
+
+        # Add todos if there's budget.
+        if todos && remaining > 100
+          state[:todos] = todos.first(10).map do |t|
+            { task: t[:task] || t["task"], status: t[:status] || t["status"] }
+          end
+          remaining -= state[:todos].to_s.length
+        end
+
+        state
+      end
+
+      # Truncate a string to a character budget.
+      def self.truncate(str, limit)
+        return "" if str.nil?
+        str = str.to_s
+        return str if str.length <= limit
+        "#{str[0, limit - 20]}…[#{str.length - limit + 20} chars elided]"
+      end
+
+      # Truncate an array of messages to fit within a character budget,
+      # keeping the most recent messages first.
+      def self.truncate_messages(messages, budget)
+        return [] if messages.nil? || messages.empty?
+        result = []
+        used = 0
+        # Walk from most recent to oldest.
+        messages.reverse_each do |msg|
+          text = msg[:content] || msg["content"] || msg.to_s
+          entry = { role: msg[:role] || msg["role"], content: truncate(text, 2000) }
+          entry_len = entry.to_s.length
+          break if used + entry_len > budget
+          result.unshift(entry)
+          used += entry_len
+        end
+        result
+      end
+    end
+  end
+end