ask-decisions 0.2.1 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3dd4560892d89a85e39ed1e73474268724f2f55fca39f1471e9e490e2044168c
-  data.tar.gz: d960989693373e6ce7afe0ddcec17bc88e59addafb89d5269fa67ef52bb9b299
+  metadata.gz: e94c483511119929cad9bbdf2d0925ca498e4b4f98f1119c05ac3c26a239edfc
+  data.tar.gz: e09632e3fbaa5f2dca5ffc8692afe586d1818b985db8afaa600f3498e614ab9f
 SHA512:
-  metadata.gz: 3e2616a2e442dd2c764e5dd3d020a7ba7086f40ba24acca442e272db358817727e4bbaa6c57e84b87fea7c3e64103a6c6a741d7cac24f29f72504708a086b6ea
-  data.tar.gz: 99ea57a135e7f4265245f246f908af5b821b4780ffbf9cafe8f68c94c9d63124e4298666b106c556ccbb9c9aa1be75bb45e690747f9c063fc1616ee053553248
+  metadata.gz: a55fb6e583a983afd05acde0048d203d2e81c3098ef3cfa9d278214a3e4cd2de47ba2bd0f78fcb572f078da1c246fc4dc2aa22f2d23380cd79dc7d5c5fdbfe70
+  data.tar.gz: 57e4a24f195eb27e518e88e7340850e27a90499c2a22ac5a3069cca63d787fee3623f350ccf893fb0794f4eb13667656fd82abff369b9ce71c0c62b30f1a0490

data/CHANGELOG.md

@@ -4,6 +4,27 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
+## [0.2.3] - 2026-09-18
+
+### Added
+
+- **`Ask::Decisions::ThresholdJudge`** — binary accept/reject judge for
+  simple gate decisions. Runs a single Noul question and thresholds the
+  result. Use this when you need "does this pass?" without the full
+  rubric of `QualityJudge`. Returns a `ThresholdVerdict` with `passed?`,
+  `noul`, and `confidence`.
+
+## [0.2.2] - 2026-09-18
+
+### Fixed
+
+- **A confidence of exactly 1.0 lands in the top band of
+  `CalibrationReport#summarize`.** The bands were half-open at both ends
+  (`min <= confidence < max`), so the surest answer a decider can give — 1.0,
+  and in practice the most common one — belonged to no band at all. A report
+  written for setting a threshold was quietly dropping the decisions that most
+  justify one. The top band is now inclusive of 1.0.
+
 ## [0.2.1] - 2026-09-17
 
 ### Added

data/lib/ask/decisions/agent_adapter.rb

@@ -44,6 +44,16 @@ module Ask
         @tool_repairer = ToolRepairer.new(@provider)
       end
 
+      # Build a decision-based compactor for the current provider.
+      #
+      #   compactor = adapter.build_compactor(preserve_recent: 6)
+      #   result = compactor.compact(session.messages)
+      #   result.messages  # => pruned conversation
+      #
+      def build_compactor(**opts)
+        Ask::Decisions::Compactor.new(@provider, **opts)
+      end
+
       # Generate before_tool hooks for ask-agent.
       # Returns an array of callables that match the Hooks interface.
       def before_tool_hooks

data/lib/ask/decisions/calibration_report.rb

@@ -125,9 +125,14 @@ module Ask
         end
 
         # Bucket records by confidence ranges.
+        #
+        # The top band includes 1.0: a confidence of exactly 1.0 is the most
+        # common answer a calibrated decider gives, and with the upper bound
+        # exclusive it fell out of every band — so the surest decisions were
+        # the ones the report never showed.
         def bucket_by_confidence(records)
           ranges = [
-            { min: 0.9, max: 1.0, label: "0.9–1.0" },
+            { min: 0.9, max: 1.0, label: "0.9–1.0", top: true },
             { min: 0.7, max: 0.9, label: "0.7–0.9" },
             { min: 0.5, max: 0.7, label: "0.5–0.7" },
             { min: 0.3, max: 0.5, label: "0.3–0.5" },
@@ -135,7 +140,10 @@ module Ask
           ]
 
           ranges.filter_map do |range|
-            bucket = records.select { |r| r[:confidence] >= range[:min] && r[:confidence] < range[:max] }
+            bucket = records.select do |r|
+              r[:confidence] >= range[:min] &&
+                (range[:top] ? r[:confidence] <= range[:max] : r[:confidence] < range[:max])
+            end
             next if bucket.empty?
             correct = bucket.count { |r| r[:predicted] == r[:outcome] }
             acc = correct.to_f / bucket.size

data/lib/ask/decisions/compactor.rb

@@ -0,0 +1,412 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # Decision-based conversation compactor. Replaces lossy summarization with
+    # Jev-scored pruning: every tool call and result is evaluated for relevance,
+    # and the irrelevant ones are dropped or truncated — everything kept stays
+    # verbatim.
+    #
+    # This is the Ruby equivalent of the fast-jev-compaction approach: rather
+    # than asking an LLM to summarize old turns (which loses file paths, exact
+    # errors, constraints, and command details), Jev scores each tool call and
+    # result independently, and the compactor applies three-tier decisions.
+    #
+    #   compactor = Ask::Decisions::Compactor.new(provider)
+    #   result = compactor.compact(messages)
+    #   result.messages   # => pruned message list
+    #   result.stats      # => { kept: 12, dropped: 5, truncated: 3, ... }
+    #   result.reduction_ratio  # => 0.29 (29% of tool content removed)
+    #
+    # Message format (hashes):
+    #
+    #   { role: "user", content: "Fix the failing test" }
+    #   { role: "assistant", content: "",
+    #     tool_calls: [{ id: "call_1", name: "Read", input: { path: "src/a.rb" } }] }
+    #   { role: "tool", content: "file contents...",
+    #     tool_call_id: "call_1" }
+    #
+    # The first message is always kept. The most recent +preserve_recent+
+    # messages are never touched. Everything else is a candidate for pruning.
+    #
+    class Compactor
+      # Approximate chars per token for budget calculations.
+      CHARS_PER_TOKEN = 4
+
+      # Jev Noul questions asked per tool call.
+      CALL_QUESTION = "Should this tool call be kept — does knowing it was " \
+                       "made still matter for the current conversation?"
+      RESULT_QUESTION = "Should this tool result be kept verbatim — are its " \
+                         "contents still needed and would re-running the tool " \
+                         "not suffice?"
+
+      # @param provider [Ask::DecisionProvider] the Jev/decision provider
+      # @param keep_threshold [Float] minimum Noul probability to keep (0.0–1.0)
+      # @param preserve_recent [Integer] newest messages never touched
+      # @param max_state_tokens [Integer] token ceiling for state sent to Jev
+      # @param max_request_tokens [Integer] token ceiling per batch request
+      # @param truncate_head_chars [Integer] chars retained from dropped results
+      # @param goal [String, nil] ongoing task description for Jev context
+      def initialize(provider,
+                     keep_threshold: 0.5,
+                     preserve_recent: 4,
+                     max_state_tokens: 25_000,
+                     max_request_tokens: 30_000,
+                     truncate_head_chars: 300,
+                     goal: nil)
+        @provider = provider
+        @keep_threshold = keep_threshold
+        @preserve_recent = preserve_recent
+        @max_state_tokens = max_state_tokens
+        @max_request_tokens = max_request_tokens
+        @truncate_head_chars = truncate_head_chars
+        @goal = goal
+      end
+
+      # Compact a conversation by scoring every tool call and result with Jev,
+      # then dropping or truncating the irrelevant ones.
+      #
+      # @param messages [Array<Hash>] conversation messages
+      # @return [Result] compacted messages and stats
+      def compact(messages)
+        messages = normalize(messages)
+        return Result.new(messages.dup, original_count: messages.size) if messages.size < 2
+
+        pairs = build_pairs(messages)
+        pinned = compute_pinned(messages)
+
+        return Result.new(messages.dup, original_count: messages.size) if pairs.empty?
+
+        state = build_state(messages, pairs)
+        questions = build_questions(pairs, pinned)
+        batch = batch_and_execute(state, questions)
+        decisions = extract_decisions(batch, pairs)
+        pruned = apply_decisions(messages, decisions, pinned)
+        stats = compute_stats(messages, pruned, decisions)
+
+        Result.new(pruned, stats: stats, original_count: messages.size)
+      end
+
+      private
+
+      # ── Normalization ────────────────────────────────────────────────
+
+      def normalize(messages)
+        Array(messages).map { |m| m.is_a?(Hash) ? m : hashify(m) }
+      end
+
+      def hashify(msg)
+        {
+          role: msg.respond_to?(:role) ? msg.role.to_s : "unknown",
+          content: msg.respond_to?(:content) ? msg.content.to_s : msg.to_s,
+          tool_calls: extract_tool_calls_from_object(msg),
+          tool_call_id: msg.respond_to?(:tool_call_id) ? msg.tool_call_id : nil
+        }.compact
+      end
+
+      def extract_tool_calls_from_object(msg)
+        return nil unless msg.respond_to?(:tool_calls) && msg.tool_calls
+        Array(msg.tool_calls).map do |tc|
+          {
+            id: tc.respond_to?(:id) ? tc.id : tc[:id],
+            name: tc.respond_to?(:name) ? tc.name : tc[:name],
+            input: tc.respond_to?(:arguments) ? tc.arguments : (tc[:input] || tc[:arguments])
+          }
+        end
+      end
+
+      # ── Pairing ──────────────────────────────────────────────────────
+
+      # Group messages into tool_call / tool_result pairs by tool_call_id.
+      # Returns an array of { call_msg:, result_msg:, call: } hashes.
+      def build_pairs(messages)
+        call_index = {}
+        messages.each_with_index do |msg, idx|
+          Array(msg[:tool_calls]).each do |tc|
+            call_index[tc[:id]] = { call_msg_idx: idx, call: tc }
+          end
+        end
+
+        pairs = []
+        messages.each_with_index do |msg, idx|
+          tcid = msg[:tool_call_id]
+          next unless tcid && call_index[tcid]
+
+          entry = call_index[tcid]
+          pairs << {
+            call_msg_idx: entry[:call_msg_idx],
+            result_msg_idx: idx,
+            call: entry[:call],
+            call_id: tcid
+          }
+        end
+
+        pairs.sort_by { |p| p[:call_msg_idx] }
+      end
+
+      # First message and the most recent +preserve_recent+ messages are pinned.
+      def compute_pinned(messages)
+        pinned = Set.new([0])
+        start = [messages.size - @preserve_recent, 1].max
+        (start...messages.size).each { |i| pinned.add(i) }
+        pinned
+      end
+
+      # ── State ────────────────────────────────────────────────────────
+
+      # Build the state Jev sees: full conversation with tool results replaced
+      # by short placeholder notes. Tool inputs are included, text is included,
+      # nothing is summarized.
+      def build_state(messages, pairs)
+        result_index = {}
+        pairs.each { |p| result_index[p[:result_msg_idx]] = p }
+
+        lines = messages.each_with_index.map do |msg, idx|
+          role = msg[:role] || "unknown"
+          content = msg[:content].to_s
+
+          if result_index[idx]
+            result_index[idx][:result_preview] = content
+            chars = content.length
+            "[tool result: ok, #{chars} chars (omitted)]"
+          elsif msg[:tool_calls] && !msg[:tool_calls].empty?
+            tc_strs = msg[:tool_calls].map do |tc|
+              input_str = format_input(tc[:input])
+              "#{tc[:name]}(#{input_str})"
+            end
+            tc_strs.empty? ? content : "#{content}\n#{tc_strs.join("\n")}"
+          else
+            content
+          end
+        end
+
+        goal_section = @goal ? "\n\nGoal: #{@goal}" : ""
+        { conversation: lines, goal: @goal }.compact
+      end
+
+      def format_input(input)
+        return "" unless input
+        str = input.is_a?(String) ? input : JSON.generate(input)
+        str.length > 100 ? "#{str[0, 100]}..." : str
+      end
+
+      # ── Questions ────────────────────────────────────────────────────
+
+      def build_questions(pairs, pinned)
+        questions = {}
+        pairs.each do |pair|
+          next if pinned.include?(pair[:call_msg_idx]) && pinned.include?(pair[:result_msg_idx])
+
+          call_id = pair[:call_id]
+          tc = pair[:call]
+          input_str = format_input(tc[:input])
+
+          questions["keep_call_#{call_id}"] = Ask::Decision::Noul.new(
+            instructions: "#{CALL_QUESTION}\n\nTool: #{tc[:name]}\nInput: #{input_str}"
+          )
+          questions["keep_result_#{call_id}"] = Ask::Decision::Noul.new(
+            instructions: "#{RESULT_QUESTION}\n\nTool: #{tc[:name]}\nInput: #{input_str}"
+          )
+        end
+        questions
+      end
+
+      # ── Batching ─────────────────────────────────────────────────────
+
+      # Split questions into batches that fit the request token budget,
+      # send each batch concurrently (sequentially in Ruby, but the same
+      # pattern as fast-jev-compaction), and merge results.
+      def batch_and_execute(state, questions)
+        return empty_batch if questions.empty?
+
+        batches = partition_batches(state, questions)
+        return execute_batch(state, questions) if batches.size <= 1
+
+        results = batches.map { |batch| execute_batch(state, batch) }
+        merge_batches(results)
+      end
+
+      def partition_batches(state, questions)
+        state_tokens = estimate_tokens(JSON.generate(state))
+        pairs = questions.to_a
+        batches = []
+        current = {}
+        current_tokens = state_tokens
+
+        pairs.each do |id, decision|
+          q_tokens = estimate_tokens(JSON.generate(decision.to_h))
+          if current_tokens + q_tokens > @max_request_tokens && current.any?
+            batches << current
+            current = {}
+            current_tokens = state_tokens
+          end
+          current[id] = decision
+          current_tokens += q_tokens
+        end
+
+        batches << current if current.any?
+        batches
+      end
+
+      def execute_batch(state, questions)
+        @provider.evaluate(
+          state: JSON.generate(state),
+          decisions: questions
+        )
+      end
+
+      def empty_batch
+        Ask::DecisionResult::Batch.new(answers: {})
+      end
+
+      def merge_batches(batches)
+        merged = {}
+        batches.each do |batch|
+          batch.answers.each { |k, v| merged[k] = v }
+        end
+        Ask::DecisionResult::Batch.new(answers: merged)
+      end
+
+      # ── Decisions ────────────────────────────────────────────────────
+
+      def extract_decisions(batch, pairs)
+        decisions = {}
+        pairs.each do |pair|
+          call_id = pair[:call_id]
+          keep_call_answer = batch["keep_call_#{call_id}"]
+          keep_result_answer = batch["keep_result_#{call_id}"]
+
+          keep_call = keep_call_answer&.noul || 0.0
+          keep_result = keep_result_answer&.noul || 0.0
+
+          decisions[call_id] = {
+            keep_call: keep_call,
+            keep_result: keep_result,
+            action: classify_action(keep_call, keep_result)
+          }
+        end
+        decisions
+      end
+
+      # Three-tier decision classification.
+      def classify_action(keep_call, keep_result)
+        if keep_result >= @keep_threshold
+          :keep
+        elsif keep_call >= @keep_threshold
+          :truncate
+        else
+          :drop
+        end
+      end
+
+      # ── Apply ────────────────────────────────────────────────────────
+
+      def apply_decisions(messages, decisions, pinned)
+        pairs = build_pairs(messages)
+        remove_indices = Set.new
+        truncate_indices = {}
+
+        pairs.each do |pair|
+          decision = decisions[pair[:call_id]]
+          next unless decision
+
+          case decision[:action]
+          when :drop
+            remove_indices.add(pair[:call_msg_idx])
+            remove_indices.add(pair[:result_msg_idx])
+          when :truncate
+            truncate_indices[pair[:result_msg_idx]] = pair[:call]
+          end
+        end
+
+        messages.each_with_index.filter_map do |msg, idx|
+          next if remove_indices.include?(idx)
+
+          if truncate_indices[idx]
+            truncate_message(msg, truncate_indices[idx])
+          else
+            msg
+          end
+        end
+      end
+
+      def truncate_message(msg, call)
+        original = msg[:content].to_s
+        return msg if original.length <= @truncate_head_chars
+
+        head = original[0, @truncate_head_chars]
+        truncated_content = "#{head}\n...[result truncated: #{original.length - @truncate_head_chars} chars omitted — #{call[:name]} result was kept as call-only]"
+        msg.merge(content: truncated_content)
+      end
+
+      # ── Stats ────────────────────────────────────────────────────────
+
+      def compute_stats(original, pruned, decisions)
+        action_counts = decisions.values.tally { |d| d[:action] }
+        original_chars = original.sum { |m| m[:content].to_s.length }
+        pruned_chars = pruned.sum { |m| m[:content].to_s.length }
+
+        {
+          messages_before: original.size,
+          messages_after: pruned.size,
+          kept: action_counts[:keep] || 0,
+          truncated: action_counts[:truncate] || 0,
+          dropped: action_counts[:drop] || 0,
+          tool_pairs_evaluated: decisions.size,
+          chars_before: original_chars,
+          chars_after: pruned_chars,
+          reduction_ratio: original_chars > 0 ? (1.0 - pruned_chars.to_f / original_chars) : 0.0
+        }
+      end
+
+      # ── Token estimation ─────────────────────────────────────────────
+
+      def estimate_tokens(text)
+        str = text.to_s
+        letters = str.count("a-zA-Z")
+        digits = str.count("0-9")
+        others = str.length - letters - digits
+        (letters / 6.0 + digits / 2.0 + others).ceil
+      end
+
+      # ── Batch result ─────────────────────────────────────────────────
+
+      def extract_call_id(answer_id, prefix)
+        answer_id.to_s.sub(/\A#{prefix}/, "")
+      end
+
+      # ── Result ───────────────────────────────────────────────────────
+
+      # Holds the compacted messages and compaction statistics.
+      class Result
+        attr_reader :messages, :stats, :original_count
+
+        def initialize(messages, stats: {}, original_count: nil)
+          @messages = messages
+          @stats = stats
+          @original_count = original_count || messages.size
+        end
+
+        # Fraction of tool content removed (0.0 = nothing removed, 1.0 = everything).
+        def reduction_ratio
+          stats[:reduction_ratio] || 0.0
+        end
+
+        # Whether any tool calls were dropped or truncated.
+        def compacted?
+          (stats[:dropped] || 0) > 0 || (stats[:truncated] || 0) > 0
+        end
+
+        def to_s
+          if compacted?
+            "compacted #{stats[:messages_before]}→#{stats[:messages_after]} messages " \
+            "(#{stats[:dropped]} dropped, #{stats[:truncated]} truncated, " \
+            "#{(reduction_ratio * 100).round(1)}% reduction)"
+          else
+            "no compaction needed (#{original_count} messages)"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ask/decisions/threshold_judge.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Ask
+  module Decisions
+    # Binary accept/reject judge — simpler than QualityJudge when you
+    # just need "does this pass?" without a full rubric. Runs a single
+    # Noul question and thresholds the result.
+    #
+    #   judge = Ask::Decisions::ThresholdJudge.new(provider)
+    #   verdict = judge.evaluate(
+    #     subject: "The code changes look correct and have tests.",
+    #     question: "Is this code change safe to merge?"
+    #   )
+    #   verdict.passed?  # => true
+    #   verdict.confidence  # => 0.92
+    #
+    # Use QualityJudge when you need multi-dimensional scoring.
+    # Use ThresholdJudge when you need a fast binary gate.
+    #
+    class ThresholdJudge
+      # @param provider [Ask::DecisionProvider]
+      def initialize(provider)
+        @provider = provider
+      end
+
+      # Evaluate a subject against a yes/no question.
+      #
+      # @param subject [String] the content to evaluate
+      # @param question [String] the yes/no question
+      # @param threshold [Float] minimum noul value to pass (default 0.7)
+      # @return [ThresholdVerdict]
+      def evaluate(subject:, question:, threshold: 0.7)
+        decisions = {
+          "verdict" => Ask::Decision::Noul.new(
+            instructions: question
+          )
+        }
+
+        result = @provider.evaluate(state: subject, decisions: decisions)
+        ThresholdVerdict.new(result, threshold)
+      end
+
+      # Verdict from threshold evaluation.
+      class ThresholdVerdict
+        attr_reader :result, :threshold
+
+        def initialize(result, threshold)
+          @result = result
+          @threshold = threshold
+        end
+
+        # The raw noul value (0–1, where 1 = strong yes).
+        def noul
+          answer = @result["verdict"]
+          answer&.noul || 0.0
+        end
+
+        # Did the subject pass the threshold?
+        def passed?
+          noul >= @threshold
+        end
+
+        # Confidence: distance from 0.5 (0 = uncertain, 0.5 = certain).
+        def confidence
+          (noul - 0.5).abs
+        end
+
+        def to_s
+          status = passed? ? "PASS" : "FAIL"
+          "#{status} (noul: #{('%.2f' % noul)}, confidence: #{('%.2f' % confidence)})"
+        end
+      end
+    end
+  end
+end

data/lib/ask/decisions/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module Decisions
-    VERSION = "0.2.1"
+    VERSION = "0.2.3"
   end
 end

data/lib/ask-decisions.rb

@@ -21,12 +21,14 @@ require_relative "ask/decisions/failure_classifier"
 require_relative "ask/decisions/loop_detector"
 require_relative "ask/decisions/confidence_policy"
 require_relative "ask/decisions/quality_judge"
+require_relative "ask/decisions/threshold_judge"
 require_relative "ask/decisions/reflection_judge"
 require_relative "ask/decisions/tool_repairer"
 require_relative "ask/decisions/reranker"
 require_relative "ask/decisions/structured_state_loop"
 require_relative "ask/decisions/calibration_report"
 require_relative "ask/decisions/calibration_harness"
+require_relative "ask/decisions/compactor"
 
 # The Decide tool requires ask-tools (Ask::Tool base class). Load conditionally
 # so the core gem works without a tools dependency.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-decisions
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.3
 platform: ruby
 authors:
 - Kaka Ruto
@@ -154,6 +154,7 @@ files:
 - lib/ask/decisions/cache.rb
 - lib/ask/decisions/calibration_harness.rb
 - lib/ask/decisions/calibration_report.rb
+- lib/ask/decisions/compactor.rb
 - lib/ask/decisions/confidence_policy.rb
 - lib/ask/decisions/decision_state.rb
 - lib/ask/decisions/failure_classifier.rb
@@ -168,6 +169,7 @@ files:
 - lib/ask/decisions/reranker.rb
 - lib/ask/decisions/static.rb
 - lib/ask/decisions/structured_state_loop.rb
+- lib/ask/decisions/threshold_judge.rb
 - lib/ask/decisions/tool_repairer.rb
 - lib/ask/decisions/triage.rb
 - lib/ask/decisions/typesafe.rb