@miller-tech/uap 1.84.1 → 1.85.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@miller-tech/uap",
-  "version": "1.84.1",
+  "version": "1.85.0",
   "description": "Autonomous AI agent memory system with CLAUDE.md protocol enforcement",
   "type": "module",
   "main": "dist/index.js",

package/tools/agents/scripts/anthropic_proxy.py

@@ -93,6 +93,11 @@ import sys
 import time
 import uuid
 from collections import OrderedDict, defaultdict, deque
+
+try:
+    import confidence_escalation as _ce  # serving-layer Confidence recipe
+except Exception:  # pragma: no cover - fail open if module missing
+    _ce = None
 from dataclasses import dataclass, field
 from pathlib import Path
 
@@ -2792,6 +2797,16 @@ def _should_use_guarded_non_stream(
     if PROXY_MALFORMED_TOOL_STREAM_STRICT and has_tools:
         return True
 
+    # Confidence-escalation (vLLM "Confidence" recipe) needs the FULL answer
+    # buffered to score it before deciding whether to escalate. Buffer single-
+    # answer (no-tool) streaming turns when escalation is enabled. Default OFF.
+    if not has_tools and _ce is not None:
+        try:
+            if _ce.Settings.from_env().enabled:
+                return True
+        except Exception:
+            pass
+
     # A2: when stream-passthrough is enabled, do NOT buffer required-tool turns
     # (native tool_choice='required' constrains them); force-non-stream and
     # malformed-strict above still apply.
@@ -7988,6 +8003,40 @@ def openai_to_anthropic_response(
     }
 
 
+async def _maybe_confidence_escalate(anthropic_resp, anthropic_body, client):
+    """Confidence recipe: if the cheap primary answer scores below threshold,
+    escalate the same (non-tool) turn to the stronger backend and return its
+    answer. Default OFF; fails open (returns the original) on any problem."""
+    if _ce is None or not isinstance(anthropic_resp, dict):
+        return anthropic_resp
+    try:
+        settings = _ce.Settings.from_env()
+        text = _ce.extract_text(anthropic_resp)
+        if not _ce.should_escalate(text, settings, _has_tool_definitions(anthropic_body)):
+            return anthropic_resp
+        payload = _ce.build_escalation_payload(anthropic_body, settings)
+        url = settings.endpoint.rstrip("/") + "/v1/messages"
+        headers = {
+            "Content-Type": "application/json",
+            "x-api-key": settings.api_key,
+            "anthropic-version": "2023-06-01",
+        }
+        resp = await client.post(url, json=payload, headers=headers, timeout=120.0)
+        if resp.status_code == 200:
+            logger.warning(
+                "CONFIDENCE ESCALATION: primary confidence below %.2f -> escalated to %s",
+                settings.threshold, settings.model,
+            )
+            return resp.json()
+        logger.warning(
+            "CONFIDENCE ESCALATION: backend %s returned %d; keeping primary",
+            settings.model, resp.status_code,
+        )
+    except Exception as exc:
+        logger.warning("CONFIDENCE ESCALATION: failed (%s); keeping primary", exc)
+    return anthropic_resp
+
+
 async def _heartbeat_then_buffered(produce_coro, model: str):
     """SSE generator: keep-alive heartbeat wrapper for the guarded-non-stream path.
 
@@ -8953,6 +9002,7 @@ async def messages(request: Request):
                     "REQUIRED TOOL STREAM GUARDRAIL: served stream response via guarded non-stream path"
                 )
 
+            anthropic_resp = await _maybe_confidence_escalate(anthropic_resp, body, client)
             return anthropic_resp
 
         if PROXY_STREAM_HEARTBEAT_SECS > 0:

package/tools/agents/scripts/confidence_escalation.py

@@ -0,0 +1,104 @@
+"""Confidence-escalation looper (vLLM Semantic Router "Confidence" recipe).
+
+Try the cheap primary model first; if a confidence signal on its answer is below
+threshold, escalate the SAME request to a stronger backend and return that
+answer instead. Bounded (one escalation), default OFF, fails open. Scoped to
+non-tool (single-answer) turns — escalating a mid-loop tool turn would need the
+stronger model to share the whole tool/context state, which is out of scope for
+this primitive.
+
+The confidence signal is intentionally pluggable. This module ships a cheap
+text heuristic; a stronger signal (token logprob margin, or UAP's real
+execution/acceptance GATE-pass — the differentiator vs logprob) can replace
+``text_confidence`` without touching the proxy hook.
+
+All config is env-driven and OFF by default:
+  PROXY_CONFIDENCE_ESCALATE=on        enable the looper
+  PROXY_CONFIDENCE_THRESHOLD=0.5      escalate when confidence < this
+  PROXY_ESCALATE_MODEL=<id>           model id sent to the escalation backend
+  PROXY_ESCALATE_ENDPOINT=<url>       Anthropic-compatible /v1/messages base
+  PROXY_ESCALATE_API_KEY=<key>        x-api-key for the escalation backend
+"""
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass
+
+
+@dataclass
+class Settings:
+    enabled: bool
+    threshold: float
+    model: str
+    endpoint: str
+    api_key: str
+
+    @classmethod
+    def from_env(cls) -> "Settings":
+        on = os.environ.get("PROXY_CONFIDENCE_ESCALATE", "off").lower() not in {
+            "", "0", "off", "false", "no",
+        }
+        try:
+            thr = float(os.environ.get("PROXY_CONFIDENCE_THRESHOLD", "0.5"))
+        except ValueError:
+            thr = 0.5
+        return cls(
+            enabled=on,
+            threshold=thr,
+            model=os.environ.get("PROXY_ESCALATE_MODEL", ""),
+            endpoint=os.environ.get("PROXY_ESCALATE_ENDPOINT", ""),
+            api_key=os.environ.get("PROXY_ESCALATE_API_KEY", ""),
+        )
+
+    def backend_configured(self) -> bool:
+        return bool(self.model and self.endpoint)
+
+
+_UNCERTAIN = re.compile(
+    r"\b(i'?m not sure|i am not sure|i don'?t know|i cannot|i can'?t (?:help|do|determine)|"
+    r"unable to|not certain|no idea|as an ai|i'?m sorry,? but)\b",
+    re.I,
+)
+
+
+def text_confidence(text: str) -> float:
+    """Cheap heuristic confidence in [0,1]. Conservative: only clearly weak
+    answers (empty, refusal/uncertainty, trivially short) score low."""
+    t = (text or "").strip()
+    if not t:
+        return 0.0
+    if _UNCERTAIN.search(t):
+        return 0.2
+    if len(t) < 20:
+        return 0.3
+    return 0.9
+
+
+def extract_text(anthropic_resp: dict) -> str:
+    """Concatenate text blocks of an Anthropic message response."""
+    parts = []
+    for blk in (anthropic_resp or {}).get("content", []) or []:
+        if isinstance(blk, dict) and blk.get("type") == "text":
+            parts.append(blk.get("text", ""))
+    return "".join(parts)
+
+
+def should_escalate(text: str, settings: Settings, has_tools: bool) -> bool:
+    if not settings.enabled or not settings.backend_configured():
+        return False
+    if has_tools:  # single-answer scope only
+        return False
+    return text_confidence(text) < settings.threshold
+
+
+def build_escalation_payload(anthropic_body: dict, settings: Settings) -> dict:
+    """Re-issue the same conversation to the stronger model (no tools)."""
+    payload = {
+        "model": settings.model,
+        "max_tokens": anthropic_body.get("max_tokens", 4096),
+        "messages": anthropic_body.get("messages", []),
+    }
+    if anthropic_body.get("system"):
+        payload["system"] = anthropic_body["system"]
+    return payload

package/tools/agents/tests/test_confidence_escalation.py

@@ -0,0 +1,56 @@
+"""Tests for the confidence-escalation looper (vLLM Confidence recipe)."""
+import importlib.util
+import unittest
+from pathlib import Path
+
+mod_path = Path(__file__).resolve().parents[3] / "tools" / "agents" / "scripts" / "confidence_escalation.py"
+spec = importlib.util.spec_from_file_location("confidence_escalation", mod_path)
+ce = importlib.util.module_from_spec(spec)
+import sys as _sys; _sys.modules["confidence_escalation"]=ce; spec.loader.exec_module(ce)
+
+
+def S(enabled=True, threshold=0.5, model="opus", endpoint="http://x/", api_key="k"):
+    return ce.Settings(enabled=enabled, threshold=threshold, model=model, endpoint=endpoint, api_key=api_key)
+
+
+class ConfidenceTest(unittest.TestCase):
+    def test_text_confidence_levels(self):
+        self.assertEqual(ce.text_confidence(""), 0.0)
+        self.assertEqual(ce.text_confidence("I don't know"), 0.2)
+        self.assertEqual(ce.text_confidence("ok"), 0.3)
+        self.assertGreater(ce.text_confidence("Here is a complete, detailed answer to your question."), 0.5)
+
+    def test_extract_text(self):
+        resp = {"content": [{"type": "text", "text": "a"}, {"type": "tool_use"}, {"type": "text", "text": "b"}]}
+        self.assertEqual(ce.extract_text(resp), "ab")
+
+
+class ShouldEscalateTest(unittest.TestCase):
+    def test_low_confidence_escalates(self):
+        self.assertTrue(ce.should_escalate("I cannot help", S(), has_tools=False))
+
+    def test_high_confidence_does_not(self):
+        self.assertFalse(ce.should_escalate("A thorough and confident full answer here.", S(), has_tools=False))
+
+    def test_disabled_never(self):
+        self.assertFalse(ce.should_escalate("", S(enabled=False), has_tools=False))
+
+    def test_no_backend_never(self):
+        self.assertFalse(ce.should_escalate("", S(endpoint=""), has_tools=False))
+
+    def test_tool_turn_never(self):
+        self.assertFalse(ce.should_escalate("", S(), has_tools=True))
+
+
+class PayloadTest(unittest.TestCase):
+    def test_payload_uses_escalation_model_and_messages(self):
+        body = {"model": "qwen", "max_tokens": 100, "system": "sys", "messages": [{"role": "user", "content": "hi"}]}
+        p = ce.build_escalation_payload(body, S(model="opus"))
+        self.assertEqual(p["model"], "opus")
+        self.assertEqual(p["messages"], body["messages"])
+        self.assertEqual(p["system"], "sys")
+        self.assertNotIn("tools", p)
+
+
+if __name__ == "__main__":
+    unittest.main()