@miller-tech/uap 1.85.0 → 1.86.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@miller-tech/uap",
-  "version": "1.85.0",
+  "version": "1.86.0",
   "description": "Autonomous AI agent memory system with CLAUDE.md protocol enforcement",
   "type": "module",
   "main": "dist/index.js",

package/tools/agents/scripts/anthropic_proxy.py

@@ -8003,38 +8003,68 @@ def openai_to_anthropic_response(
     }
 
 
-async def _maybe_confidence_escalate(anthropic_resp, anthropic_body, client):
-    """Confidence recipe: if the cheap primary answer scores below threshold,
-    escalate the same (non-tool) turn to the stronger backend and return its
-    answer. Default OFF; fails open (returns the original) on any problem."""
+async def _maybe_apply_recipe(anthropic_resp, anthropic_body, openai_body, client):
+    """Serving-layer recipe runtime (confidence / fusion, signal-selected).
+    Default OFF; fails open. call_primary re-runs the cheap primary (llama) for
+    fusion breadth; call_judge talks to the stronger escalation backend."""
     if _ce is None or not isinstance(anthropic_resp, dict):
         return anthropic_resp
     try:
         settings = _ce.Settings.from_env()
-        text = _ce.extract_text(anthropic_resp)
-        if not _ce.should_escalate(text, settings, _has_tool_definitions(anthropic_body)):
+        if not settings.enabled:
             return anthropic_resp
-        payload = _ce.build_escalation_payload(anthropic_body, settings)
-        url = settings.endpoint.rstrip("/") + "/v1/messages"
-        headers = {
-            "Content-Type": "application/json",
-            "x-api-key": settings.api_key,
-            "anthropic-version": "2023-06-01",
-        }
-        resp = await client.post(url, json=payload, headers=headers, timeout=120.0)
-        if resp.status_code == 200:
+        model_name = (openai_body or {}).get("model", "default")
+
+        async def call_primary(openai_variant):
+            try:
+                r = await _post_with_generation_timeout(
+                    client,
+                    f"{LLAMA_CPP_BASE}/chat/completions",
+                    openai_variant,
+                    {"Content-Type": "application/json"},
+                )
+                if getattr(r, "status_code", 0) != 200:
+                    return None
+                return openai_to_anthropic_response(r.json(), model_name)
+            except Exception:
+                return None
+
+        async def call_judge(anthropic_payload):
+            try:
+                url = settings.endpoint.rstrip("/") + "/v1/messages"
+                r = await client.post(
+                    url,
+                    json=anthropic_payload,
+                    headers={
+                        "Content-Type": "application/json",
+                        "x-api-key": settings.api_key,
+                        "anthropic-version": "2023-06-01",
+                    },
+                    timeout=120.0,
+                )
+                return r.json() if getattr(r, "status_code", 0) == 200 else None
+            except Exception:
+                return None
+
+        result = await _ce.apply_recipe(
+            anthropic_resp,
+            anthropic_body,
+            openai_body,
+            settings,
+            _has_tool_definitions(anthropic_body),
+            call_primary,
+            call_judge,
+        )
+        if isinstance(result, dict) and result is not anthropic_resp:
             logger.warning(
-                "CONFIDENCE ESCALATION: primary confidence below %.2f -> escalated to %s",
-                settings.threshold, settings.model,
+                "RECIPE applied: recipe=%s signal=%s -> response changed",
+                _ce.select_recipe(anthropic_body, settings, _has_tool_definitions(anthropic_body)),
+                settings.signal,
             )
-            return resp.json()
-        logger.warning(
-            "CONFIDENCE ESCALATION: backend %s returned %d; keeping primary",
-            settings.model, resp.status_code,
-        )
+        return result if isinstance(result, dict) else anthropic_resp
     except Exception as exc:
-        logger.warning("CONFIDENCE ESCALATION: failed (%s); keeping primary", exc)
-    return anthropic_resp
+        logger.warning("RECIPE: failed (%s); keeping primary answer", exc)
+        return anthropic_resp
 
 
 async def _heartbeat_then_buffered(produce_coro, model: str):
@@ -9002,7 +9032,7 @@ async def messages(request: Request):
                     "REQUIRED TOOL STREAM GUARDRAIL: served stream response via guarded non-stream path"
                 )
 
-            anthropic_resp = await _maybe_confidence_escalate(anthropic_resp, body, client)
+            anthropic_resp = await _maybe_apply_recipe(anthropic_resp, body, openai_body, client)
             return anthropic_resp
 
         if PROXY_STREAM_HEARTBEAT_SECS > 0:

package/tools/agents/scripts/confidence_escalation.py

@@ -1,26 +1,32 @@
-"""Confidence-escalation looper (vLLM Semantic Router "Confidence" recipe).
-
-Try the cheap primary model first; if a confidence signal on its answer is below
-threshold, escalate the SAME request to a stronger backend and return that
-answer instead. Bounded (one escalation), default OFF, fails open. Scoped to
-non-tool (single-answer) turns — escalating a mid-loop tool turn would need the
-stronger model to share the whole tool/context state, which is out of scope for
-this primitive.
-
-The confidence signal is intentionally pluggable. This module ships a cheap
-text heuristic; a stronger signal (token logprob margin, or UAP's real
-execution/acceptance GATE-pass — the differentiator vs logprob) can replace
-``text_confidence`` without touching the proxy hook.
-
-All config is env-driven and OFF by default:
-  PROXY_CONFIDENCE_ESCALATE=on        enable the looper
-  PROXY_CONFIDENCE_THRESHOLD=0.5      escalate when confidence < this
-  PROXY_ESCALATE_MODEL=<id>           model id sent to the escalation backend
-  PROXY_ESCALATE_ENDPOINT=<url>       Anthropic-compatible /v1/messages base
-  PROXY_ESCALATE_API_KEY=<key>        x-api-key for the escalation backend
+"""Serving-layer recipe runtime (vLLM Semantic Router micro-agent patterns).
+
+One bounded micro-agent loop behind one model API. Recipes:
+  - single     : no collaboration (default).
+  - confidence : try the cheap primary; if a confidence signal is below
+                 threshold, escalate the same single-answer turn to a stronger
+                 backend (#1). The signal is pluggable: "heuristic" (cheap text)
+                 or "selfverify" (a judge model rates the answer 0-10 -- the
+                 gate-as-confidence idea at the serving layer, #5).
+  - fusion     : run N breadth candidates from the primary, then a judge picks
+                 the best; falls back to the primary on judge failure (#3).
+A signal-driven selector (#2) chooses the recipe per request.
+
+All orchestration is via INJECTED async callables (call_primary, call_judge) so
+it is testable without a live backend. The proxy supplies real ones. Everything
+is OFF by default and fails open.
+
+Env:
+  PROXY_CONFIDENCE_ESCALATE=on        master switch
+  PROXY_RECIPE=auto|single|confidence|fusion   (default auto)
+  PROXY_CONFIDENCE_SIGNAL=heuristic|selfverify (default heuristic)
+  PROXY_CONFIDENCE_THRESHOLD=0.5
+  PROXY_FUSION_N=3                    candidates incl. the primary (2..6)
+  PROXY_AUTO_FUSION_CHARS=600         auto: prompts longer than this -> fusion
+  PROXY_ESCALATE_MODEL / _ENDPOINT / _API_KEY   stronger backend (judge/escalate)
 """
 from __future__ import annotations
 
+import asyncio
 import os
 import re
 from dataclasses import dataclass
@@ -29,23 +35,39 @@ from dataclasses import dataclass
 @dataclass
 class Settings:
     enabled: bool
+    recipe: str
+    signal: str
     threshold: float
+    fusion_n: int
+    auto_fusion_chars: int
     model: str
     endpoint: str
     api_key: str
 
     @classmethod
     def from_env(cls) -> "Settings":
-        on = os.environ.get("PROXY_CONFIDENCE_ESCALATE", "off").lower() not in {
-            "", "0", "off", "false", "no",
-        }
-        try:
-            thr = float(os.environ.get("PROXY_CONFIDENCE_THRESHOLD", "0.5"))
-        except ValueError:
-            thr = 0.5
+        def flag(name, default="off"):
+            return os.environ.get(name, default).lower() not in {"", "0", "off", "false", "no"}
+
+        def num(name, default, cast=float):
+            try:
+                return cast(os.environ.get(name, str(default)))
+            except (ValueError, TypeError):
+                return default
+
+        recipe = os.environ.get("PROXY_RECIPE", "auto").lower()
+        if recipe not in {"auto", "single", "confidence", "fusion"}:
+            recipe = "auto"
+        signal = os.environ.get("PROXY_CONFIDENCE_SIGNAL", "heuristic").lower()
+        if signal not in {"heuristic", "selfverify"}:
+            signal = "heuristic"
         return cls(
-            enabled=on,
-            threshold=thr,
+            enabled=flag("PROXY_CONFIDENCE_ESCALATE"),
+            recipe=recipe,
+            signal=signal,
+            threshold=num("PROXY_CONFIDENCE_THRESHOLD", 0.5, float),
+            fusion_n=max(2, min(6, num("PROXY_FUSION_N", 3, int))),
+            auto_fusion_chars=num("PROXY_AUTO_FUSION_CHARS", 600, int),
             model=os.environ.get("PROXY_ESCALATE_MODEL", ""),
             endpoint=os.environ.get("PROXY_ESCALATE_ENDPOINT", ""),
             api_key=os.environ.get("PROXY_ESCALATE_API_KEY", ""),
@@ -55,6 +77,41 @@ class Settings:
         return bool(self.model and self.endpoint)
 
 
+# ---- helpers --------------------------------------------------------------
+def latest_user_text(anthropic_body: dict) -> str:
+    for msg in reversed((anthropic_body or {}).get("messages", []) or []):
+        if msg.get("role") != "user":
+            continue
+        c = msg.get("content")
+        if isinstance(c, str):
+            return c
+        if isinstance(c, list):
+            return "".join(b.get("text", "") for b in c if isinstance(b, dict) and b.get("type") == "text")
+    return ""
+
+
+def extract_text(anthropic_resp: dict) -> str:
+    parts = []
+    for blk in (anthropic_resp or {}).get("content", []) or []:
+        if isinstance(blk, dict) and blk.get("type") == "text":
+            parts.append(blk.get("text", ""))
+    return "".join(parts)
+
+
+# ---- #2 recipe selection --------------------------------------------------
+def select_recipe(anthropic_body: dict, settings: Settings, has_tools: bool) -> str:
+    if not settings.enabled or has_tools:
+        return "single"
+    if settings.recipe != "auto":
+        return settings.recipe
+    # Auto: signal-driven. Longer/harder prompts have higher reasoning variance
+    # -> fusion (breadth+judge); shorter -> confidence (cheap, escalate if weak).
+    if len(latest_user_text(anthropic_body)) >= settings.auto_fusion_chars and settings.backend_configured():
+        return "fusion"
+    return "confidence"
+
+
+# ---- #1/#5 confidence signal ----------------------------------------------
 _UNCERTAIN = re.compile(
     r"\b(i'?m not sure|i am not sure|i don'?t know|i cannot|i can'?t (?:help|do|determine)|"
     r"unable to|not certain|no idea|as an ai|i'?m sorry,? but)\b",
@@ -63,8 +120,6 @@ _UNCERTAIN = re.compile(
 
 
 def text_confidence(text: str) -> float:
-    """Cheap heuristic confidence in [0,1]. Conservative: only clearly weak
-    answers (empty, refusal/uncertainty, trivially short) score low."""
     t = (text or "").strip()
     if not t:
         return 0.0
@@ -75,30 +130,115 @@ def text_confidence(text: str) -> float:
     return 0.9
 
 
-def extract_text(anthropic_resp: dict) -> str:
-    """Concatenate text blocks of an Anthropic message response."""
-    parts = []
-    for blk in (anthropic_resp or {}).get("content", []) or []:
-        if isinstance(blk, dict) and blk.get("type") == "text":
-            parts.append(blk.get("text", ""))
-    return "".join(parts)
+def build_verify_payload(anthropic_body: dict, answer_text: str, settings: Settings) -> dict:
+    q = latest_user_text(anthropic_body)
+    prompt = (
+        "Rate from 0 to 10 how well this ANSWER satisfies the REQUEST "
+        "(0 = wrong or incomplete, 10 = fully correct and complete). "
+        "Reply with ONLY the number.\n\nREQUEST:\n" + q + "\n\nANSWER:\n" + answer_text
+    )
+    return {"model": settings.model, "max_tokens": 16,
+            "messages": [{"role": "user", "content": prompt}]}
 
 
-def should_escalate(text: str, settings: Settings, has_tools: bool) -> bool:
-    if not settings.enabled or not settings.backend_configured():
-        return False
-    if has_tools:  # single-answer scope only
-        return False
-    return text_confidence(text) < settings.threshold
+def parse_verify_score(text: str):
+    m = re.search(r"\d+(?:\.\d+)?", text or "")
+    if not m:
+        return None
+    return max(0.0, min(1.0, float(m.group(0)) / 10.0))
 
 
+# ---- escalation / fusion payloads -----------------------------------------
 def build_escalation_payload(anthropic_body: dict, settings: Settings) -> dict:
-    """Re-issue the same conversation to the stronger model (no tools)."""
-    payload = {
-        "model": settings.model,
-        "max_tokens": anthropic_body.get("max_tokens", 4096),
-        "messages": anthropic_body.get("messages", []),
-    }
+    payload = {"model": settings.model,
+               "max_tokens": anthropic_body.get("max_tokens", 4096),
+               "messages": anthropic_body.get("messages", [])}
     if anthropic_body.get("system"):
         payload["system"] = anthropic_body["system"]
     return payload
+
+
+def build_fusion_variants(openai_body: dict, n: int) -> list[dict]:
+    """N-1 extra primary variants (the live response is candidate 0). Vary
+    temperature for breadth; everything else identical."""
+    out = []
+    for i in range(max(0, n - 1)):
+        v = dict(openai_body)
+        v["stream"] = False
+        v["temperature"] = round(0.4 + 0.2 * i, 2)
+        out.append(v)
+    return out
+
+
+def build_judge_payload(anthropic_body: dict, candidate_texts: list[str], settings: Settings) -> dict:
+    q = latest_user_text(anthropic_body)
+    listing = "\n\n".join(f"[{i}]\n{t}" for i, t in enumerate(candidate_texts))
+    prompt = (
+        "You are a strict judge. Choose the SINGLE best answer to the REQUEST. "
+        "Reply with ONLY the index number of the best answer.\n\nREQUEST:\n"
+        + q + "\n\nANSWERS:\n" + listing
+    )
+    return {"model": settings.model, "max_tokens": 8,
+            "messages": [{"role": "user", "content": prompt}]}
+
+
+def parse_judge_index(text: str, n: int):
+    m = re.search(r"\d+", text or "")
+    if not m:
+        return None
+    i = int(m.group(0))
+    return i if 0 <= i < n else None
+
+
+def should_escalate(text: str, settings: Settings, has_tools: bool) -> bool:
+    """Back-compat (heuristic confidence path)."""
+    if not settings.enabled or not settings.backend_configured() or has_tools:
+        return False
+    return text_confidence(text) < settings.threshold
+
+
+# ---- orchestration (injected callables) -----------------------------------
+async def _confidence_score(text, anthropic_body, settings, call_judge):
+    if settings.signal == "selfverify" and settings.backend_configured() and call_judge is not None:
+        jr = await call_judge(build_verify_payload(anthropic_body, text, settings))
+        score = parse_verify_score(extract_text(jr)) if isinstance(jr, dict) else None
+        if score is not None:
+            return score
+    return text_confidence(text)
+
+
+async def apply_recipe(primary_resp, anthropic_body, openai_body, settings, has_tools,
+                       call_primary, call_judge):
+    """Dispatch to the selected recipe. Returns an anthropic response dict.
+    call_primary(openai_variant)->anthropic_resp|None ; call_judge(anthropic_payload)->anthropic_resp|None.
+    Fails open: returns primary_resp on any problem."""
+    try:
+        recipe = select_recipe(anthropic_body, settings, has_tools)
+        if recipe == "single" or not isinstance(primary_resp, dict):
+            return primary_resp
+        primary_text = extract_text(primary_resp)
+
+        if recipe == "confidence":
+            conf = await _confidence_score(primary_text, anthropic_body, settings, call_judge)
+            if conf < settings.threshold and settings.backend_configured() and call_judge is not None:
+                esc = await call_judge(build_escalation_payload(anthropic_body, settings))
+                if isinstance(esc, dict):
+                    return esc
+            return primary_resp
+
+        if recipe == "fusion":
+            variants = build_fusion_variants(openai_body or {}, settings.fusion_n)
+            results = await asyncio.gather(*[call_primary(v) for v in variants],
+                                           return_exceptions=True)
+            candidates = [primary_resp] + [r for r in results if isinstance(r, dict)]
+            if len(candidates) <= 1 or not settings.backend_configured() or call_judge is None:
+                return primary_resp
+            texts = [extract_text(c) for c in candidates]
+            jr = await call_judge(build_judge_payload(anthropic_body, texts, settings))
+            idx = parse_judge_index(extract_text(jr), len(candidates)) if isinstance(jr, dict) else None
+            if idx is not None:
+                return candidates[idx]
+            return primary_resp  # fallback to best valid evidence
+    except Exception:
+        return primary_resp
+    return primary_resp

package/tools/agents/tests/test_confidence_escalation.py

@@ -1,55 +1,108 @@
-"""Tests for the confidence-escalation looper (vLLM Confidence recipe)."""
+"""Tests for the serving-layer recipe runtime (Confidence #1/#5, Fusion #3, selector #2)."""
+import asyncio
 import importlib.util
+import sys
 import unittest
 from pathlib import Path
 
 mod_path = Path(__file__).resolve().parents[3] / "tools" / "agents" / "scripts" / "confidence_escalation.py"
 spec = importlib.util.spec_from_file_location("confidence_escalation", mod_path)
 ce = importlib.util.module_from_spec(spec)
-import sys as _sys; _sys.modules["confidence_escalation"]=ce; spec.loader.exec_module(ce)
+sys.modules["confidence_escalation"] = ce
+spec.loader.exec_module(ce)
 
 
-def S(enabled=True, threshold=0.5, model="opus", endpoint="http://x/", api_key="k"):
-    return ce.Settings(enabled=enabled, threshold=threshold, model=model, endpoint=endpoint, api_key=api_key)
+def S(enabled=True, recipe="auto", signal="heuristic", threshold=0.5, fusion_n=3,
+      auto_chars=600, model="opus", endpoint="http://x/", key="k"):
+    return ce.Settings(enabled=enabled, recipe=recipe, signal=signal, threshold=threshold,
+                       fusion_n=fusion_n, auto_fusion_chars=auto_chars, model=model,
+                       endpoint=endpoint, api_key=key)
 
 
-class ConfidenceTest(unittest.TestCase):
-    def test_text_confidence_levels(self):
+def resp(text):
+    return {"content": [{"type": "text", "text": text}]}
+
+
+def body(user="hi"):
+    return {"model": "qwen", "max_tokens": 100, "messages": [{"role": "user", "content": user}]}
+
+
+def run(coro):
+    return asyncio.new_event_loop().run_until_complete(coro)
+
+
+class HelpersTest(unittest.TestCase):
+    def test_text_confidence(self):
         self.assertEqual(ce.text_confidence(""), 0.0)
         self.assertEqual(ce.text_confidence("I don't know"), 0.2)
-        self.assertEqual(ce.text_confidence("ok"), 0.3)
-        self.assertGreater(ce.text_confidence("Here is a complete, detailed answer to your question."), 0.5)
+        self.assertGreater(ce.text_confidence("A thorough complete answer to the question."), 0.5)
 
-    def test_extract_text(self):
-        resp = {"content": [{"type": "text", "text": "a"}, {"type": "tool_use"}, {"type": "text", "text": "b"}]}
-        self.assertEqual(ce.extract_text(resp), "ab")
+    def test_parse_verify_score(self):
+        self.assertAlmostEqual(ce.parse_verify_score("8"), 0.8)
+        self.assertEqual(ce.parse_verify_score("score: 3 of 10"), 0.3)
+        self.assertIsNone(ce.parse_verify_score("n/a"))
 
+    def test_parse_judge_index(self):
+        self.assertEqual(ce.parse_judge_index("The best is [2]", 3), 2)
+        self.assertIsNone(ce.parse_judge_index("9", 3))
 
-class ShouldEscalateTest(unittest.TestCase):
-    def test_low_confidence_escalates(self):
-        self.assertTrue(ce.should_escalate("I cannot help", S(), has_tools=False))
+    def test_fusion_variants(self):
+        v = ce.build_fusion_variants({"model": "qwen", "messages": []}, 3)
+        self.assertEqual(len(v), 2)  # n-1 (primary is candidate 0)
+        self.assertEqual([x["temperature"] for x in v], [0.4, 0.6])
 
-    def test_high_confidence_does_not(self):
-        self.assertFalse(ce.should_escalate("A thorough and confident full answer here.", S(), has_tools=False))
 
-    def test_disabled_never(self):
-        self.assertFalse(ce.should_escalate("", S(enabled=False), has_tools=False))
+class SelectorTest(unittest.TestCase):  # #2
+    def test_disabled_or_tools_single(self):
+        self.assertEqual(ce.select_recipe(body(), S(enabled=False), False), "single")
+        self.assertEqual(ce.select_recipe(body(), S(), True), "single")
 
-    def test_no_backend_never(self):
-        self.assertFalse(ce.should_escalate("", S(endpoint=""), has_tools=False))
+    def test_explicit_recipe(self):
+        self.assertEqual(ce.select_recipe(body(), S(recipe="fusion"), False), "fusion")
 
-    def test_tool_turn_never(self):
-        self.assertFalse(ce.should_escalate("", S(), has_tools=True))
+    def test_auto_short_confidence_long_fusion(self):
+        self.assertEqual(ce.select_recipe(body("hi"), S(recipe="auto"), False), "confidence")
+        self.assertEqual(ce.select_recipe(body("x" * 700), S(recipe="auto"), False), "fusion")
 
 
-class PayloadTest(unittest.TestCase):
-    def test_payload_uses_escalation_model_and_messages(self):
-        body = {"model": "qwen", "max_tokens": 100, "system": "sys", "messages": [{"role": "user", "content": "hi"}]}
-        p = ce.build_escalation_payload(body, S(model="opus"))
-        self.assertEqual(p["model"], "opus")
-        self.assertEqual(p["messages"], body["messages"])
-        self.assertEqual(p["system"], "sys")
-        self.assertNotIn("tools", p)
+class ApplyConfidenceTest(unittest.TestCase):  # #1 / #5
+    def test_low_confidence_escalates(self):
+        async def judge(p): return resp("ESCALATED ANSWER")
+        out = run(ce.apply_recipe(resp("I cannot help"), body(), {}, S(recipe="confidence"),
+                                  False, None, judge))
+        self.assertEqual(ce.extract_text(out), "ESCALATED ANSWER")
+
+    def test_high_confidence_keeps_primary(self):
+        async def judge(p): raise AssertionError("should not be called")
+        out = run(ce.apply_recipe(resp("A complete confident answer here."), body(), {},
+                                  S(recipe="confidence"), False, None, judge))
+        self.assertEqual(ce.extract_text(out), "A complete confident answer here.")
+
+    def test_selfverify_uses_judge_score(self):  # #5
+        calls = {"n": 0}
+        async def judge(p):
+            calls["n"] += 1
+            return resp("9")  # high score -> no escalation
+        out = run(ce.apply_recipe(resp("short"), body(), {}, S(recipe="confidence", signal="selfverify"),
+                                  False, None, judge))
+        self.assertEqual(calls["n"], 1)  # verify call happened
+        self.assertEqual(ce.extract_text(out), "short")  # high score, kept primary
+
+
+class ApplyFusionTest(unittest.TestCase):  # #3
+    def test_fusion_fans_out_and_judge_picks(self):
+        async def primary(v): return resp("candidate-1")
+        async def judge(p): return resp("1")  # pick index 1
+        out = run(ce.apply_recipe(resp("candidate-0"), body(), {"model": "qwen", "messages": []},
+                                  S(recipe="fusion", fusion_n=2), False, primary, judge))
+        self.assertEqual(ce.extract_text(out), "candidate-1")
+
+    def test_fusion_judge_failure_falls_back_to_primary(self):
+        async def primary(v): return resp("c1")
+        async def judge(p): return None  # judge failed
+        out = run(ce.apply_recipe(resp("c0"), body(), {"model": "qwen", "messages": []},
+                                  S(recipe="fusion", fusion_n=2), False, primary, judge))
+        self.assertEqual(ce.extract_text(out), "c0")
 
 
 if __name__ == "__main__":