@miller-tech/uap 1.179.4 → 1.179.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/guides/PROXY.md +2 -0
- package/package.json +2 -3
- package/src/policies/enforcers/__pycache__/_common.cpython-312.pyc +0 -0
- package/templates/hooks/__pycache__/deliver_autoroute.cpython-312.pyc +0 -0
- package/tools/agents/scripts/__pycache__/toolcall_path_normalizer.cpython-312.pyc +0 -0
- package/tools/agents/scripts/anthropic_proxy.py +94 -34
- package/tools/agents/tests/test_anthropic_proxy_streaming.py +395 -65
- package/tools/agents/tests/test_delivery_enforcement_all_langs.py +7 -1
- package/tools/agents/tests/test_delivery_enforcement_exemptions.py +7 -1
- package/tools/agents/tests/test_delivery_enforcement_filepath.py +7 -1
- package/tools/agents/tests/test_delivery_enforcement_web_and_bash.py +7 -1
- package/tools/agents/tests/test_delivery_enforcement_worktree.py +38 -6
- package/tools/agents/tests/test_enforcer_suite_coverage.py +58 -2
package/docs/guides/PROXY.md
CHANGED
|
@@ -179,6 +179,8 @@ set is below.
|
|
|
179
179
|
| Env (default) | Behavior |
|
|
180
180
|
|---|---|
|
|
181
181
|
| `PROXY_MAX_CONNECTIONS` (20) | httpx pool size; a large pool + 529 backoff absorbs connection churn gracefully |
|
|
182
|
+
| `PROXY_UPSTREAM_RETRY_MAX` (3) / `PROXY_UPSTREAM_RETRY_DELAY_SECS` (5) | Transient-failure retry budget for upstream calls, on both the buffered and the streaming path. Values below 1 are clamped to 1 |
|
|
183
|
+
| **503 "Loading model"** (no toggle) | llama-server answers 503 while a GGUF is still being mapped. Both paths treat it as transient: wait for `/health` (up to 60s), then retry. A wait that times out surfaces the 503 rather than stacking another — so a cold start costs at most ~60s of extra latency per attempt, not `RETRY_MAX × 60s` of dead air |
|
|
182
184
|
| **529 backpressure** (no toggle) | A pool timeout returns HTTP **529 `overloaded_error`** with `retry-after` — pure graceful degradation, not a hard failure |
|
|
183
185
|
| `PROXY_CLOSEWAIT_REAP_INTERVAL` (0 = **off**) | Opt-in CLOSE-WAIT reaper (pool self-heal); off by default because pool-swap churn can harm a saturated upstream |
|
|
184
186
|
| `PROXY_TOOL_NARROWING` (**off**) | Opt-in: drop cycling/banned tools from the set on loops — always keeps the Bash/WebFetch/Agent escape hatch + write tools (a floor invariant that never strands the agent) |
|
package/package.json
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@miller-tech/uap",
|
|
3
|
-
"version": "1.179.
|
|
4
|
-
"version": "1.179.4",
|
|
3
|
+
"version": "1.179.6",
|
|
5
4
|
"description": "Autonomous AI agent memory system with CLAUDE.md protocol enforcement",
|
|
6
5
|
"type": "module",
|
|
7
6
|
"main": "dist/index.js",
|
|
@@ -22,7 +21,7 @@
|
|
|
22
21
|
"start": "node dist/bin/cli.js",
|
|
23
22
|
"test": "vitest",
|
|
24
23
|
"test:ci": "vitest run",
|
|
25
|
-
"test:enforcers": "python3 -m unittest tools.agents.tests.test_enforcement_self_protect tools.agents.tests.test_schema_diff_gate tools.agents.tests.test_stream_telemetry tools.agents.tests.test_project_telemetry_events tools.agents.tests.test_workdir_scope_enforcer tools.agents.tests.test_path_containment tools.agents.tests.test_path_normalizer_hardened tools.agents.tests.test_sandbox_tool_strip tools.agents.tests.test_proxy_env_loader tools.agents.tests.test_stream_required_tool tools.agents.tests.test_tool_call_wrapper_profiles tools.agents.tests.test_tool_convert_cache tools.agents.tests.test_doubling_break tools.agents.tests.test_error_loop_ignores_correctives tools.agents.tests.test_attractor_detection tools.agents.tests.test_client_disconnect tools.agents.tests.test_confidence_escalation tools.agents.tests.test_coordination_ban tools.agents.tests.test_coordination_early_ban tools.agents.tests.test_cycle_break_exploration tools.agents.tests.test_deferral_break tools.agents.tests.test_deliver_autoroute tools.agents.tests.test_delivery_enforcement_all_langs tools.agents.tests.test_delivery_enforcement_exemptions tools.agents.tests.test_delivery_enforcement_filepath tools.agents.tests.test_delivery_enforcement_web_and_bash tools.agents.tests.test_disconnect_watcher tools.agents.tests.test_empty_maxtokens_recovery tools.agents.tests.test_empty_tool_loop_break tools.agents.tests.test_enforcer_escape_hatches tools.agents.tests.test_error_loop_break tools.agents.tests.test_finalize_suppression tools.agents.tests.test_malformed_unclosed_think tools.agents.tests.test_mandate_beats_recon tools.agents.tests.test_mandate_deliver tools.agents.tests.test_overflow_truncate_count_tokens tools.agents.tests.test_passthrough_oauth tools.agents.tests.test_project_telemetry tools.agents.tests.test_proxy_auth_headers tools.agents.tests.test_prune_preserve_force_write tools.agents.tests.test_recon_deliver_gate tools.agents.tests.test_session_admission tools.agents.tests.test_stream_heartbeat tools.agents.tests.test_stuck_break_reattach tools.agents.tests.test_turn_count_breaker_periodic tools.agents.tests.test_upstream_chokepoint tools.agents.tests.test_vision_passthrough tools.agents.tests.test_worktree_required tools.agents.tests.test_enforcer_suite_coverage tools.agents.tests.test_validate_plan_gate tools.agents.tests.test_validate_plan_inside_project",
|
|
24
|
+
"test:enforcers": "python3 -m unittest tools.agents.tests.test_enforcement_self_protect tools.agents.tests.test_schema_diff_gate tools.agents.tests.test_stream_telemetry tools.agents.tests.test_project_telemetry_events tools.agents.tests.test_workdir_scope_enforcer tools.agents.tests.test_path_containment tools.agents.tests.test_path_normalizer_hardened tools.agents.tests.test_sandbox_tool_strip tools.agents.tests.test_proxy_env_loader tools.agents.tests.test_stream_required_tool tools.agents.tests.test_tool_call_wrapper_profiles tools.agents.tests.test_tool_convert_cache tools.agents.tests.test_doubling_break tools.agents.tests.test_error_loop_ignores_correctives tools.agents.tests.test_attractor_detection tools.agents.tests.test_client_disconnect tools.agents.tests.test_confidence_escalation tools.agents.tests.test_coordination_ban tools.agents.tests.test_coordination_early_ban tools.agents.tests.test_cycle_break_exploration tools.agents.tests.test_deferral_break tools.agents.tests.test_deliver_autoroute tools.agents.tests.test_delivery_enforcement_all_langs tools.agents.tests.test_delivery_enforcement_exemptions tools.agents.tests.test_delivery_enforcement_filepath tools.agents.tests.test_delivery_enforcement_web_and_bash tools.agents.tests.test_disconnect_watcher tools.agents.tests.test_empty_maxtokens_recovery tools.agents.tests.test_empty_tool_loop_break tools.agents.tests.test_enforcer_escape_hatches tools.agents.tests.test_error_loop_break tools.agents.tests.test_finalize_suppression tools.agents.tests.test_malformed_unclosed_think tools.agents.tests.test_mandate_beats_recon tools.agents.tests.test_mandate_deliver tools.agents.tests.test_overflow_truncate_count_tokens tools.agents.tests.test_passthrough_oauth tools.agents.tests.test_project_telemetry tools.agents.tests.test_proxy_auth_headers tools.agents.tests.test_prune_preserve_force_write tools.agents.tests.test_recon_deliver_gate tools.agents.tests.test_session_admission tools.agents.tests.test_stream_heartbeat tools.agents.tests.test_stuck_break_reattach tools.agents.tests.test_turn_count_breaker_periodic tools.agents.tests.test_upstream_chokepoint tools.agents.tests.test_vision_passthrough tools.agents.tests.test_worktree_required tools.agents.tests.test_enforcer_suite_coverage tools.agents.tests.test_validate_plan_gate tools.agents.tests.test_validate_plan_inside_project tools.agents.tests.test_anthropic_proxy_streaming tools.agents.tests.test_delivery_enforcement_worktree",
|
|
26
25
|
"test:coverage": "vitest --coverage",
|
|
27
26
|
"bench": "vitest --config vitest.bench.config.ts",
|
|
28
27
|
"lint": "eslint src --ext .ts",
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -3560,6 +3560,92 @@ class DisconnectAwareClient(httpx.AsyncClient):
|
|
|
3560
3560
|
task.cancel()
|
|
3561
3561
|
|
|
3562
3562
|
|
|
3563
|
+
async def _send_stream_with_retry(
|
|
3564
|
+
client: httpx.AsyncClient,
|
|
3565
|
+
url: str,
|
|
3566
|
+
payload: dict,
|
|
3567
|
+
) -> httpx.Response:
|
|
3568
|
+
"""POST a streaming completion, retrying transient failures and 503 "Loading model".
|
|
3569
|
+
|
|
3570
|
+
The streaming sibling of `_post_with_retry_inner`. Both retry the
|
|
3571
|
+
`_UPSTREAM_RETRY_EXCEPTIONS` set — which is wider than connect errors; a
|
|
3572
|
+
ReadTimeout/ReadError mid-setup is retried too — so a llama-server restart
|
|
3573
|
+
doesn't fail the turn. Both also treat a 503 "Loading model" as transient:
|
|
3574
|
+
llama-server answers that while a GGUF is still being mapped, so the right
|
|
3575
|
+
move is to wait for /health and retry rather than surface a fatal error.
|
|
3576
|
+
|
|
3577
|
+
Raises the last transient exception when every attempt failed. Otherwise
|
|
3578
|
+
returns the response, which may itself be a non-200 the caller must handle —
|
|
3579
|
+
including a loading 503 that outlived the retry budget, or one whose health
|
|
3580
|
+
wait timed out.
|
|
3581
|
+
"""
|
|
3582
|
+
# Clamp: a configured 0 would skip the loop entirely and fall through to the
|
|
3583
|
+
# terminal raise with last_exc unset, escaping the caller's narrow except as
|
|
3584
|
+
# an unhandled 500. One attempt is the floor.
|
|
3585
|
+
retry_max = max(1, PROXY_UPSTREAM_RETRY_MAX)
|
|
3586
|
+
last_exc: Exception | None = None
|
|
3587
|
+
for attempt in range(retry_max):
|
|
3588
|
+
try:
|
|
3589
|
+
resp = await client.send(
|
|
3590
|
+
client.build_request(
|
|
3591
|
+
"POST",
|
|
3592
|
+
url,
|
|
3593
|
+
json=payload,
|
|
3594
|
+
headers={"Content-Type": "application/json"},
|
|
3595
|
+
),
|
|
3596
|
+
stream=True,
|
|
3597
|
+
)
|
|
3598
|
+
setattr(resp, "_uap_client", client)
|
|
3599
|
+
if resp.status_code == 503:
|
|
3600
|
+
# _is_loading_model_503 reads .text, which raises ResponseNotRead
|
|
3601
|
+
# on an unread streaming body — and its bare `except` turns that
|
|
3602
|
+
# into a silent False, so without this read every loading 503
|
|
3603
|
+
# would look permanent and the retry would never fire. httpx
|
|
3604
|
+
# caches the body, so the caller's non-200 handler re-reads free.
|
|
3605
|
+
try:
|
|
3606
|
+
await resp.aread()
|
|
3607
|
+
except BaseException:
|
|
3608
|
+
# Never leave the connection checked out: a ReadError here is
|
|
3609
|
+
# retryable and would otherwise loop with resp unreferenced
|
|
3610
|
+
# and unclosed, accruing CLOSE-WAIT until the pool saturates.
|
|
3611
|
+
_detach_aclose(resp)
|
|
3612
|
+
raise
|
|
3613
|
+
if _is_loading_model_503(resp) and attempt < retry_max - 1:
|
|
3614
|
+
logger.warning(
|
|
3615
|
+
"Upstream 503 Loading model (stream attempt %d/%d) – waiting for health",
|
|
3616
|
+
attempt + 1,
|
|
3617
|
+
retry_max,
|
|
3618
|
+
)
|
|
3619
|
+
healthy = await _wait_for_upstream_health(client, max_wait=60.0)
|
|
3620
|
+
if not healthy:
|
|
3621
|
+
# Matches _post_with_retry_inner: once a health wait has
|
|
3622
|
+
# timed out, stacking another one just buys the client
|
|
3623
|
+
# minutes of dead air before the same failure. Surface it.
|
|
3624
|
+
return resp
|
|
3625
|
+
_detach_aclose(resp)
|
|
3626
|
+
continue
|
|
3627
|
+
return resp
|
|
3628
|
+
except _UPSTREAM_RETRY_EXCEPTIONS as exc:
|
|
3629
|
+
last_exc = exc
|
|
3630
|
+
if attempt < retry_max - 1:
|
|
3631
|
+
logger.warning(
|
|
3632
|
+
"Upstream transient error (stream attempt %d/%d): %s – retrying in %.0fs",
|
|
3633
|
+
attempt + 1,
|
|
3634
|
+
retry_max,
|
|
3635
|
+
type(exc).__name__,
|
|
3636
|
+
PROXY_UPSTREAM_RETRY_DELAY_SECS,
|
|
3637
|
+
)
|
|
3638
|
+
await asyncio.sleep(PROXY_UPSTREAM_RETRY_DELAY_SECS)
|
|
3639
|
+
else:
|
|
3640
|
+
logger.error(
|
|
3641
|
+
"Upstream stream failed after %d attempts: %s: %s",
|
|
3642
|
+
retry_max,
|
|
3643
|
+
type(exc).__name__,
|
|
3644
|
+
exc,
|
|
3645
|
+
)
|
|
3646
|
+
raise last_exc if last_exc else RuntimeError("upstream stream retry failed")
|
|
3647
|
+
|
|
3648
|
+
|
|
3563
3649
|
async def _post_with_retry_inner(
|
|
3564
3650
|
client: httpx.AsyncClient,
|
|
3565
3651
|
url: str,
|
|
@@ -11676,43 +11762,17 @@ async def messages(request: Request):
|
|
|
11676
11762
|
# Retry upstream connection with backoff to handle
|
|
11677
11763
|
# llama-server restarts gracefully instead of 500-ing to the client.
|
|
11678
11764
|
MAX_UPSTREAM_RETRIES = PROXY_UPSTREAM_RETRY_MAX
|
|
11679
|
-
RETRY_DELAY_SECS = PROXY_UPSTREAM_RETRY_DELAY_SECS
|
|
11680
11765
|
last_exc: Exception | None = None
|
|
11681
11766
|
resp: httpx.Response | None = None
|
|
11682
11767
|
|
|
11683
|
-
|
|
11684
|
-
|
|
11685
|
-
|
|
11686
|
-
|
|
11687
|
-
|
|
11688
|
-
|
|
11689
|
-
|
|
11690
|
-
|
|
11691
|
-
),
|
|
11692
|
-
stream=True,
|
|
11693
|
-
)
|
|
11694
|
-
setattr(resp, "_uap_client", client)
|
|
11695
|
-
# Connection succeeded – break out of retry loop
|
|
11696
|
-
last_exc = None
|
|
11697
|
-
break
|
|
11698
|
-
except _UPSTREAM_RETRY_EXCEPTIONS as exc:
|
|
11699
|
-
last_exc = exc
|
|
11700
|
-
if attempt < MAX_UPSTREAM_RETRIES - 1:
|
|
11701
|
-
logger.warning(
|
|
11702
|
-
"Upstream connect failed (attempt %d/%d): %s – retrying in %.0fs",
|
|
11703
|
-
attempt + 1,
|
|
11704
|
-
MAX_UPSTREAM_RETRIES,
|
|
11705
|
-
type(exc).__name__,
|
|
11706
|
-
RETRY_DELAY_SECS,
|
|
11707
|
-
)
|
|
11708
|
-
await asyncio.sleep(RETRY_DELAY_SECS)
|
|
11709
|
-
else:
|
|
11710
|
-
logger.error(
|
|
11711
|
-
"Upstream connect failed after %d attempts: %s: %s",
|
|
11712
|
-
MAX_UPSTREAM_RETRIES,
|
|
11713
|
-
type(exc).__name__,
|
|
11714
|
-
exc,
|
|
11715
|
-
)
|
|
11768
|
+
try:
|
|
11769
|
+
resp = await _send_stream_with_retry(
|
|
11770
|
+
client,
|
|
11771
|
+
f"{LLAMA_CPP_BASE}/chat/completions",
|
|
11772
|
+
openai_body,
|
|
11773
|
+
)
|
|
11774
|
+
except _UPSTREAM_RETRY_EXCEPTIONS as exc:
|
|
11775
|
+
last_exc = exc
|
|
11716
11776
|
|
|
11717
11777
|
if last_exc is not None:
|
|
11718
11778
|
return Response(
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import importlib.util
|
|
5
5
|
import json
|
|
6
|
+
import os
|
|
6
7
|
import unittest
|
|
7
8
|
import unittest.mock
|
|
8
9
|
from pathlib import Path
|
|
@@ -91,36 +92,44 @@ class TestProxyConfigTuning(unittest.TestCase):
|
|
|
91
92
|
finally:
|
|
92
93
|
setattr(proxy, "PROXY_MAX_TOKENS_FLOOR", old_floor)
|
|
93
94
|
|
|
94
|
-
def
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
95
|
+
def test_thinking_disabled_flag_no_longer_changes_the_tool_turn_budget(self):
|
|
96
|
+
# This test used to assert 512 while toggling
|
|
97
|
+
# PROXY_DISABLE_THINKING_ON_TOOL_TURNS. That gating was deliberately
|
|
98
|
+
# removed — it skipped the floor on every tool turn once thinking was
|
|
99
|
+
# off, which re-introduced truncated tool calls on long edits — and the
|
|
100
|
+
# flag now only feeds a log line. Asserting the post-removal number
|
|
101
|
+
# alone would just duplicate test_max_tokens_floor_bypassed_for_small_
|
|
102
|
+
# preflight, so pin the removal itself: the flag must not move the
|
|
103
|
+
# budget in either position. 512 lands on THINKING_MIN_FOR_TOOLS (2048)
|
|
104
|
+
# because Qwen emits <think> regardless of the flag, and on a tool turn
|
|
105
|
+
# those blocks alone eat ~400-1000 tokens, leaving nothing for the
|
|
106
|
+
# tool_call (observed live as ~5 required_tool_miss retries per turn).
|
|
107
|
+
def budget(disable_thinking):
|
|
108
|
+
with unittest.mock.patch.object(proxy, "PROXY_MAX_TOKENS_FLOOR", 4096), \
|
|
109
|
+
unittest.mock.patch.object(
|
|
110
|
+
proxy, "PROXY_DISABLE_THINKING_ON_TOOL_TURNS", disable_thinking):
|
|
111
|
+
return proxy.build_openai_request(
|
|
106
112
|
{
|
|
107
|
-
"
|
|
108
|
-
"
|
|
109
|
-
"
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
+
"model": "test",
|
|
114
|
+
"max_tokens": 512,
|
|
115
|
+
"messages": [{"role": "user", "content": "run pwd"}],
|
|
116
|
+
"tools": [
|
|
117
|
+
{
|
|
118
|
+
"name": "Bash",
|
|
119
|
+
"description": "run command",
|
|
120
|
+
"input_schema": {"type": "object"},
|
|
121
|
+
}
|
|
122
|
+
],
|
|
123
|
+
},
|
|
124
|
+
proxy.SessionMonitor(context_window=0),
|
|
125
|
+
).get("max_tokens")
|
|
113
126
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
)
|
|
117
|
-
self.assertEqual(openai.get("max_tokens"), 512)
|
|
118
|
-
finally:
|
|
119
|
-
setattr(proxy, "PROXY_MAX_TOKENS_FLOOR", old_floor)
|
|
120
|
-
setattr(proxy, "PROXY_DISABLE_THINKING_ON_TOOL_TURNS", old_disable)
|
|
127
|
+
self.assertEqual(budget(True), budget(False), "flag must no longer gate the floor")
|
|
128
|
+
self.assertEqual(budget(True), 2048)
|
|
121
129
|
|
|
122
|
-
def
|
|
123
|
-
"""Non-tool requests
|
|
130
|
+
def test_non_tool_turn_takes_thinking_floor_not_the_big_floor(self):
|
|
131
|
+
"""Non-tool requests skip PROXY_MAX_TOKENS_FLOOR and land on
|
|
132
|
+
THINKING_MIN_NO_TOOLS instead — a different, larger mechanism."""
|
|
124
133
|
old_floor = getattr(proxy, "PROXY_MAX_TOKENS_FLOOR")
|
|
125
134
|
old_disable = getattr(proxy, "PROXY_DISABLE_THINKING_ON_TOOL_TURNS")
|
|
126
135
|
try:
|
|
@@ -133,11 +142,20 @@ class TestProxyConfigTuning(unittest.TestCase):
|
|
|
133
142
|
"messages": [{"role": "user", "content": "say ok"}],
|
|
134
143
|
}
|
|
135
144
|
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
)
|
|
139
|
-
|
|
140
|
-
|
|
145
|
+
with unittest.mock.patch.dict(
|
|
146
|
+
os.environ, {"PROXY_THINKING_MIN_NO_TOOLS": "8192"}
|
|
147
|
+
):
|
|
148
|
+
openai = proxy.build_openai_request(
|
|
149
|
+
body, proxy.SessionMonitor(context_window=0)
|
|
150
|
+
)
|
|
151
|
+
# The big PROXY_MAX_TOKENS_FLOOR (4096) is still skipped for
|
|
152
|
+
# non-tool turns — that is what this test guards. What the request
|
|
153
|
+
# lands on instead is THINKING_MIN_NO_TOOLS: Qwen spends small
|
|
154
|
+
# no-tool budgets entirely inside <think>, the EMPTY-OUTPUT GUARD
|
|
155
|
+
# then promotes truncated reasoning as the body, and evaluator
|
|
156
|
+
# callers get an unparseable verdict. Distinct mechanism, distinct
|
|
157
|
+
# value — assert it explicitly rather than the raw 512.
|
|
158
|
+
self.assertEqual(openai.get("max_tokens"), 8192)
|
|
141
159
|
finally:
|
|
142
160
|
setattr(proxy, "PROXY_MAX_TOKENS_FLOOR", old_floor)
|
|
143
161
|
setattr(proxy, "PROXY_DISABLE_THINKING_ON_TOOL_TURNS", old_disable)
|
|
@@ -170,30 +188,56 @@ class TestProfileSelection(unittest.TestCase):
|
|
|
170
188
|
)
|
|
171
189
|
self.assertIn(suffix, openai_body["messages"][0]["content"])
|
|
172
190
|
|
|
191
|
+
@staticmethod
|
|
192
|
+
def _grammar_body():
|
|
193
|
+
return {
|
|
194
|
+
"model": "default",
|
|
195
|
+
"max_tokens": 128,
|
|
196
|
+
"messages": [{"role": "user", "content": "run pwd"}],
|
|
197
|
+
"tools": [
|
|
198
|
+
{
|
|
199
|
+
"name": "Bash",
|
|
200
|
+
"description": "run command",
|
|
201
|
+
"input_schema": {"type": "object"},
|
|
202
|
+
}
|
|
203
|
+
],
|
|
204
|
+
}
|
|
205
|
+
|
|
173
206
|
def test_build_request_uses_profile_grammar_override(self):
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
"name": "Bash",
|
|
184
|
-
"description": "run command",
|
|
185
|
-
"input_schema": {"type": "object"},
|
|
186
|
-
}
|
|
187
|
-
],
|
|
188
|
-
}
|
|
207
|
+
# PROXY_TOOL_CALL_GRAMMAR_REQUIRED_ONLY defaults to True and gates the
|
|
208
|
+
# whole grammar path on tool_choice == "required". It postdates this
|
|
209
|
+
# test, which is why the override looked broken: _apply_tool_call_grammar
|
|
210
|
+
# returned before ever consulting it. Relax the gate so this test covers
|
|
211
|
+
# what its name says — the override winning over the default GBNF — and
|
|
212
|
+
# let the test below cover the gate itself.
|
|
213
|
+
with unittest.mock.patch.object(proxy, "PROXY_TOOL_CALL_GRAMMAR", True), \
|
|
214
|
+
unittest.mock.patch.object(
|
|
215
|
+
proxy, "PROXY_TOOL_CALL_GRAMMAR_REQUIRED_ONLY", False):
|
|
189
216
|
openai_body = proxy.build_openai_request(
|
|
190
|
-
|
|
217
|
+
self._grammar_body(),
|
|
191
218
|
proxy.SessionMonitor(context_window=0),
|
|
192
219
|
profile_grammar="grammar-test",
|
|
193
220
|
)
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
221
|
+
self.assertEqual(openai_body.get("grammar"), "grammar-test")
|
|
222
|
+
|
|
223
|
+
def test_required_only_gate_is_what_decides_whether_grammar_attaches(self):
|
|
224
|
+
# The gate that broke the test above is real behaviour and was untested.
|
|
225
|
+
# Assert it on the DISCRIMINATING axis — tool_choice — rather than only
|
|
226
|
+
# the negative case: a bare assertIsNone would also pass if the grammar
|
|
227
|
+
# flag were off, if tools were stripped, or if the _apply_tool_call_grammar
|
|
228
|
+
# call site were deleted outright, so it would prove nothing.
|
|
229
|
+
def grammar_for(tool_choice):
|
|
230
|
+
body = {"tools": [{"type": "function"}], "tool_choice": tool_choice}
|
|
231
|
+
with unittest.mock.patch.object(proxy, "PROXY_TOOL_CALL_GRAMMAR", True), \
|
|
232
|
+
unittest.mock.patch.object(
|
|
233
|
+
proxy, "PROXY_TOOL_CALL_GRAMMAR_REQUIRED_ONLY", True):
|
|
234
|
+
proxy._apply_tool_call_grammar(
|
|
235
|
+
body, tool_choice=tool_choice, grammar_override="grammar-test"
|
|
236
|
+
)
|
|
237
|
+
return body.get("grammar")
|
|
238
|
+
|
|
239
|
+
self.assertEqual(grammar_for("required"), "grammar-test")
|
|
240
|
+
self.assertIsNone(grammar_for("auto"))
|
|
197
241
|
|
|
198
242
|
def test_prune_target_fraction_uses_config_or_default(self):
|
|
199
243
|
old_target = getattr(proxy, "PROXY_CONTEXT_PRUNE_TARGET_FRACTION")
|
|
@@ -408,7 +452,18 @@ class TestStreamGuardedPathSelection(unittest.TestCase):
|
|
|
408
452
|
|
|
409
453
|
|
|
410
454
|
class TestMalformedToolGuardrail(unittest.TestCase):
|
|
411
|
-
def
|
|
455
|
+
def test_tolerates_orphan_parameter_closer_after_a_valid_answer(self):
|
|
456
|
+
# Reversed deliberately on 2026-05-12 (_strip_orphan_tool_xml): an orphan
|
|
457
|
+
# closer with no opener is no longer treated as a malformed tool call,
|
|
458
|
+
# because Qwen3.6 leaks bare </parameter> training residue after a valid
|
|
459
|
+
# answer when forced into tool_choice='required' with nothing to call, and
|
|
460
|
+
# rejecting those cost ~11 false rejections in 40 min on a related branch.
|
|
461
|
+
#
|
|
462
|
+
# Be honest about the trade-off this pins: THIS payload is the model
|
|
463
|
+
# regurgitating its own tool schema twice, which is a plausible genuine
|
|
464
|
+
# failure. It survives only because the separator is '= {' rather than
|
|
465
|
+
# '=\n{' — one whitespace character flips the verdict. We accept that
|
|
466
|
+
# false negative to kill a much larger false-positive class.
|
|
412
467
|
openai_resp = {
|
|
413
468
|
"choices": [
|
|
414
469
|
{
|
|
@@ -427,9 +482,43 @@ class TestMalformedToolGuardrail(unittest.TestCase):
|
|
|
427
482
|
"tools": [{"name": "Read", "input_schema": {"type": "object"}}],
|
|
428
483
|
"messages": [{"role": "user", "content": "fix this"}],
|
|
429
484
|
}
|
|
430
|
-
self.
|
|
485
|
+
self.assertFalse(proxy._is_malformed_tool_response(openai_resp, anthropic_body))
|
|
431
486
|
|
|
432
|
-
def
|
|
487
|
+
def test_still_detects_a_payload_that_retains_its_opener(self):
|
|
488
|
+
# The other half of the contract the strip promises: tolerating orphan
|
|
489
|
+
# closers must not blind the guardrail to a genuine malformed attempt,
|
|
490
|
+
# which keeps its opener. Without this, the test above alone would be
|
|
491
|
+
# satisfied by deleting the detector outright.
|
|
492
|
+
anthropic_body = {
|
|
493
|
+
"tools": [{"name": "Read", "input_schema": {"type": "object"}}],
|
|
494
|
+
"messages": [{"role": "user", "content": "fix this"}],
|
|
495
|
+
}
|
|
496
|
+
for payload in (
|
|
497
|
+
# The load-bearing case: an orphan closer AND a real opener in the
|
|
498
|
+
# same text. Only this one proves the strip removes the residue
|
|
499
|
+
# without also swallowing the genuine attempt beside it — the other
|
|
500
|
+
# payloads return early before _strip_orphan_tool_xml ever runs.
|
|
501
|
+
'</function> then <function=Bash><parameter name="c">ls</parameter></function>',
|
|
502
|
+
'<parameter name="cmd">ls</parameter>',
|
|
503
|
+
'<tool_call>{"name":"Read"}',
|
|
504
|
+
'<function=Bash>{"cmd":"ls"}',
|
|
505
|
+
):
|
|
506
|
+
with self.subTest(payload=payload):
|
|
507
|
+
openai_resp = {
|
|
508
|
+
"choices": [
|
|
509
|
+
{
|
|
510
|
+
"finish_reason": "stop",
|
|
511
|
+
"message": {"content": payload, "tool_calls": []},
|
|
512
|
+
}
|
|
513
|
+
]
|
|
514
|
+
}
|
|
515
|
+
self.assertTrue(
|
|
516
|
+
proxy._is_malformed_tool_response(openai_resp, anthropic_body)
|
|
517
|
+
)
|
|
518
|
+
|
|
519
|
+
def test_tolerates_orphan_function_closer_after_a_valid_answer(self):
|
|
520
|
+
# Same 2026-05-12 reversal as the </parameter> case above: a trailing
|
|
521
|
+
# </function> with no opener is training residue, not a tool call.
|
|
433
522
|
openai_resp = {
|
|
434
523
|
"choices": [
|
|
435
524
|
{
|
|
@@ -449,7 +538,7 @@ class TestMalformedToolGuardrail(unittest.TestCase):
|
|
|
449
538
|
"tools": [{"name": "Bash", "input_schema": {"type": "object"}}],
|
|
450
539
|
"messages": [{"role": "user", "content": "list root docs/json files"}],
|
|
451
540
|
}
|
|
452
|
-
self.
|
|
541
|
+
self.assertFalse(proxy._is_malformed_tool_response(openai_resp, anthropic_body))
|
|
453
542
|
|
|
454
543
|
def test_detects_think_tag_with_repeated_policy_phrase(self):
|
|
455
544
|
openai_resp = {
|
|
@@ -1840,7 +1929,14 @@ class TestTurnCountFinalizeBreaker(unittest.TestCase):
|
|
|
1840
1929
|
self._body(45), proxy.SessionMonitor(context_window=262144)
|
|
1841
1930
|
)
|
|
1842
1931
|
self.assertFalse(out.get("tools")) # tools stripped
|
|
1843
|
-
|
|
1932
|
+
# The breaker's wording moved from a hard "STOP now" to a periodic
|
|
1933
|
+
# progress checkpoint that explicitly resumes next turn. Assert the
|
|
1934
|
+
# contract that survives rewording rather than the prose: the turn
|
|
1935
|
+
# count is cited, and the model is told not to emit a tool call —
|
|
1936
|
+
# which is the half that must agree with the stripped toolset.
|
|
1937
|
+
nudge = out["messages"][-1]["content"]
|
|
1938
|
+
self.assertIn("45", nudge)
|
|
1939
|
+
self.assertRegex(nudge, r"(?i)do NOT emit any tool call|no tools are available")
|
|
1844
1940
|
finally:
|
|
1845
1941
|
setattr(proxy, "PROXY_HARD_FINALIZE_TURNS", old)
|
|
1846
1942
|
|
|
@@ -3643,8 +3739,9 @@ class TestDegenerateRepetitionDetection(unittest.TestCase):
|
|
|
3643
3739
|
self.assertFalse(truncated)
|
|
3644
3740
|
self.assertEqual(result["choices"][0]["message"]["content"], text)
|
|
3645
3741
|
|
|
3646
|
-
def
|
|
3647
|
-
"""
|
|
3742
|
+
def test_non_tool_request_lands_below_the_big_floor(self):
|
|
3743
|
+
"""The 16384 floor is skipped for non-tool turns; the much smaller
|
|
3744
|
+
THINKING_MIN_NO_TOOLS applies instead."""
|
|
3648
3745
|
old_floor = getattr(proxy, "PROXY_MAX_TOKENS_FLOOR")
|
|
3649
3746
|
old_disable = getattr(proxy, "PROXY_DISABLE_THINKING_ON_TOOL_TURNS")
|
|
3650
3747
|
try:
|
|
@@ -3656,11 +3753,17 @@ class TestDegenerateRepetitionDetection(unittest.TestCase):
|
|
|
3656
3753
|
"max_tokens": 100,
|
|
3657
3754
|
"messages": [{"role": "user", "content": "generate a title"}],
|
|
3658
3755
|
}
|
|
3659
|
-
|
|
3660
|
-
|
|
3661
|
-
)
|
|
3662
|
-
|
|
3663
|
-
|
|
3756
|
+
with unittest.mock.patch.dict(
|
|
3757
|
+
os.environ, {"PROXY_THINKING_MIN_NO_TOOLS": "8192"}
|
|
3758
|
+
):
|
|
3759
|
+
openai = proxy.build_openai_request(
|
|
3760
|
+
body, proxy.SessionMonitor(context_window=0)
|
|
3761
|
+
)
|
|
3762
|
+
# No tools = the 16384 floor is skipped. The result is the much
|
|
3763
|
+
# smaller THINKING_MIN_NO_TOOLS, not the raw 100: a 100-token
|
|
3764
|
+
# no-tool budget is consumed entirely by Qwen's mandatory <think>,
|
|
3765
|
+
# leaving nothing for the answer. Still well under the floor.
|
|
3766
|
+
self.assertEqual(openai.get("max_tokens"), 8192)
|
|
3664
3767
|
finally:
|
|
3665
3768
|
setattr(proxy, "PROXY_MAX_TOKENS_FLOOR", old_floor)
|
|
3666
3769
|
setattr(proxy, "PROXY_DISABLE_THINKING_ON_TOOL_TURNS", old_disable)
|
|
@@ -3826,8 +3929,10 @@ class EmptyOutputGuardTest(unittest.TestCase):
|
|
|
3826
3929
|
self.assertFalse(any(b.get("type") == "text" for b in out["content"]))
|
|
3827
3930
|
|
|
3828
3931
|
|
|
3829
|
-
|
|
3830
|
-
|
|
3932
|
+
# NB: the `unittest.main()` entrypoint is at the END of this file. It used to
|
|
3933
|
+
# sit here, above ~2200 further lines of TestCase classes, so running the file
|
|
3934
|
+
# directly exited after this point and silently skipped roughly 40% of the
|
|
3935
|
+
# module — including TestSendStreamWithRetry — while still printing OK.
|
|
3831
3936
|
|
|
3832
3937
|
|
|
3833
3938
|
class TestCompletionContractGuardrails(unittest.TestCase):
|
|
@@ -5834,3 +5939,228 @@ class TestPassthroughTimeout(unittest.IsolatedAsyncioTestCase):
|
|
|
5834
5939
|
|
|
5835
5940
|
self.assertIsNotNone(captured.get("timeout"))
|
|
5836
5941
|
self.assertEqual(captured["timeout"].read, proxy.PROXY_PASSTHROUGH_TIMEOUT)
|
|
5942
|
+
|
|
5943
|
+
|
|
5944
|
+
class _FakeStreamResponse:
|
|
5945
|
+
"""Stand-in for a streamed httpx.Response, faithful to two contracts.
|
|
5946
|
+
|
|
5947
|
+
Both are load-bearing for _send_stream_with_retry, and a fake that models
|
|
5948
|
+
neither lets the fix be deleted with the suite still green:
|
|
5949
|
+
|
|
5950
|
+
1. `.text` raises ResponseNotRead until the body has been read. That is why
|
|
5951
|
+
the production code calls aread() before _is_loading_model_503 — whose
|
|
5952
|
+
bare `except` would otherwise swallow the raise into a silent False and
|
|
5953
|
+
silently disable the retry.
|
|
5954
|
+
2. aread() caches. The caller re-reads the same response in its non-200
|
|
5955
|
+
handler, which only works because httpx serves the second read from
|
|
5956
|
+
cache rather than raising StreamConsumed.
|
|
5957
|
+
"""
|
|
5958
|
+
|
|
5959
|
+
def __init__(self, status_code, text=""):
|
|
5960
|
+
self.status_code = status_code
|
|
5961
|
+
self._body = text
|
|
5962
|
+
self._read = False
|
|
5963
|
+
self.read_count = 0
|
|
5964
|
+
self.closed = False
|
|
5965
|
+
|
|
5966
|
+
@property
|
|
5967
|
+
def text(self):
|
|
5968
|
+
if not self._read:
|
|
5969
|
+
raise httpx.ResponseNotRead()
|
|
5970
|
+
return self._body
|
|
5971
|
+
|
|
5972
|
+
async def aread(self):
|
|
5973
|
+
if self._read:
|
|
5974
|
+
return self._body.encode() # cached: no second stream consumption
|
|
5975
|
+
self._read = True
|
|
5976
|
+
self.read_count += 1
|
|
5977
|
+
return self._body.encode()
|
|
5978
|
+
|
|
5979
|
+
async def aclose(self):
|
|
5980
|
+
self.closed = True
|
|
5981
|
+
|
|
5982
|
+
|
|
5983
|
+
class _FakeStreamClient:
|
|
5984
|
+
"""Replays a queued script of responses/exceptions for client.send()."""
|
|
5985
|
+
|
|
5986
|
+
def __init__(self, scripted):
|
|
5987
|
+
self._scripted = list(scripted)
|
|
5988
|
+
self.sends = 0
|
|
5989
|
+
self.requests = []
|
|
5990
|
+
self.stream_flags = []
|
|
5991
|
+
|
|
5992
|
+
def build_request(self, method, url, **kwargs):
|
|
5993
|
+
return {"method": method, "url": url, **kwargs}
|
|
5994
|
+
|
|
5995
|
+
async def send(self, request, stream=False):
|
|
5996
|
+
self.sends += 1
|
|
5997
|
+
self.requests.append(request)
|
|
5998
|
+
self.stream_flags.append(stream)
|
|
5999
|
+
if not self._scripted:
|
|
6000
|
+
raise AssertionError("no scripted response left")
|
|
6001
|
+
nxt = self._scripted.pop(0)
|
|
6002
|
+
if isinstance(nxt, Exception):
|
|
6003
|
+
raise nxt
|
|
6004
|
+
return nxt
|
|
6005
|
+
|
|
6006
|
+
|
|
6007
|
+
class TestSendStreamWithRetry(unittest.TestCase):
|
|
6008
|
+
"""The streaming path must treat 503 'Loading model' as transient.
|
|
6009
|
+
|
|
6010
|
+
Regression guard: this path used to break out of its retry loop on any HTTP
|
|
6011
|
+
response, so a streamed turn that began while llama-server was still mapping
|
|
6012
|
+
a GGUF surfaced 'AI_APICallError: Loading model' to the client as a fatal
|
|
6013
|
+
error. The non-streaming sibling (_post_with_retry_inner) retried it —
|
|
6014
|
+
health-gated and bounded by PROXY_UPSTREAM_RETRY_MAX — while the streaming
|
|
6015
|
+
path, the one every opencode turn actually uses, did not retry it at all.
|
|
6016
|
+
"""
|
|
6017
|
+
|
|
6018
|
+
def _run(self, client, health_ok=True, retry_max=3):
|
|
6019
|
+
waited = []
|
|
6020
|
+
|
|
6021
|
+
async def _fake_health(_client, max_wait=60.0, poll_interval=5.0):
|
|
6022
|
+
waited.append(max_wait)
|
|
6023
|
+
return health_ok
|
|
6024
|
+
|
|
6025
|
+
async def _no_sleep(_secs):
|
|
6026
|
+
return None
|
|
6027
|
+
|
|
6028
|
+
def _sync_close(closeable):
|
|
6029
|
+
# Production schedules the close on a detached task; inside a test
|
|
6030
|
+
# the loop closes before it runs. Close eagerly so `.closed` is a
|
|
6031
|
+
# deterministic assertion rather than a race.
|
|
6032
|
+
closeable.closed = True
|
|
6033
|
+
|
|
6034
|
+
with unittest.mock.patch.object(proxy, "_wait_for_upstream_health", _fake_health), \
|
|
6035
|
+
unittest.mock.patch.object(proxy, "_detach_aclose", _sync_close), \
|
|
6036
|
+
unittest.mock.patch.object(proxy, "PROXY_UPSTREAM_RETRY_MAX", retry_max), \
|
|
6037
|
+
unittest.mock.patch.object(asyncio, "sleep", _no_sleep):
|
|
6038
|
+
resp = asyncio.run(
|
|
6039
|
+
proxy._send_stream_with_retry(client, "http://x/v1/chat/completions", {})
|
|
6040
|
+
)
|
|
6041
|
+
return resp, waited
|
|
6042
|
+
|
|
6043
|
+
def test_retries_loading_model_503_then_returns_success(self):
|
|
6044
|
+
loading = _FakeStreamResponse(503, "Loading model")
|
|
6045
|
+
ok = _FakeStreamResponse(200, "")
|
|
6046
|
+
client = _FakeStreamClient([loading, ok])
|
|
6047
|
+
|
|
6048
|
+
resp, waited = self._run(client)
|
|
6049
|
+
|
|
6050
|
+
self.assertIs(resp, ok)
|
|
6051
|
+
self.assertEqual(client.sends, 2, "should have retried after the 503")
|
|
6052
|
+
self.assertEqual(len(waited), 1, "should have waited for upstream health")
|
|
6053
|
+
self.assertTrue(loading.closed, "the discarded 503 must be closed")
|
|
6054
|
+
|
|
6055
|
+
def test_does_not_retry_a_non_loading_503(self):
|
|
6056
|
+
# A 503 that is not a model load is a real upstream failure; passing it
|
|
6057
|
+
# straight through preserves the existing error contract.
|
|
6058
|
+
overloaded = _FakeStreamResponse(503, "server is overloaded")
|
|
6059
|
+
client = _FakeStreamClient([overloaded, _FakeStreamResponse(200, "")])
|
|
6060
|
+
|
|
6061
|
+
resp, waited = self._run(client)
|
|
6062
|
+
|
|
6063
|
+
self.assertIs(resp, overloaded)
|
|
6064
|
+
self.assertEqual(client.sends, 1)
|
|
6065
|
+
self.assertEqual(waited, [])
|
|
6066
|
+
|
|
6067
|
+
def test_returns_503_after_exhausting_retry_budget(self):
|
|
6068
|
+
responses = [_FakeStreamResponse(503, "Loading model") for _ in range(3)]
|
|
6069
|
+
client = _FakeStreamClient(list(responses))
|
|
6070
|
+
|
|
6071
|
+
resp, waited = self._run(client, retry_max=3)
|
|
6072
|
+
|
|
6073
|
+
self.assertEqual(client.sends, 3)
|
|
6074
|
+
self.assertEqual(resp.status_code, 503)
|
|
6075
|
+
self.assertIs(resp, responses[-1], "last 503 is surfaced to the caller")
|
|
6076
|
+
self.assertEqual(len(waited), 2, "waits between attempts, not after the last")
|
|
6077
|
+
|
|
6078
|
+
def test_surfaces_the_503_when_the_health_wait_times_out(self):
|
|
6079
|
+
# Mirrors _post_with_retry_inner: once upstream is still unhealthy after
|
|
6080
|
+
# a full wait, stacking more waits only buys the client dead air. Without
|
|
6081
|
+
# this gate a dead upstream costs retry_max x 60s before the same failure.
|
|
6082
|
+
loading = _FakeStreamResponse(503, "Loading model")
|
|
6083
|
+
client = _FakeStreamClient([loading, _FakeStreamResponse(200, "")])
|
|
6084
|
+
|
|
6085
|
+
resp, waited = self._run(client, health_ok=False)
|
|
6086
|
+
|
|
6087
|
+
self.assertIs(resp, loading, "unhealthy upstream surfaces the 503 now")
|
|
6088
|
+
self.assertEqual(client.sends, 1, "must not retry a still-unhealthy upstream")
|
|
6089
|
+
self.assertEqual(len(waited), 1)
|
|
6090
|
+
|
|
6091
|
+
def test_reads_the_body_before_classifying_a_503(self):
|
|
6092
|
+
# The fix hinges on aread() preceding _is_loading_model_503, whose bare
|
|
6093
|
+
# `except` would turn httpx's ResponseNotRead into a silent False and
|
|
6094
|
+
# disable the retry entirely. Deleting that aread() must fail here.
|
|
6095
|
+
loading = _FakeStreamResponse(503, "Loading model")
|
|
6096
|
+
ok = _FakeStreamResponse(200, "")
|
|
6097
|
+
client = _FakeStreamClient([loading, ok])
|
|
6098
|
+
|
|
6099
|
+
resp, _ = self._run(client)
|
|
6100
|
+
|
|
6101
|
+
self.assertIs(resp, ok)
|
|
6102
|
+
self.assertEqual(loading.read_count, 1, "body read exactly once, before .text")
|
|
6103
|
+
|
|
6104
|
+
def test_closes_the_response_when_the_body_read_fails(self):
|
|
6105
|
+
# A ReadError while reading the 503 body is retryable; if the discarded
|
|
6106
|
+
# response is not closed the connection accrues as CLOSE-WAIT until the
|
|
6107
|
+
# pool saturates, with the client seeing nothing wrong.
|
|
6108
|
+
class _ExplodingResponse(_FakeStreamResponse):
|
|
6109
|
+
async def aread(self):
|
|
6110
|
+
raise httpx.ReadError("reset mid-body")
|
|
6111
|
+
|
|
6112
|
+
boom = _ExplodingResponse(503, "Loading model")
|
|
6113
|
+
ok = _FakeStreamResponse(200, "")
|
|
6114
|
+
client = _FakeStreamClient([boom, ok])
|
|
6115
|
+
|
|
6116
|
+
resp, _ = self._run(client)
|
|
6117
|
+
|
|
6118
|
+
self.assertIs(resp, ok, "the read failure is retried")
|
|
6119
|
+
self.assertTrue(boom.closed, "the abandoned response must be closed")
|
|
6120
|
+
|
|
6121
|
+
def test_still_retries_connect_errors(self):
|
|
6122
|
+
# Regression guard for the pre-existing behaviour the extraction replaced.
|
|
6123
|
+
ok = _FakeStreamResponse(200, "")
|
|
6124
|
+
client = _FakeStreamClient([httpx.ConnectError("boom"), ok])
|
|
6125
|
+
|
|
6126
|
+
resp, _ = self._run(client)
|
|
6127
|
+
|
|
6128
|
+
self.assertIs(resp, ok)
|
|
6129
|
+
self.assertEqual(client.sends, 2)
|
|
6130
|
+
|
|
6131
|
+
def test_raises_when_every_connect_attempt_fails(self):
|
|
6132
|
+
client = _FakeStreamClient([httpx.ConnectError("boom") for _ in range(3)])
|
|
6133
|
+
|
|
6134
|
+
with self.assertRaises(httpx.ConnectError):
|
|
6135
|
+
self._run(client, retry_max=3)
|
|
6136
|
+
|
|
6137
|
+
self.assertEqual(client.sends, 3)
|
|
6138
|
+
|
|
6139
|
+
def test_retry_max_of_zero_still_makes_one_attempt(self):
|
|
6140
|
+
# An unclamped 0 skipped the loop and raised a bare RuntimeError, which
|
|
6141
|
+
# the call site's narrow `except` misses — an unhandled 500 where the
|
|
6142
|
+
# old code returned a clean 529.
|
|
6143
|
+
ok = _FakeStreamResponse(200, "")
|
|
6144
|
+
client = _FakeStreamClient([ok])
|
|
6145
|
+
|
|
6146
|
+
resp, _ = self._run(client, retry_max=0)
|
|
6147
|
+
|
|
6148
|
+
self.assertIs(resp, ok)
|
|
6149
|
+
self.assertEqual(client.sends, 1)
|
|
6150
|
+
|
|
6151
|
+
def test_preserves_the_streaming_contract_the_caller_depends_on(self):
|
|
6152
|
+
# stream=True keeps the turn incremental (buffering it would silently
|
|
6153
|
+
# reintroduce full-generation latency), and _uap_client drives the
|
|
6154
|
+
# inflight accounting that the pool-retire logic waits on.
|
|
6155
|
+
ok = _FakeStreamResponse(200, "")
|
|
6156
|
+
client = _FakeStreamClient([ok])
|
|
6157
|
+
|
|
6158
|
+
resp, _ = self._run(client)
|
|
6159
|
+
|
|
6160
|
+
self.assertEqual(client.stream_flags, [True])
|
|
6161
|
+
self.assertIs(getattr(resp, "_uap_client"), client)
|
|
6162
|
+
self.assertEqual(client.requests[0]["url"], "http://x/v1/chat/completions")
|
|
6163
|
+
|
|
6164
|
+
|
|
6165
|
+
if __name__ == "__main__":
|
|
6166
|
+
unittest.main()
|
|
@@ -45,7 +45,13 @@ NOT_GATED = ["package.json", "config.yaml", "data.xml", "README.md", "notes.txt"
|
|
|
45
45
|
|
|
46
46
|
def run(path, content="x" * 2000):
|
|
47
47
|
e = {**os.environ, "UAP_ENFORCE_DELIVERY": "block", "UAP_INFERENCE_ENDPOINT": "http://172.17.0.1:8080/v1"}
|
|
48
|
-
|
|
48
|
+
# ANTHROPIC_BASE_URL / OPENAI_BASE_URL must be stripped too: the enforcer
|
|
49
|
+
# downgrades block -> advisory for a local-model session, so a developer
|
|
50
|
+
# with a loopback base URL exported (the normal shape of a local session)
|
|
51
|
+
# sees every block-expecting test here allow instead. CI has them unset, so
|
|
52
|
+
# this is green in CI and red on the developer's machine.
|
|
53
|
+
for k in ("UAP_DELIVER_ACTIVE", "UAP_DELIVER_BYPASS", "UAP_DELIVER_LOCAL_MODE",
|
|
54
|
+
"UAP_DELIVER_LOCAL_ADVISORY", "ANTHROPIC_BASE_URL", "OPENAI_BASE_URL"):
|
|
49
55
|
e.pop(k, None)
|
|
50
56
|
p = subprocess.run(
|
|
51
57
|
[sys.executable, str(ENF), "--operation", "Write", "--args", json.dumps({"file_path": path, "content": content})],
|
|
@@ -21,7 +21,13 @@ ENF = Path(__file__).resolve().parents[3] / "src" / "policies" / "enforcers" / "
|
|
|
21
21
|
|
|
22
22
|
def run(path):
|
|
23
23
|
e = {**os.environ, "UAP_ENFORCE_DELIVERY": "block", "UAP_INFERENCE_ENDPOINT": "http://172.17.0.1:8080/v1"}
|
|
24
|
-
|
|
24
|
+
# ANTHROPIC_BASE_URL / OPENAI_BASE_URL must be stripped too: the enforcer
|
|
25
|
+
# downgrades block -> advisory for a local-model session, so a developer
|
|
26
|
+
# with a loopback base URL exported (the normal shape of a local session)
|
|
27
|
+
# sees every block-expecting test here allow instead. CI has them unset, so
|
|
28
|
+
# this is green in CI and red on the developer's machine.
|
|
29
|
+
for k in ("UAP_DELIVER_ACTIVE", "UAP_DELIVER_BYPASS", "UAP_DELIVER_LOCAL_MODE",
|
|
30
|
+
"UAP_DELIVER_LOCAL_ADVISORY", "ANTHROPIC_BASE_URL", "OPENAI_BASE_URL"):
|
|
25
31
|
e.pop(k, None)
|
|
26
32
|
p = subprocess.run(
|
|
27
33
|
[sys.executable, str(ENF), "--operation", "Write",
|
|
@@ -9,7 +9,13 @@ ENF = Path(__file__).resolve().parents[3] / "src" / "policies" / "enforcers" / "
|
|
|
9
9
|
|
|
10
10
|
def run(args, env=None):
|
|
11
11
|
e = {**os.environ, "UAP_ENFORCE_DELIVERY": "block", "UAP_INFERENCE_ENDPOINT": "http://172.17.0.1:8080/v1"}
|
|
12
|
-
|
|
12
|
+
# ANTHROPIC_BASE_URL / OPENAI_BASE_URL must be stripped too: the enforcer
|
|
13
|
+
# downgrades block -> advisory for a local-model session, so a developer
|
|
14
|
+
# with a loopback base URL exported (the normal shape of a local session)
|
|
15
|
+
# sees every block-expecting test here allow instead. CI has them unset, so
|
|
16
|
+
# this is green in CI and red on the developer's machine.
|
|
17
|
+
for k in ("UAP_DELIVER_ACTIVE", "UAP_DELIVER_BYPASS", "UAP_DELIVER_LOCAL_MODE",
|
|
18
|
+
"UAP_DELIVER_LOCAL_ADVISORY", "ANTHROPIC_BASE_URL", "OPENAI_BASE_URL"):
|
|
13
19
|
e.pop(k, None)
|
|
14
20
|
if env:
|
|
15
21
|
e.update(env)
|
|
@@ -17,7 +17,13 @@ ENF = Path(__file__).resolve().parents[3] / "src" / "policies" / "enforcers" / "
|
|
|
17
17
|
|
|
18
18
|
def run(op, args, env=None):
|
|
19
19
|
e = {**os.environ, "UAP_ENFORCE_DELIVERY": "block", "UAP_INFERENCE_ENDPOINT": "http://172.17.0.1:8080/v1"}
|
|
20
|
-
|
|
20
|
+
# ANTHROPIC_BASE_URL / OPENAI_BASE_URL must be stripped too: the enforcer
|
|
21
|
+
# downgrades block -> advisory for a local-model session, so a developer
|
|
22
|
+
# with a loopback base URL exported (the normal shape of a local session)
|
|
23
|
+
# sees every block-expecting test here allow instead. CI has them unset, so
|
|
24
|
+
# this is green in CI and red on the developer's machine.
|
|
25
|
+
for k in ("UAP_DELIVER_ACTIVE", "UAP_DELIVER_BYPASS", "UAP_DELIVER_LOCAL_MODE",
|
|
26
|
+
"UAP_DELIVER_LOCAL_ADVISORY", "ANTHROPIC_BASE_URL", "OPENAI_BASE_URL"):
|
|
21
27
|
e.pop(k, None)
|
|
22
28
|
if env:
|
|
23
29
|
e.update(env)
|
|
@@ -24,8 +24,17 @@ def run(path, root, env_extra=None):
|
|
|
24
24
|
env = dict(os.environ)
|
|
25
25
|
env["UAP_REPO_ROOT"] = str(root)
|
|
26
26
|
env["UAP_ENFORCE_DELIVERY"] = "block"
|
|
27
|
-
|
|
28
|
-
|
|
27
|
+
# Every ambient input that can turn "block" into "allow" must be cleared,
|
|
28
|
+
# or these tests assert the developer's shell rather than the enforcer.
|
|
29
|
+
# ANTHROPIC_BASE_URL is the subtle one: delivery_enforcement downgrades
|
|
30
|
+
# block -> advisory for a local-model session, so with a loopback base URL
|
|
31
|
+
# exported (the normal shape of a local session here) the three
|
|
32
|
+
# block-expecting tests below flip to allowed. CI has it unset, so this
|
|
33
|
+
# fails only for developers — presenting as "behaviour drifted".
|
|
34
|
+
for k in ("UAP_DELIVER_ACTIVE", "UAP_DELIVER_BYPASS",
|
|
35
|
+
"UAP_DELIVER_LOCAL_MODE", "UAP_DELIVER_LOCAL_ADVISORY",
|
|
36
|
+
"ANTHROPIC_BASE_URL", "OPENAI_BASE_URL"):
|
|
37
|
+
env.pop(k, None)
|
|
29
38
|
if env_extra:
|
|
30
39
|
env.update(env_extra)
|
|
31
40
|
p = subprocess.run(
|
|
@@ -82,8 +91,12 @@ class TestDeliveryEnforcementWorktree(unittest.TestCase):
|
|
|
82
91
|
)
|
|
83
92
|
|
|
84
93
|
|
|
85
|
-
|
|
86
|
-
|
|
94
|
+
# NB: the `unittest.main()` entrypoint lives at the END of this file, not here.
|
|
95
|
+
# It used to sit at this point, above the two classes below, so running the file
|
|
96
|
+
# directly (`python3 tools/agents/tests/test_delivery_enforcement_worktree.py`)
|
|
97
|
+
# ran 7 tests and printed OK while `python -m unittest` ran 14 — the 7 it
|
|
98
|
+
# skipped being exactly LocalAdvisoryTest + LocalModeTest. Anyone verifying a
|
|
99
|
+
# change to those got a false green.
|
|
87
100
|
|
|
88
101
|
|
|
89
102
|
import os as _os, subprocess as _sp, sys as _sys, json as _json, tempfile as _tf
|
|
@@ -96,7 +109,21 @@ class LocalAdvisoryTest(unittest.TestCase):
|
|
|
96
109
|
root = _Path(td); (root/".git").mkdir()
|
|
97
110
|
f = root/"src"/"a.ts"; f.parent.mkdir(parents=True); f.write_text("x")
|
|
98
111
|
e = dict(_os.environ); e["UAP_REPO_ROOT"]=str(root)
|
|
99
|
-
for
|
|
112
|
+
# This test asserts the DEFAULT for a local session, so every input
|
|
113
|
+
# that overrides that default has to be cleared:
|
|
114
|
+
# - UAP_DELIVER_BYPASS: run() already strips it. An agent shell
|
|
115
|
+
# commonly exports it, the enforcer then allows every write, and
|
|
116
|
+
# the block-expecting tests fail rc 0 != 2 — which reads as
|
|
117
|
+
# "behaviour drifted" rather than "your env leaked".
|
|
118
|
+
# - UAP_DELIVER_LOCAL_MODE: LocalModeTest._run already strips it;
|
|
119
|
+
# omitting it here was an inconsistency. Importing the proxy
|
|
120
|
+
# module (any test that does _load_proxy_module()) runs
|
|
121
|
+
# _load_proxy_env_file(), which loads .uap/proxy.env into the
|
|
122
|
+
# real os.environ — and that file sets UAP_DELIVER_LOCAL_MODE.
|
|
123
|
+
# So this test's result depended on whether a proxy-importing
|
|
124
|
+
# module ran before it in the same process.
|
|
125
|
+
for k in ("ANTHROPIC_BASE_URL","UAP_DELIVER_LOCAL_ADVISORY","UAP_DELIVER_ACTIVE",
|
|
126
|
+
"UAP_DELIVER_BYPASS","UAP_DELIVER_LOCAL_MODE"): e.pop(k, None)
|
|
100
127
|
e.update(extra_env)
|
|
101
128
|
p = _sp.run([_sys.executable, str(_ENF), "--operation","Write","--args",_json.dumps({"file_path":str(f)})], capture_output=True, text=True, env=e)
|
|
102
129
|
return p.returncode, _json.loads(p.stdout) if p.stdout.strip() else {}
|
|
@@ -120,7 +147,8 @@ class LocalModeTest(unittest.TestCase):
|
|
|
120
147
|
root = _Path(td); (root/".git").mkdir()
|
|
121
148
|
f = root/"src"/"a.ts"; f.parent.mkdir(parents=True); f.write_text("x")
|
|
122
149
|
e = dict(_os.environ); e["UAP_REPO_ROOT"]=str(root)
|
|
123
|
-
|
|
150
|
+
# See the note in LocalAdvisoryTest._run — same ambient-bypass leak.
|
|
151
|
+
for k in ("ANTHROPIC_BASE_URL","UAP_DELIVER_LOCAL_ADVISORY","UAP_DELIVER_LOCAL_MODE","UAP_DELIVER_ACTIVE","UAP_DELIVER_BYPASS"): e.pop(k, None)
|
|
124
152
|
e.update(env)
|
|
125
153
|
p = _sp.run([_sys.executable, str(_ENF), "--operation","Write","--args",_json.dumps({"file_path":str(f)})], capture_output=True, text=True, env=e)
|
|
126
154
|
return p.returncode, _json.loads(p.stdout) if p.stdout.strip() else {}
|
|
@@ -141,3 +169,7 @@ class LocalModeTest(unittest.TestCase):
|
|
|
141
169
|
def test_default_is_advisory(self):
|
|
142
170
|
rc, out = self._run({"ANTHROPIC_BASE_URL":"http://127.0.0.1:4000"})
|
|
143
171
|
self.assertEqual(rc, 0)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
if __name__ == "__main__":
|
|
175
|
+
unittest.main()
|
|
@@ -28,8 +28,19 @@ TESTS = Path(__file__).resolve().parent
|
|
|
28
28
|
# Verified failing for their own pre-existing reasons, not by omission. Each
|
|
29
29
|
# entry is a debt with a stated cause — not a place to park a newly broken test.
|
|
30
30
|
KNOWN_EXCLUDED = {
|
|
31
|
-
|
|
32
|
-
|
|
31
|
+
# test_anthropic_proxy_streaming and test_delivery_enforcement_worktree
|
|
32
|
+
# were both listed here as "behaviour drifted from the assertions" and are
|
|
33
|
+
# now listed in test:enforcers instead:
|
|
34
|
+
# - proxy_streaming had 7 genuinely stale assertions (max_tokens floors
|
|
35
|
+
# superseded by the thinking-floor, a grammar guard that postdated the
|
|
36
|
+
# test, the turn-count breaker's rewording, and two malformed-payload
|
|
37
|
+
# cases deliberately reversed by _strip_orphan_tool_xml). All rewritten
|
|
38
|
+
# against current intent. This is the module that covers the streaming
|
|
39
|
+
# 503 retry, so it was the gap that let that bug ship.
|
|
40
|
+
# - delivery_enforcement_worktree never drifted at all: two of its env
|
|
41
|
+
# helpers failed to strip an ambient UAP_DELIVER_BYPASS, so the
|
|
42
|
+
# enforcer allowed every write and four block-expecting tests read 0
|
|
43
|
+
# instead of 2. It passes clean; the helpers are now hermetic.
|
|
33
44
|
"test_uap_compliance": "needs a populated DB; environment-bound",
|
|
34
45
|
}
|
|
35
46
|
|
|
@@ -87,5 +98,50 @@ class TestEveryListedModuleActuallyCollects(unittest.TestCase):
|
|
|
87
98
|
assert ghosts == set(), f"test:enforcers names modules that do not exist: {sorted(ghosts)}"
|
|
88
99
|
|
|
89
100
|
|
|
101
|
+
class TestTheGateCollectsWhatItClaims(unittest.TestCase):
|
|
102
|
+
"""Every check above is structural — none pins how many tests actually run.
|
|
103
|
+
|
|
104
|
+
A listed module can define a TestCase and still contribute almost nothing.
|
|
105
|
+
Two real instances: a mid-file `unittest.main()` in
|
|
106
|
+
test_anthropic_proxy_streaming and test_delivery_enforcement_worktree cut
|
|
107
|
+
direct runs off partway through the file (273 -> 168 and 14 -> 7) while
|
|
108
|
+
still printing OK; and a mass deletion inside any listed module is invisible
|
|
109
|
+
to the listing checks. A floor on the collected count catches both.
|
|
110
|
+
"""
|
|
111
|
+
|
|
112
|
+
# Deliberately below the current count so ordinary churn doesn't trip it.
|
|
113
|
+
# Raise it when the suite grows substantially; never lower it to make a
|
|
114
|
+
# failing gate pass — that is the deletion this test exists to catch.
|
|
115
|
+
MINIMUM_COLLECTED = 750
|
|
116
|
+
|
|
117
|
+
def test_listed_modules_collect_at_least_the_expected_test_count(self):
|
|
118
|
+
loader = unittest.TestLoader()
|
|
119
|
+
suite = loader.loadTestsFromNames(
|
|
120
|
+
[f"tools.agents.tests.{m}" for m in sorted(_listed_modules())]
|
|
121
|
+
)
|
|
122
|
+
# loadTestsFromNames turns an import failure into a _FailedTest that
|
|
123
|
+
# still counts, so surface those rather than let them pad the total.
|
|
124
|
+
broken = [
|
|
125
|
+
str(t) for t in _flatten(suite)
|
|
126
|
+
if type(t).__name__ == "_FailedTest"
|
|
127
|
+
]
|
|
128
|
+
assert broken == [], f"listed modules that fail to even import: {broken}"
|
|
129
|
+
|
|
130
|
+
count = suite.countTestCases()
|
|
131
|
+
assert count >= self.MINIMUM_COLLECTED, (
|
|
132
|
+
f"the gate collects {count} tests, below the {self.MINIMUM_COLLECTED} floor. "
|
|
133
|
+
"Tests were deleted, a module stopped collecting, or an entrypoint "
|
|
134
|
+
"truncates the file. Investigate before adjusting this number."
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _flatten(suite):
|
|
139
|
+
for item in suite:
|
|
140
|
+
if isinstance(item, unittest.TestSuite):
|
|
141
|
+
yield from _flatten(item)
|
|
142
|
+
else:
|
|
143
|
+
yield item
|
|
144
|
+
|
|
145
|
+
|
|
90
146
|
if __name__ == "__main__":
|
|
91
147
|
unittest.main()
|