@yahyahammad/venastine 0.1.0 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -14
- package/CONFIG_ARCHITECTURE.md +2002 -0
- package/NOTICE +1 -1
- package/README.md +249 -93
- package/agents/builtin/build.md +73 -0
- package/agents/builtin/general.md +80 -0
- package/agents/builtin/plan.md +23 -5
- package/agents/builtin/test.md +65 -0
- package/agents/builtin/writer.md +67 -0
- package/agents/manager.py +261 -195
- package/agents/subagent_tool.py +280 -23
- package/agents/tui_commands.py +31 -21
- package/bin/venastine.mjs +528 -382
- package/bin/venastine.py +71 -0
- package/cli_commands.py +835 -0
- package/config.py +95 -1198
- package/config.yaml +1090 -0
- package/config_edit.py +1301 -0
- package/config_schema.py +966 -0
- package/config_update.py +749 -0
- package/core/agent_activity.py +256 -0
- package/core/ansi.py +452 -0
- package/core/approval.py +27 -5
- package/core/call_view.py +364 -0
- package/core/cancel.py +121 -0
- package/core/chatgpt_auth.py +747 -0
- package/core/claims_view.py +107 -0
- package/core/client.py +1249 -82
- package/core/commands.py +454 -0
- package/core/compaction.py +1735 -959
- package/core/config_loader.py +516 -97
- package/core/connect.py +686 -0
- package/core/events.py +68 -6
- package/core/file_mentions.py +280 -0
- package/core/interaction.py +87 -4
- package/core/line_pattern.py +128 -0
- package/core/loop.py +2566 -1353
- package/core/memory.py +482 -370
- package/core/pipeline_models.py +23 -9
- package/core/provider_errors.py +234 -0
- package/core/reasoning/confidence_scoring.py +70 -54
- package/core/reasoning/json_retry.py +19 -1
- package/core/reasoning/orchestrator.py +232 -53
- package/core/reasoning/output_writer.py +90 -31
- package/core/reasoning/payload_validation.py +133 -7
- package/core/reasoning/pipeline_storage.py +427 -361
- package/core/reasoning/review.py +659 -615
- package/core/reasoning/scholar.py +38 -14
- package/core/reasoning/source_corpus.py +110 -4
- package/core/replay.py +315 -167
- package/core/session.py +36 -12
- package/core/session_view.py +259 -0
- package/core/session_wake.py +259 -0
- package/core/shell_sessions.py +1553 -0
- package/core/workspace_trust.py +1 -1
- package/credentials.py +308 -180
- package/database.py +82 -1
- package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
- package/docs/SECURITY.md +108 -0
- package/docs/THIRD_PARTY_NOTICES.md +148 -0
- package/env_secrets.py +118 -77
- package/json_store.py +236 -16
- package/logging_setup.py +121 -11
- package/main.py +1672 -198
- package/mcp_client/client.py +257 -23
- package/mcp_client/config.py +484 -379
- package/mcp_client/registration.py +163 -2
- package/memories/manager.py +15 -0
- package/memories/tui_commands.py +21 -14
- package/package.json +303 -73
- package/project_init/config_files.py +4 -0
- package/project_init/generator.py +201 -37
- package/project_init/manifest.py +418 -406
- package/project_init/tui_commands.py +7 -4
- package/prompts/revalidate.md +1 -1
- package/prompts/revise.md +17 -15
- package/prompts/system_prompts.py +64 -10
- package/providers.json.example +94 -92
- package/pyproject.toml +149 -100
- package/requirements.txt +252 -29
- package/safety/policy_enforcement.py +1082 -682
- package/security/askpass.py +419 -0
- package/security/capability.py +37 -7
- package/security/confine.py +484 -0
- package/security/hosted.py +182 -0
- package/security/permissions.py +54 -7
- package/security/posture.py +124 -12
- package/security/protected_paths.py +198 -6
- package/security/sandbox.py +3262 -139
- package/security/secrets.py +855 -0
- package/skills/tui_commands.py +3 -4
- package/storage.py +1338 -1005
- package/tools/base.py +465 -305
- package/tools/builtin/_math_common.py +306 -16
- package/tools/builtin/_net_common.py +267 -61
- package/tools/builtin/arxiv.py +681 -223
- package/tools/builtin/ask_user.py +231 -213
- package/tools/builtin/discrete_math.py +35 -4
- package/tools/builtin/fetch_url.py +239 -21
- package/tools/builtin/file_ops.py +341 -33
- package/tools/builtin/geometry.py +184 -148
- package/tools/builtin/get_time.py +2 -2
- package/tools/builtin/linear_algebra.py +66 -23
- package/tools/builtin/load_skill.py +10 -0
- package/tools/builtin/logic.py +171 -12
- package/tools/builtin/pin.py +46 -12
- package/tools/builtin/probability_stats.py +224 -228
- package/tools/builtin/project_docs.py +80 -21
- package/tools/builtin/remember.py +21 -4
- package/tools/builtin/shell.py +586 -67
- package/tools/builtin/shell_sessions.py +613 -0
- package/tools/builtin/symbolic_math.py +80 -10
- package/tools/builtin/todo.py +5 -3
- package/tools/builtin/web_search.py +329 -151
- package/tools/context.py +23 -0
- package/tools/isolation.py +387 -314
- package/tools/registry.py +374 -29
- package/tui/app.py +5281 -397
- package/tui/app.tcss +290 -9
- package/tui/commands.py +26 -61
- package/tui/diffs.py +286 -265
- package/tui/history.py +128 -0
- package/tui/hosted_driver.py +102 -0
- package/tui/markdown.py +1329 -0
- package/tui/meters.py +364 -0
- package/tui/screens.py +346 -92
- package/tui/themes.py +599 -366
- package/tui/widgets.py +2673 -116
- package/SECURITY.md +0 -88
- package/THIRD_PARTY_NOTICES.md +0 -100
package/core/pipeline_models.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""
|
|
2
2
|
core/pipeline_models.py
|
|
3
3
|
|
|
4
|
-
The
|
|
4
|
+
The model roles that are not the session model, remembered.
|
|
5
5
|
|
|
6
6
|
ROADMAP_v2 §45 (SQ7). A research run reaches for three models: the
|
|
7
7
|
generator (the session pair, `/model`), the CRITIC that grounds and
|
|
@@ -19,10 +19,10 @@ import — WS6's argument for `core/model_windows.py`, unchanged, and §25's
|
|
|
19
19
|
lesson about a capability the pipeline could not reach. It is equally not
|
|
20
20
|
`core/session.py`, whose contract is that nothing there reaches disk.
|
|
21
21
|
|
|
22
|
-
|
|
23
|
-
RM4's reason: a writer serialising only its own role would
|
|
24
|
-
the
|
|
25
|
-
|
|
22
|
+
EVERY ROLE IN ONE FILE, so every writer reads, modifies and writes. RM4's
|
|
23
|
+
shape and RM4's reason: a writer serialising only its own role would
|
|
24
|
+
silently drop the others', each correct in its own test. They are one file
|
|
25
|
+
rather than a module each because they are the same KIND of fact — which model plays
|
|
26
26
|
which part in a run — and `model_windows.py` is next door as the example
|
|
27
27
|
of when a second file is right instead (it holds a different kind of fact,
|
|
28
28
|
keyed on a pair rather than naming one).
|
|
@@ -40,6 +40,17 @@ this harness's embedding calls — claim text and fetched page text — at a
|
|
|
40
40
|
provider of its choosing. `embedder_model` and `critic_model` are
|
|
41
41
|
therefore rejected there BY NAME, like `ensemble_models`, and this
|
|
42
42
|
user-tier store is the route instead.
|
|
43
|
+
|
|
44
|
+
A THIRD ROLE SINCE ROADMAP_v3 §57: `subagent`, set by `/submodel` in
|
|
45
|
+
either shell. It is the model a SPAWNED child runs on when its own file
|
|
46
|
+
pins neither half of a pair (SM6), at every depth and whatever its parent
|
|
47
|
+
runs on (SM7) -- so a cheap child no longer has to run on its parent's
|
|
48
|
+
expensive model. It is the same kind of fact as the other two, which part
|
|
49
|
+
a model plays, and so it shares their file and their read-modify-write.
|
|
50
|
+
It has NO `config.yaml` tier (SM5): `resolve` answers from this store or
|
|
51
|
+
not at all, which is why its `_FALLBACK_ATTR` entry is None rather than a
|
|
52
|
+
constant nobody sets. Nothing else reads it: the compactor, the pipeline
|
|
53
|
+
reviewer and the initializer keep the session model (SM8).
|
|
43
54
|
"""
|
|
44
55
|
|
|
45
56
|
import logging
|
|
@@ -61,10 +72,12 @@ STORE_VERSION = 1
|
|
|
61
72
|
# stored: a typo'd `/embeder` writing a key nothing reads would look like
|
|
62
73
|
# it worked and change nothing, which is the failure mode this whole
|
|
63
74
|
# module exists to remove for CRITIC_MODEL.
|
|
64
|
-
ROLES = ("critic", "embedder")
|
|
75
|
+
ROLES = ("critic", "embedder", "subagent")
|
|
65
76
|
|
|
66
|
-
# Which config.py constant answers when the store is silent.
|
|
67
|
-
|
|
77
|
+
# Which config.py constant answers when the store is silent. None means
|
|
78
|
+
# nothing does: `subagent` is remembered or unset (§57, SM5).
|
|
79
|
+
_FALLBACK_ATTR = {"critic": "CRITIC_MODEL", "embedder": "EMBEDDER_MODEL",
|
|
80
|
+
"subagent": None}
|
|
68
81
|
|
|
69
82
|
|
|
70
83
|
def store_path() -> str:
|
|
@@ -137,7 +150,8 @@ def resolve(role: str) -> Optional[dict]:
|
|
|
137
150
|
chosen = remembered(role)
|
|
138
151
|
if chosen is not None:
|
|
139
152
|
return chosen
|
|
140
|
-
|
|
153
|
+
attr = _FALLBACK_ATTR[role]
|
|
154
|
+
fallback = getattr(config, attr, None) if attr else None
|
|
141
155
|
return fallback if _valid(fallback) else None
|
|
142
156
|
|
|
143
157
|
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"""
|
|
2
|
+
core/provider_errors.py
|
|
3
|
+
|
|
4
|
+
Which provider failures are worth another attempt, how long to wait before
|
|
5
|
+
it, and the one line that describes a failure (batch 91).
|
|
6
|
+
|
|
7
|
+
WHY THIS IS NOT IN core/client.py. AGENTS.md's ownership table gives
|
|
8
|
+
client.py one call and provider format translation, and says in so many
|
|
9
|
+
words that it does not own retry. The loop owns the attempt; this module
|
|
10
|
+
answers the two questions the attempt asks -- "is this one worth another
|
|
11
|
+
try?" and "how long do I wait?" -- and holds no state of its own.
|
|
12
|
+
|
|
13
|
+
WHY NO SDK IS IMPORTED, HERE OR AT CALL TIME. core/client.py imports each SDK
|
|
14
|
+
lazily inside api_initialization, so a harness configured for one provider
|
|
15
|
+
never loads the other two -- and the suite installs FAKE openai, anthropic,
|
|
16
|
+
google.genai and httpx modules before any test imports anything (the root
|
|
17
|
+
conftest). An isinstance check against a real SDK class would load what
|
|
18
|
+
client.py deliberately does not, and under test would be comparing against
|
|
19
|
+
a fake. So an exception is read by the class names in its MRO and by the
|
|
20
|
+
attributes the SDKs themselves define: `status_code` (openai / anthropic
|
|
21
|
+
APIStatusError), an int `code` (google.genai errors.APIError), `body` (the
|
|
22
|
+
error object an already-open stream carried) and `response.headers`.
|
|
23
|
+
|
|
24
|
+
WHAT COUNTS AS TRANSIENT (owner decision, batch 91):
|
|
25
|
+
|
|
26
|
+
* a connection failure or a timeout -- openai / anthropic
|
|
27
|
+
APIConnectionError and APITimeoutError, an httpx transport error raised
|
|
28
|
+
raw while a stream is being read (and by google-genai, whose transport
|
|
29
|
+
httpx is since 2.x), a requests ConnectionError / Timeout (google-auth's
|
|
30
|
+
token refresh; google-genai 1.0's transport), and the builtin ConnectionError /
|
|
31
|
+
TimeoutError;
|
|
32
|
+
* HTTP 408, 409, 429 and any 5xx -- the statuses the openai and anthropic
|
|
33
|
+
SDKs already retry for themselves before a stream opens;
|
|
34
|
+
* an error event inside a stream that had ALREADY opened with 200. The
|
|
35
|
+
openai SDK raises its base APIError for it, with no status at all;
|
|
36
|
+
anthropic raises APIStatusError carrying the 200 the stream opened with.
|
|
37
|
+
Neither SDK retries it, and that is the gap this module exists for:
|
|
38
|
+
OpenRouter delivered "The service is temporarily unavailable" and
|
|
39
|
+
"Upstream idle timeout exceeded" exactly this way. Retried unless the
|
|
40
|
+
error object itself names a 4xx code or a permanent error type.
|
|
41
|
+
|
|
42
|
+
Everything else is NOT: a bad request, an auth failure or an unknown model
|
|
43
|
+
fails identically on every attempt, and a harness error -- D21's
|
|
44
|
+
RuntimeError, a ValueError from message translation -- is a bug, not
|
|
45
|
+
weather, and retrying it would only delay the traceback.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
from random import random
|
|
49
|
+
from time import sleep
|
|
50
|
+
from typing import Optional
|
|
51
|
+
|
|
52
|
+
#: Beside any 5xx. The same set the openai and anthropic SDKs retry.
|
|
53
|
+
RETRYABLE_STATUSES = frozenset({408, 409, 429})
|
|
54
|
+
|
|
55
|
+
#: A server's retry-after is honoured up to this, as the SDKs do.
|
|
56
|
+
RETRY_AFTER_CAP_S = 60.0
|
|
57
|
+
|
|
58
|
+
#: Up to a quarter of the delay either way, so parallel siblings that failed
|
|
59
|
+
#: together -- three children of one spawn batch, which is the case that
|
|
60
|
+
#: prompted this -- do not all come back in the same instant.
|
|
61
|
+
JITTER = 0.25
|
|
62
|
+
|
|
63
|
+
#: A transcript line, not a traceback. Long enough for a provider's own
|
|
64
|
+
#: sentence and the error object google-genai folds into its message.
|
|
65
|
+
DESCRIBE_LIMIT = 500
|
|
66
|
+
|
|
67
|
+
_CONNECTION_NAMES = frozenset({
|
|
68
|
+
"APIConnectionError", "APITimeoutError", # openai, anthropic
|
|
69
|
+
"TransportError", "TimeoutException", # httpx: raw mid-stream; google-genai
|
|
70
|
+
"ConnectionError", "Timeout", # requests (google-auth); builtin
|
|
71
|
+
"ChunkedEncodingError", # requests, a body cut off mid-read
|
|
72
|
+
"TimeoutError", # builtin
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
#: Error types an open stream's error object can name. Listed in both
|
|
76
|
+
#: directions rather than inferred from a suffix: openai's "server_error" and
|
|
77
|
+
#: anthropic's "invalid_request_error" share one.
|
|
78
|
+
_TRANSIENT_TYPES = frozenset({
|
|
79
|
+
"overloaded_error", "api_error", "rate_limit_error", "server_error",
|
|
80
|
+
})
|
|
81
|
+
_PERMANENT_TYPES = frozenset({
|
|
82
|
+
"invalid_request_error", "authentication_error", "permission_error",
|
|
83
|
+
"not_found_error", "request_too_large",
|
|
84
|
+
})
|
|
85
|
+
|
|
86
|
+
#: A 200 that failed schema validation is a mismatch between the SDK and the
|
|
87
|
+
#: response, and it will be the same response next time.
|
|
88
|
+
_NEVER_TRANSIENT_NAMES = frozenset({"APIResponseValidationError"})
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _names(exc: BaseException) -> set:
|
|
92
|
+
return {cls.__name__ for cls in type(exc).__mro__}
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _as_status(value) -> Optional[int]:
|
|
96
|
+
if isinstance(value, str) and value.isdigit():
|
|
97
|
+
value = int(value)
|
|
98
|
+
if isinstance(value, int) and not isinstance(value, bool) \
|
|
99
|
+
and 100 <= value < 600:
|
|
100
|
+
return value
|
|
101
|
+
return None
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _retryable_status(status: int) -> bool:
|
|
105
|
+
return status in RETRYABLE_STATUSES or status >= 500
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _status(exc: BaseException, names: set) -> Optional[int]:
|
|
109
|
+
"""The HTTP status an exception carries, or None.
|
|
110
|
+
|
|
111
|
+
`code` is read only on something named APIError, because google-genai
|
|
112
|
+
is the one SDK that spells its status that way and the attribute name is
|
|
113
|
+
far too common to trust anywhere else.
|
|
114
|
+
"""
|
|
115
|
+
status = _as_status(getattr(exc, "status_code", None))
|
|
116
|
+
if status is None and "APIError" in names:
|
|
117
|
+
status = _as_status(getattr(exc, "code", None))
|
|
118
|
+
return status
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _body_verdict(body) -> Optional[bool]:
|
|
122
|
+
"""What an open stream's error object says, or None when it is silent.
|
|
123
|
+
|
|
124
|
+
openai hands over the error object itself (`data["error"]`); anthropic
|
|
125
|
+
hands over the whole event (`{"type": "error", "error": {...}}`), so the
|
|
126
|
+
nested object is preferred when there is one.
|
|
127
|
+
"""
|
|
128
|
+
if not isinstance(body, dict):
|
|
129
|
+
return None
|
|
130
|
+
error = body["error"] if isinstance(body.get("error"), dict) else body
|
|
131
|
+
status = _as_status(error.get("code"))
|
|
132
|
+
if status is not None:
|
|
133
|
+
return _retryable_status(status)
|
|
134
|
+
kind = error.get("type")
|
|
135
|
+
if kind in _TRANSIENT_TYPES:
|
|
136
|
+
return True
|
|
137
|
+
if kind in _PERMANENT_TYPES:
|
|
138
|
+
return False
|
|
139
|
+
return None
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def is_transient(exc: BaseException) -> bool:
|
|
143
|
+
"""Would the same call plausibly succeed if simply made again?"""
|
|
144
|
+
names = _names(exc)
|
|
145
|
+
if names & _NEVER_TRANSIENT_NAMES:
|
|
146
|
+
return False
|
|
147
|
+
if names & _CONNECTION_NAMES:
|
|
148
|
+
return True
|
|
149
|
+
status = _status(exc, names)
|
|
150
|
+
if status is not None and status != 200:
|
|
151
|
+
return _retryable_status(status)
|
|
152
|
+
if "APIError" in names:
|
|
153
|
+
# An error event inside a stream that opened: no status (openai) or
|
|
154
|
+
# the stream's own 200 (anthropic). The provider has said something
|
|
155
|
+
# went wrong mid-generation; unless it also said the request itself
|
|
156
|
+
# is at fault, that is the weather this retry is for.
|
|
157
|
+
return open_stream_error_is_transient(getattr(exc, "body", None))
|
|
158
|
+
return False
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def open_stream_error_is_transient(body) -> bool:
|
|
162
|
+
"""The open-stream rule above, for a caller holding only the error object.
|
|
163
|
+
|
|
164
|
+
ROADMAP_v3 §83: the ChatGPT plan's `error` event and `response.failed`
|
|
165
|
+
arrive as EVENTS -- the openai SDK raises only on a top-level `error`
|
|
166
|
+
object, which neither has -- so `core/client.py` reads them and asks
|
|
167
|
+
here, rather than keeping a second copy of the rule.
|
|
168
|
+
"""
|
|
169
|
+
verdict = _body_verdict(body)
|
|
170
|
+
return True if verdict is None else verdict
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def retry_after_s(exc: BaseException) -> Optional[float]:
|
|
174
|
+
"""A server's `retry-after`, in seconds, when the failure carried one.
|
|
175
|
+
|
|
176
|
+
Only the delta-seconds form. The HTTP-date form is legal and rare enough
|
|
177
|
+
on these APIs that a date parser here would be code with no caller; an
|
|
178
|
+
unreadable value is simply ignored and the backoff decides.
|
|
179
|
+
"""
|
|
180
|
+
headers = getattr(getattr(exc, "response", None), "headers", None)
|
|
181
|
+
if headers is None:
|
|
182
|
+
return None
|
|
183
|
+
try:
|
|
184
|
+
seconds = float(headers.get("retry-after"))
|
|
185
|
+
except (AttributeError, TypeError, ValueError):
|
|
186
|
+
return None
|
|
187
|
+
if seconds < 0:
|
|
188
|
+
return None
|
|
189
|
+
return min(seconds, RETRY_AFTER_CAP_S)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def backoff_delay(retry: int, base_s: float,
|
|
193
|
+
retry_after: Optional[float] = None) -> float:
|
|
194
|
+
"""Seconds to wait before retry number `retry` (1-based).
|
|
195
|
+
|
|
196
|
+
`base_s`, doubled for each retry after the first, with JITTER either way
|
|
197
|
+
(owner decision: longer than the SDKs' own half-second start, so a
|
|
198
|
+
temporary rate limit is not met with a burst of attempts).
|
|
199
|
+
|
|
200
|
+
A server's retry-after can make the wait LONGER, never shorter: a
|
|
201
|
+
`retry-after: 0` answering a rate limit is exactly the hammering the
|
|
202
|
+
longer base exists to avoid.
|
|
203
|
+
"""
|
|
204
|
+
delay = base_s * (2 ** (retry - 1))
|
|
205
|
+
delay *= 1 + JITTER * (2 * random() - 1)
|
|
206
|
+
if retry_after is not None:
|
|
207
|
+
delay = max(delay, retry_after)
|
|
208
|
+
return delay
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def wait(seconds: float) -> None:
|
|
212
|
+
"""Sleep before a retry.
|
|
213
|
+
|
|
214
|
+
Through `sleep` bound HERE, so a test patches `core.provider_errors.sleep`
|
|
215
|
+
and reaches nothing else -- AGENTS.md's `from time import monotonic`
|
|
216
|
+
rule: patching the global `time` module froze textual's event loop and
|
|
217
|
+
conftest's `settle` along with it, and the suite hung rather than failed.
|
|
218
|
+
"""
|
|
219
|
+
sleep(seconds)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def describe(exc: BaseException) -> str:
|
|
223
|
+
"""`Type: message`, on one line -- the ONE spelling of a failure.
|
|
224
|
+
|
|
225
|
+
Shared by the transcript's log handler, the failure stored on a thread
|
|
226
|
+
and the retry warning, so the three never describe the same exception
|
|
227
|
+
three ways. Not redacted: every caller that shows or
|
|
228
|
+
stores it redacts the whole line it is part of.
|
|
229
|
+
"""
|
|
230
|
+
text = " ".join(str(exc).split())
|
|
231
|
+
if len(text) > DESCRIBE_LIMIT:
|
|
232
|
+
text = text[:DESCRIBE_LIMIT - 1].rstrip() + "…"
|
|
233
|
+
name = type(exc).__name__
|
|
234
|
+
return f"{name}: {text}" if text else name
|
|
@@ -18,7 +18,11 @@ flag it if you intended something different.
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
20
|
from core.reasoning.base import (
|
|
21
|
-
CLAIM_TYPES,
|
|
21
|
+
CLAIM_TYPES,
|
|
22
|
+
GROUNDING_STATUSES,
|
|
23
|
+
Claim,
|
|
24
|
+
ConfidenceTier,
|
|
25
|
+
PipelineRun,
|
|
22
26
|
)
|
|
23
27
|
|
|
24
28
|
# --- Tunable weights -- change these, not the logic below, to retune scoring ---
|
|
@@ -28,27 +32,32 @@ CRITIC_WEIGHT_FACTOR = 0.35
|
|
|
28
32
|
ASSUMPTION_FLAG_PENALTY = 0.15 # per flag
|
|
29
33
|
NON_FACTUAL_SCORE_CAP = 0.65 # claims never grounded can't reach HIGH
|
|
30
34
|
|
|
31
|
-
#
|
|
32
|
-
#
|
|
33
|
-
#
|
|
35
|
+
# ROADMAP_v3 §90 (A85, issue #12). Cross-candidate disagreement is ROUTED,
|
|
36
|
+
# NOT SCORED. A factual claim asserted by at most this share of the
|
|
37
|
+
# candidates is `contested`, and D1 sends it to the 6a/6b loop to be
|
|
38
|
+
# checked; nothing is subtracted for it.
|
|
34
39
|
#
|
|
35
|
-
# §10
|
|
36
|
-
#
|
|
40
|
+
# It was a penalty before (§10's revisit, E8): 0.15 * (1 - consistency),
|
|
41
|
+
# itself the repair of §10's first shape, a fourth weighted term that made
|
|
42
|
+
# grounding count for less whenever ensemble mode was on. Two things were
|
|
43
|
+
# wrong with the penalty, and the second was measured:
|
|
37
44
|
#
|
|
38
|
-
# 1.
|
|
39
|
-
# the pipeline
|
|
40
|
-
#
|
|
41
|
-
#
|
|
42
|
-
#
|
|
43
|
-
#
|
|
45
|
+
# 1. It answered the wrong question. Disagreement is the one signal in
|
|
46
|
+
# the pipeline that points at a specific claim and says "the models do
|
|
47
|
+
# not concur here", which is what a verification pass wants as input,
|
|
48
|
+
# and a deduction spends it on a number instead.
|
|
49
|
+
# 2. It could send a claim into the loop that no revision could get out
|
|
50
|
+
# of. A partially grounded claim scores 0.60 (MEDIUM); at 1 of 3 the
|
|
51
|
+
# penalty took it to 0.50 (LOW), so D1 flagged it, 6a was never told
|
|
52
|
+
# why, and 6b re-scored it under the same penalty -- who asserted a
|
|
53
|
+
# claim does not change when it is reworded -- until D2 published it
|
|
54
|
+
# UNVERIFIED.
|
|
44
55
|
#
|
|
45
|
-
#
|
|
46
|
-
#
|
|
47
|
-
#
|
|
48
|
-
#
|
|
49
|
-
|
|
50
|
-
# byte-for-byte regression guard holds by construction.
|
|
51
|
-
DISAGREEMENT_PENALTY_FACTOR = 0.15
|
|
56
|
+
# AT MOST HALF, so 1 of 2, 1 of 3 and 2 of 4 are contested and 2 of 3 is
|
|
57
|
+
# not: exactly the shapes where the penalty used to change a tier. The
|
|
58
|
+
# non-ensemble formula is untouched, and the ensemble one is now the same
|
|
59
|
+
# formula, so §10's byte-for-byte regression guard holds for both.
|
|
60
|
+
CONTESTED_SHARE = 0.5
|
|
52
61
|
|
|
53
62
|
# ROADMAP_v2 §45 (SQ3). What a grounding is WORTH.
|
|
54
63
|
#
|
|
@@ -113,7 +122,6 @@ def default_weights() -> dict:
|
|
|
113
122
|
"grounding_weight_factor": GROUNDING_WEIGHT_FACTOR,
|
|
114
123
|
"critic_weight_factor": CRITIC_WEIGHT_FACTOR,
|
|
115
124
|
"assumption_flag_penalty": ASSUMPTION_FLAG_PENALTY,
|
|
116
|
-
"disagreement_penalty_factor": DISAGREEMENT_PENALTY_FACTOR,
|
|
117
125
|
"non_factual_score_cap": NON_FACTUAL_SCORE_CAP,
|
|
118
126
|
"source_quality_floor": SOURCE_QUALITY_FLOOR,
|
|
119
127
|
"authority_full_credit": AUTHORITY_FULL_CREDIT,
|
|
@@ -194,9 +202,10 @@ def score_claim(claim: Claim, ensemble_n: int = 0,
|
|
|
194
202
|
"""Returns (tier, score_breakdown). Pure function -- no side effects,
|
|
195
203
|
easy to unit test claim-by-claim without running any pass.
|
|
196
204
|
|
|
197
|
-
When ensemble_n > 0 (ROADMAP §10),
|
|
198
|
-
|
|
199
|
-
|
|
205
|
+
When ensemble_n > 0 (ROADMAP §10), the share of candidates that
|
|
206
|
+
asserted this claim is recorded, and a factual claim asserted by at
|
|
207
|
+
most CONTESTED_SHARE of them is marked `contested` for D1 to route
|
|
208
|
+
(ROADMAP_v3 §90). It changes no score. ensemble_n is the number of
|
|
200
209
|
candidates that actually produced text, which on a run where a
|
|
201
210
|
candidate's provider failed is fewer than the configured roster.
|
|
202
211
|
|
|
@@ -251,29 +260,25 @@ def score_claim(claim: Claim, ensemble_n: int = 0,
|
|
|
251
260
|
critic_component = 1.0 - claim.critic_severity
|
|
252
261
|
assumption_penalty = w["assumption_flag_penalty"] * len(claim.assumption_flags)
|
|
253
262
|
|
|
254
|
-
# None when ensemble mode is off
|
|
255
|
-
#
|
|
256
|
-
#
|
|
257
|
-
#
|
|
263
|
+
# None when ensemble mode is off -- and (B7) also when ensemble mode is
|
|
264
|
+
# ON and Pass 2 reported nothing for this claim, which is a different
|
|
265
|
+
# fact from "no candidate asserted it". An unreported claim is never
|
|
266
|
+
# contested.
|
|
258
267
|
consistency_score = None
|
|
259
268
|
consistency_reported = None
|
|
260
|
-
disagreement_penalty = 0.0
|
|
261
269
|
if ensemble_n > 0 and claim.asserted_by_candidates:
|
|
262
|
-
# B8. Deduplicated and clamped. `[1, 2, 3, 3]` made
|
|
263
|
-
#
|
|
264
|
-
# claim
|
|
265
|
-
#
|
|
266
|
-
#
|
|
267
|
-
#
|
|
270
|
+
# B8. Deduplicated and clamped. `[1, 2, 3, 3]` made consistency
|
|
271
|
+
# 1.3333, which under E8 made the penalty NEGATIVE and scored a
|
|
272
|
+
# MEDIUM claim HIGH; here a duplicate would make one candidate's
|
|
273
|
+
# claim read as two agreeing, and `[1, 1]` of 3 would escape
|
|
274
|
+
# routing. `consistency_reported` keeps the unclamped value when
|
|
275
|
+
# it fired, because a numerator above the denominator is itself
|
|
276
|
+
# evidence that Pass 2 misread the candidate labelling, and
|
|
268
277
|
# clamping silently would hide the evidence along with the bug.
|
|
269
278
|
reported = len(set(claim.asserted_by_candidates)) / ensemble_n
|
|
270
279
|
consistency_score = min(1.0, reported)
|
|
271
280
|
if reported > 1.0:
|
|
272
281
|
consistency_reported = round(reported, 4)
|
|
273
|
-
# E9: computed only where it is applied, so the breakdown records
|
|
274
|
-
# what was subtracted rather than what was available (B10).
|
|
275
|
-
if claim_type == "factual":
|
|
276
|
-
disagreement_penalty = w["disagreement_penalty_factor"] * (1.0 - consistency_score)
|
|
277
282
|
|
|
278
283
|
if claim_type != "factual":
|
|
279
284
|
# Cap applies to the critic component BEFORE subtracting the
|
|
@@ -283,11 +288,6 @@ def score_claim(claim: Claim, ensemble_n: int = 0,
|
|
|
283
288
|
# claim that was already clean under critique. Found this via a
|
|
284
289
|
# direct test case (two speculative claims, one flagged one not,
|
|
285
290
|
# both landing at the same tier) before it shipped.
|
|
286
|
-
#
|
|
287
|
-
# E9: no disagreement penalty here. §10 decided consistency does not
|
|
288
|
-
# RESCUE a non-factual claim from the cap; whether dissent should
|
|
289
|
-
# penalise a speculative claim is a separate judgment about what
|
|
290
|
-
# candidates declining to speculate means, and it was not made.
|
|
291
291
|
capped_critic = min(critic_component, w["non_factual_score_cap"])
|
|
292
292
|
raw_score = capped_critic - assumption_penalty
|
|
293
293
|
formula = "non_factual"
|
|
@@ -298,7 +298,6 @@ def score_claim(claim: Claim, ensemble_n: int = 0,
|
|
|
298
298
|
(w["grounding_weight_factor"] * grounding_component)
|
|
299
299
|
+ (w["critic_weight_factor"] * critic_component)
|
|
300
300
|
- assumption_penalty
|
|
301
|
-
- disagreement_penalty
|
|
302
301
|
)
|
|
303
302
|
|
|
304
303
|
# ROUND ONCE, then use that one value for BOTH the tier comparison and
|
|
@@ -309,8 +308,9 @@ def score_claim(claim: Claim, ensemble_n: int = 0,
|
|
|
309
308
|
# auditing a run artifact read a number that contradicted the published
|
|
310
309
|
# thresholds, and the tier was the one that mattered.
|
|
311
310
|
#
|
|
312
|
-
# Binary floating point
|
|
313
|
-
# from an exotic input: 0.5 + 0.35 - 0.15 * (1/3)
|
|
311
|
+
# Binary floating point made this reachable from ordinary weights, not
|
|
312
|
+
# from an exotic input: 0.5 + 0.35 - 0.15 * (1/3), E8's 2-of-3 penalty,
|
|
313
|
+
# was exactly that value.
|
|
314
314
|
# Sweeping the whole non-ensemble input space, exactly ONE shape changes
|
|
315
315
|
# tier under this fix -- a speculative claim at severity 0.55 with one
|
|
316
316
|
# assumption flag, which records 0.3 and now tiers LOW instead of
|
|
@@ -336,7 +336,7 @@ def score_claim(claim: Claim, ensemble_n: int = 0,
|
|
|
336
336
|
# extra key: "anyone auditing a run artifact read a number that
|
|
337
337
|
# contradicted the published thresholds."
|
|
338
338
|
#
|
|
339
|
-
# factual 0.5*grounding + 0.35*critic - assumption
|
|
339
|
+
# factual 0.5*grounding + 0.35*critic - assumption
|
|
340
340
|
# non_factual capped_critic - assumption
|
|
341
341
|
breakdown = {
|
|
342
342
|
"formula": formula,
|
|
@@ -371,11 +371,18 @@ def score_claim(claim: Claim, ensemble_n: int = 0,
|
|
|
371
371
|
# B7. None means PASS 2 REPORTED NOTHING, which is not 0.0 -- and
|
|
372
372
|
# the two were indistinguishable, so a claim one candidate raised
|
|
373
373
|
# uncontested took the maximum 0.15 penalty rather than the 0.05 it
|
|
374
|
-
# earned
|
|
375
|
-
# apart too.
|
|
374
|
+
# earned under E8. `contested` needs them apart for the same reason.
|
|
376
375
|
breakdown["consistency_score"] = (
|
|
377
376
|
None if consistency_score is None else round(consistency_score, 4))
|
|
378
|
-
|
|
377
|
+
# §90: what D1 routes on, and the record of why a claim that scored
|
|
378
|
+
# HIGH went through 6a/6b anyway. E9 LIVES HERE, and only here:
|
|
379
|
+
# whether candidates declining to speculate says anything about a
|
|
380
|
+
# speculative claim is a judgment §10 never made, so the non-factual
|
|
381
|
+
# branch is never asked and carries no key (B10: a field that did
|
|
382
|
+
# nothing is never present).
|
|
383
|
+
if formula == "factual":
|
|
384
|
+
breakdown["contested"] = (consistency_score is not None
|
|
385
|
+
and consistency_score <= CONTESTED_SHARE)
|
|
379
386
|
if consistency_reported is not None:
|
|
380
387
|
breakdown["consistency_reported"] = consistency_reported
|
|
381
388
|
if unrecognised_status is not None:
|
|
@@ -385,6 +392,14 @@ def score_claim(claim: Claim, ensemble_n: int = 0,
|
|
|
385
392
|
return tier, breakdown
|
|
386
393
|
|
|
387
394
|
|
|
395
|
+
def is_contested(claim: Claim) -> bool:
|
|
396
|
+
"""Whether Pass 5 marked this claim contested (ROADMAP_v3 §90).
|
|
397
|
+
|
|
398
|
+
Read off the breakdown rather than recomputed, so D1 routes on exactly
|
|
399
|
+
what the stored claim says and the type coercion above happens once."""
|
|
400
|
+
return claim.score_breakdown.get("contested") is True
|
|
401
|
+
|
|
402
|
+
|
|
388
403
|
def build_coverage_gap_entries(completeness: dict) -> list[dict]:
|
|
389
404
|
"""Gaps from Pass 3c become separate coverage-gap entries, not Claim
|
|
390
405
|
objects -- they were never asserted, so there's nothing to re-tier."""
|
|
@@ -398,8 +413,9 @@ def run_confidence_tiering(
|
|
|
398
413
|
weights: dict | None = None,
|
|
399
414
|
) -> None:
|
|
400
415
|
"""The actual Pass 5 entry point the orchestrator calls. Mutates
|
|
401
|
-
the claims in place; zero LLM calls. Pass ensemble_n > 0 to
|
|
402
|
-
the
|
|
416
|
+
the claims in place; zero LLM calls. Pass ensemble_n > 0 to record
|
|
417
|
+
each claim's consistency and mark the contested ones (ROADMAP §10,
|
|
418
|
+
ROADMAP_v3 §90); the formula is the same either way.
|
|
403
419
|
|
|
404
420
|
`claims` narrows WHICH claims are re-tiered, and defaults to all of
|
|
405
421
|
them -- which is Pass 5 itself, where every claim is in play. The
|
|
@@ -449,8 +465,8 @@ def _report_coercions(run: PipelineRun, claim: Claim, breakdown: dict,
|
|
|
449
465
|
if ensemble_n > 0 and breakdown.get("consistency_score") is None:
|
|
450
466
|
run.log(
|
|
451
467
|
f"Pass 5: claim {claim.id} carries no asserted_by_candidates on "
|
|
452
|
-
f"an ensemble run --
|
|
453
|
-
f"
|
|
468
|
+
f"an ensemble run -- not treated as contested, since an "
|
|
469
|
+
f"unreported numerator is not a numerator of zero."
|
|
454
470
|
)
|
|
455
471
|
if "unrecognised_grounding_status" in breakdown:
|
|
456
472
|
run.log(
|
|
@@ -44,7 +44,8 @@ status='failed' record.
|
|
|
44
44
|
from __future__ import annotations
|
|
45
45
|
|
|
46
46
|
import json
|
|
47
|
-
from
|
|
47
|
+
from collections.abc import Callable
|
|
48
|
+
from typing import Any, Optional
|
|
48
49
|
|
|
49
50
|
import config
|
|
50
51
|
from core.reasoning.payload_validation import PayloadShapeError
|
|
@@ -85,6 +86,7 @@ def retry_until_json(
|
|
|
85
86
|
max_total_tokens: Optional[int] = None,
|
|
86
87
|
validate: Optional[Callable] = None,
|
|
87
88
|
effort: Optional[str] = None,
|
|
89
|
+
activity=None,
|
|
88
90
|
) -> str:
|
|
89
91
|
"""Returns the first response text that parses as JSON, retrying by
|
|
90
92
|
continuing `response`'s thread.
|
|
@@ -121,6 +123,16 @@ def retry_until_json(
|
|
|
121
123
|
PayloadShapeError re-enters this same loop. None means
|
|
122
124
|
"parse is the only contract", which is what the reviewer
|
|
123
125
|
wants and what every caller had before §30.
|
|
126
|
+
activity §47's agent-activity sink, forwarded for the same reason
|
|
127
|
+
as everything above it: a retry is the same run
|
|
128
|
+
continuing. It matters MORE here than elsewhere, not
|
|
129
|
+
less -- `continue_conversation` drains its own loop, so
|
|
130
|
+
a subagent spawned while the model is being corrected
|
|
131
|
+
reaches a shell through this channel or through nothing
|
|
132
|
+
(`tui/app.py`'s one-shot path carries it for exactly
|
|
133
|
+
that reason). No span is opened here: the caller's own
|
|
134
|
+
brackets the whole attempt-and-retry sequence, and a row
|
|
135
|
+
per correction would report one run as several.
|
|
124
136
|
"""
|
|
125
137
|
from core.loop import RunAgentLoop
|
|
126
138
|
|
|
@@ -165,6 +177,12 @@ def retry_until_json(
|
|
|
165
177
|
# continuing, so it carries the pass's effort too --
|
|
166
178
|
# effort_for re-validates against this thread's model.
|
|
167
179
|
effort=effort,
|
|
180
|
+
# And the same rule again for §47's sink (batch 75). This
|
|
181
|
+
# hop dropped it, so a spawn inside a corrective attempt
|
|
182
|
+
# was invisible -- and this is the one path where the
|
|
183
|
+
# channel is the ONLY route, because continue_conversation
|
|
184
|
+
# drains its own loop.
|
|
185
|
+
activity=activity,
|
|
168
186
|
**({} if max_total_tokens is None
|
|
169
187
|
else {"max_total_tokens": max_total_tokens}),
|
|
170
188
|
)
|