@yahyahammad/venastine 0.3.8 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -19
- package/CONFIG_ARCHITECTURE.md +2030 -2002
- package/LICENSE +201 -201
- package/README.md +1034 -1034
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -106
- package/agents/builtin/review.md +74 -74
- package/agents/manager.py +261 -261
- package/agents/subagent_tool.py +472 -472
- package/agents/tui_commands.py +238 -146
- package/bin/venastine.mjs +528 -528
- package/bin/venastine.py +71 -71
- package/cli_commands.py +838 -835
- package/config.yaml +13 -0
- package/config_edit.py +1301 -1301
- package/config_schema.py +971 -966
- package/config_update.py +749 -749
- package/core/agent_activity.py +256 -256
- package/core/ansi.py +452 -452
- package/core/approval.py +137 -137
- package/core/call_view.py +364 -364
- package/core/cancel.py +121 -121
- package/core/chatgpt_auth.py +747 -747
- package/core/claims_view.py +107 -107
- package/core/client.py +2715 -2715
- package/core/commands.py +460 -454
- package/core/compaction.py +1735 -1735
- package/core/config_loader.py +1982 -1982
- package/core/connect.py +686 -686
- package/core/events.py +123 -123
- package/core/file_mentions.py +280 -280
- package/core/interaction.py +411 -411
- package/core/loop.py +2566 -2566
- package/core/memory.py +482 -482
- package/core/pipeline_models.py +190 -190
- package/core/provider_errors.py +234 -234
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -484
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/orchestrator.py +1561 -1561
- package/core/reasoning/output_writer.py +345 -345
- package/core/reasoning/payload_validation.py +524 -524
- package/core/reasoning/pipeline_storage.py +427 -427
- package/core/reasoning/scholar.py +458 -458
- package/core/reasoning/source_corpus.py +433 -433
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -315
- package/core/shell_sessions.py +1553 -1553
- package/credentials.py +307 -307
- package/database.py +333 -333
- package/docs/PRIVACY.md +78 -78
- package/docs/SECURITY.md +108 -108
- package/docs/THIRD_PARTY_NOTICES.md +148 -148
- package/env_secrets.py +118 -118
- package/json_store.py +343 -343
- package/logging_setup.py +295 -295
- package/main.py +3433 -3433
- package/mcp_client/client.py +733 -733
- package/mcp_client/config.py +484 -484
- package/mcp_client/registration.py +343 -343
- package/memories/manager.py +147 -147
- package/memories/tui_commands.py +319 -239
- package/package.json +303 -303
- package/project_init/config_files.py +282 -282
- package/project_init/generator.py +728 -728
- package/project_init/manifest.py +418 -418
- package/project_init/tui_commands.py +147 -147
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +17 -17
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -316
- package/providers.json.example +94 -94
- package/pyproject.toml +149 -149
- package/requirements.txt +295 -295
- package/safety/policy_enforcement.py +1082 -1082
- package/security/askpass.py +419 -419
- package/security/capability.py +207 -207
- package/security/confine.py +484 -484
- package/security/hosted.py +182 -182
- package/security/permissions.py +237 -237
- package/security/protected_paths.py +501 -501
- package/security/sandbox.py +4108 -4108
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -110
- package/storage.py +1338 -1338
- package/tools/base.py +465 -465
- package/tools/builtin/_math_common.py +667 -667
- package/tools/builtin/_net_common.py +267 -267
- package/tools/builtin/arxiv.py +681 -681
- package/tools/builtin/fetch_url.py +371 -371
- package/tools/builtin/file_ops.py +720 -720
- package/tools/builtin/geometry.py +184 -184
- package/tools/builtin/load_skill.py +76 -76
- package/tools/builtin/pin.py +227 -227
- package/tools/builtin/probability_stats.py +224 -224
- package/tools/builtin/project_docs.py +464 -464
- package/tools/builtin/remember.py +175 -175
- package/tools/builtin/shell.py +980 -980
- package/tools/builtin/todo.py +149 -149
- package/tools/builtin/web_search.py +328 -328
- package/tools/context.py +159 -159
- package/tools/isolation.py +387 -387
- package/tools/registry.py +1242 -1242
- package/tui/app.py +8318 -8307
- package/tui/app.tcss +747 -739
- package/tui/commands.py +26 -26
- package/tui/diffs.py +286 -286
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -102
- package/tui/markdown.py +1329 -1329
- package/tui/meters.py +364 -364
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -1197
- package/tui/themes.py +599 -599
- package/tui/widgets.py +3813 -3796
|
@@ -1,267 +1,267 @@
|
|
|
1
|
-
"""
|
|
2
|
-
tools/builtin/_net_common.py
|
|
3
|
-
|
|
4
|
-
What the two network-backed search tools share. `_math_common.py`'s
|
|
5
|
-
precedent, one directory over: a private module the built-in tools import
|
|
6
|
-
from, never registered and never advertised.
|
|
7
|
-
|
|
8
|
-
TWO THINGS NOW. The first is the in-process response cache: `web_search.py`
|
|
9
|
-
and `arxiv.py` each carried a `_cache_get` / `_cache_set` pair that was
|
|
10
|
-
identical apart from the type of the value being stored, plus a
|
|
11
|
-
`CACHE_TTL_S = 300` each, plus a bare dict each. Three copies of an expiry
|
|
12
|
-
rule is how one of them comes to be measured in minutes.
|
|
13
|
-
|
|
14
|
-
The second is `ThirdPartyText` (batch 156, ROADMAP_v3 §73), which is where
|
|
15
|
-
the two search tools now bound and redact the strings they did not write.
|
|
16
|
-
It is here rather than in either tool for the reason the cache is: both
|
|
17
|
-
were doing the same thing to the same kind of value, and the one that got
|
|
18
|
-
it wrong got it wrong in the half the other had already fixed.
|
|
19
|
-
|
|
20
|
-
There are THREE callers now -- `fetch_url.py` joined them in ROADMAP_v3 §51 --
|
|
21
|
-
and the third is why §52 gave this class a size bound as well as a time one.
|
|
22
|
-
|
|
23
|
-
WHAT IS DELIBERATELY *NOT* SHARED. Each tool keeps its OWN instance, so the
|
|
24
|
-
two key spaces stay separate -- a `TTLCache` shared between them would let
|
|
25
|
-
an arXiv query and a web query collide on an equal key string, which is a
|
|
26
|
-
correctness bug rather than a tidiness one. Their `@field_validator`s
|
|
27
|
-
(`strip_keywords`, `strip_query`) are also left alone: they are three lines
|
|
28
|
-
each, they differ in the message the model is shown, and folding them into
|
|
29
|
-
a factory would make the params class harder to read to save two lines.
|
|
30
|
-
"""
|
|
31
|
-
|
|
32
|
-
from __future__ import annotations
|
|
33
|
-
|
|
34
|
-
import logging
|
|
35
|
-
import threading
|
|
36
|
-
import time
|
|
37
|
-
from typing import Any, Optional
|
|
38
|
-
|
|
39
|
-
from safety.policy_enforcement import redact_output_text
|
|
40
|
-
|
|
41
|
-
logger = logging.getLogger(__name__)
|
|
42
|
-
|
|
43
|
-
# ROADMAP_v3 §52 (RB6). How many entries any one of these may hold.
|
|
44
|
-
#
|
|
45
|
-
# Derived rather than picked: `config.MAX_ITERATIONS` is 50, so one pass can
|
|
46
|
-
# make at most 50 tool calls, and a `fetch_url` whose URL redirects stores
|
|
47
|
-
# TWO keys for one body (the requested spelling and the answering one). 100
|
|
48
|
-
# keys is therefore the most a single pass can reach, and this sits above it
|
|
49
|
-
# so that the cap bounds a PROCESS -- which runs many passes and many runs --
|
|
50
|
-
# without ever biting inside one.
|
|
51
|
-
#
|
|
52
|
-
# That "without ever biting" is the requirement, not a nicety. A `fetch_url`
|
|
53
|
-
# page-2 request that misses degrades to a re-fetch, which is exactly the
|
|
54
|
-
# spliced-document failure §51 measured and built PG5 to prevent, so an
|
|
55
|
-
# eviction that fires in ordinary use would trade a bounded memory cost for
|
|
56
|
-
# an unbounded correctness one.
|
|
57
|
-
#
|
|
58
|
-
# In entries and not in bytes, because a generic cache cannot measure the
|
|
59
|
-
# size of what a caller hands it. For the caller that made this necessary the
|
|
60
|
-
# two are interchangeable anyway: `fetch_url`'s MAX_CONTENT_BYTES holds each
|
|
61
|
-
# body to ~64 KB whatever the encoding, so PG6's refusal to raise that cap is
|
|
62
|
-
# what turns this count into a memory bound at all.
|
|
63
|
-
MAX_ENTRIES = 128
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
# ROADMAP_v3 §73 (A58). How long a URL this harness is willing to SHOW the
|
|
67
|
-
# model may be.
|
|
68
|
-
#
|
|
69
|
-
# DERIVED, NOT PICKED, and the derivation is the argument for dropping an
|
|
70
|
-
# over-long URL rather than cutting it: `fetch_url` already bounds the URL
|
|
71
|
-
# the model may ASK for, at `max_length=400` on `FetchURLParams.url`.
|
|
72
|
-
# Measured (batch 156), a 401-character URL comes back from fetch_url's own
|
|
73
|
-
# gate as an "Invalid arguments" refusal while a 400-character one is
|
|
74
|
-
# accepted. So a longer URL is one THIS HARNESS refuses to take, and
|
|
75
|
-
# offering it in a search result is offering a route to nowhere.
|
|
76
|
-
#
|
|
77
|
-
# ONE DEFINITION: `fetch_url` reads it from here rather than spelling 400
|
|
78
|
-
# again. The two were the same number in two files, which is the state a
|
|
79
|
-
# pair of numbers is in just before it stops being a pair.
|
|
80
|
-
MAX_URL_CHARS = 400
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
class ThirdPartyText:
|
|
84
|
-
"""One tool call's worth of strings nobody here wrote, REDACTED and
|
|
85
|
-
THEN cut.
|
|
86
|
-
|
|
87
|
-
ORDER IS THE WHOLE POINT, and it is RB1's point restated for the two
|
|
88
|
-
tools RB1 did not reach. `fetch_url` redacts its whole body above both
|
|
89
|
-
the hold and the slice, because a pattern only matches what it can see
|
|
90
|
-
whole; `web_search` and `arxiv_search` cut first and let
|
|
91
|
-
`registry.dispatch`'s `check_output_policy` redact the piece that was
|
|
92
|
-
kept. A fixed-length credential straddling such a cut is then invisible
|
|
93
|
-
to both halves -- the kept prefix is not the token and the dropped
|
|
94
|
-
suffix is gone. Measured (batch 156) at all three of their cut points:
|
|
95
|
-
a `ghp_` token survived post-hoc redaction at 36 of the 39 offsets it
|
|
96
|
-
could straddle, and redacting the whole value first caught every one.
|
|
97
|
-
|
|
98
|
-
TWO KINDS OF VALUE, because a cut means different things to them (A58).
|
|
99
|
-
`text()` cuts and flags: half an abstract is still an abstract, and PG4
|
|
100
|
-
says the flag is owed. `url()` DROPS and flags: half a URL is not a
|
|
101
|
-
URL, it is a fetchable-looking string that is not the resource, and
|
|
102
|
-
`pdf_url` and arXiv's `full_*_url` pair exist precisely to be the route
|
|
103
|
-
the model takes to recover what was cut. A broken route offered as a
|
|
104
|
-
route is worse than an absent one, which the flag makes visible.
|
|
105
|
-
|
|
106
|
-
TOTAL OVER WHATEVER THE PROVIDER EMITTED, for the reason `tools/base.py`
|
|
107
|
-
gives about tool arguments: a key that is present with a wrong-typed
|
|
108
|
-
value is not an absent key, and `r.get("title", "(untitled)")` returns
|
|
109
|
-
None for `{"title": null}` rather than the default. Both accessors
|
|
110
|
-
answer None for a non-string instead of raising, and the caller decides
|
|
111
|
-
what an absent value means -- which is a decision the two tools make
|
|
112
|
-
differently and should (A61).
|
|
113
|
-
|
|
114
|
-
ONE WARNING PER CALL, not per value (#49, RB2). Redacting here means
|
|
115
|
-
`check_output_policy` finds nothing left to alter, so the obligation to
|
|
116
|
-
say that something WAS altered moves here with the redaction; and a
|
|
117
|
-
feed with twenty-five authors must not produce twenty-five warning
|
|
118
|
-
lines. The count is carried on the instance and `report()` spends it.
|
|
119
|
-
The match is never echoed, for the reason `check_output_policy` and
|
|
120
|
-
`fetch_url._redacted` both state: this function exists because
|
|
121
|
-
something credential-shaped was found, so quoting it to say so would
|
|
122
|
-
put it in app.log and every sink a WARNING reaches.
|
|
123
|
-
"""
|
|
124
|
-
|
|
125
|
-
def __init__(self, tool: str) -> None:
|
|
126
|
-
self._tool = tool
|
|
127
|
-
self._redacted = 0
|
|
128
|
-
|
|
129
|
-
def text(self, value: Any, cap: int) -> tuple:
|
|
130
|
-
"""`(text, truncated)` -- `value` redacted and THEN cut to `cap`.
|
|
131
|
-
|
|
132
|
-
`(None, False)` for anything that is not a string, so a caller can
|
|
133
|
-
tell "the provider did not give me this" from "it gave me an empty
|
|
134
|
-
one" and choose its own substitute.
|
|
135
|
-
|
|
136
|
-
THE FLAG COUNTS REDACTED CHARACTERS, which is RB3's consequence
|
|
137
|
-
arriving here: redaction can SHRINK a value past the cap, and when
|
|
138
|
-
it does there is nothing further the cut is withholding. A flag
|
|
139
|
-
measured on the raw length would claim there is.
|
|
140
|
-
"""
|
|
141
|
-
if not isinstance(value, str):
|
|
142
|
-
return None, False
|
|
143
|
-
clean = self._clean(value)
|
|
144
|
-
return clean[:cap], len(clean) > cap
|
|
145
|
-
|
|
146
|
-
def url(self, value: Any, cap: int = MAX_URL_CHARS) -> tuple:
|
|
147
|
-
"""`(url, withheld)` -- a URL past `cap` is DROPPED, not cut (A58).
|
|
148
|
-
|
|
149
|
-
`(None, False)` for a non-string or an empty one: nothing was
|
|
150
|
-
withheld, there was nothing there. `(None, True)` says a URL was
|
|
151
|
-
found and is not being shown, which is the fact the flag carries.
|
|
152
|
-
"""
|
|
153
|
-
if not isinstance(value, str) or not value:
|
|
154
|
-
return None, False
|
|
155
|
-
clean = self._clean(value)
|
|
156
|
-
if len(clean) > cap:
|
|
157
|
-
return None, True
|
|
158
|
-
return clean, False
|
|
159
|
-
|
|
160
|
-
def _clean(self, value: str) -> str:
|
|
161
|
-
clean = redact_output_text(value)
|
|
162
|
-
if clean != value:
|
|
163
|
-
self._redacted += 1
|
|
164
|
-
return clean
|
|
165
|
-
|
|
166
|
-
def report(self) -> None:
|
|
167
|
-
"""The one warning this call owes (#49), or silence."""
|
|
168
|
-
if self._redacted:
|
|
169
|
-
logger.warning(
|
|
170
|
-
"Output policy altered %s result: credentials replaced in "
|
|
171
|
-
"%d value(s) before they were cut.",
|
|
172
|
-
self._tool, self._redacted)
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
class TTLCache:
|
|
176
|
-
"""A tiny time-boxed cache, keyed by whatever string a caller builds.
|
|
177
|
-
|
|
178
|
-
TWO BOUNDS, ANSWERING DIFFERENT QUESTIONS. The TTL is about FRESHNESS: a
|
|
179
|
-
stale entry is dropped as it is read, which is what makes a repeated
|
|
180
|
-
query after the TTL a real request again rather than a permanent miss
|
|
181
|
-
against a growing dict. `max_entries` is about SIZE, and it is the only
|
|
182
|
-
thing that bounds a cache nothing reads a second time.
|
|
183
|
-
|
|
184
|
-
THE SWEEP ON WRITE IS §52 (RB5), AND THE REASON THIS DOCSTRING USED TO
|
|
185
|
-
GIVE FOR NOT HAVING ONE IS NOW FALSE. It said a sweep "would be machinery
|
|
186
|
-
for a problem neither tool has" -- true of two callers holding result
|
|
187
|
-
lists, and not of the third, which holds bodies. Measured (batch 106):
|
|
188
|
-
100 fetches of a capped page held 6.3 MB and not one entry was ever
|
|
189
|
-
dropped, because expiry ran only in `get` and nothing read those keys
|
|
190
|
-
again. Expiring on read alone means the TTL bounds staleness and nothing
|
|
191
|
-
whatever bounds memory.
|
|
192
|
-
|
|
193
|
-
READ IS DELIBERATELY UNCHANGED. `get` still expires the one key it
|
|
194
|
-
touches, because that path is what makes a post-TTL query a real request
|
|
195
|
-
again; the sweep is additional, not a replacement, and a cache that only
|
|
196
|
-
swept would keep serving a stale entry until someone else wrote.
|
|
197
|
-
"""
|
|
198
|
-
|
|
199
|
-
def __init__(self, ttl_s: float, max_entries: int = MAX_ENTRIES) -> None:
|
|
200
|
-
self.ttl_s = ttl_s
|
|
201
|
-
self.max_entries = max_entries
|
|
202
|
-
self._entries: dict[str, tuple[float, Any]] = {}
|
|
203
|
-
# LOCKED BECAUSE THE REGISTRY ALREADY SAYS IT IS (batch 117).
|
|
204
|
-
# `spawn_subagent` is the one tool with `parallel=True`, and the
|
|
205
|
-
# comment beside that field claims "everything the child's run
|
|
206
|
-
# touches was made concurrency-safe first -- the budget's lock, the
|
|
207
|
-
# serialised question, the per-future authorization". The child
|
|
208
|
-
# reaches this cache through `arxiv_search` and `fetch_url`, and it
|
|
209
|
-
# was the one item on that list without a lock.
|
|
210
|
-
#
|
|
211
|
-
# Single dict operations are atomic under CPython, so a corrupt
|
|
212
|
-
# dict was never the risk. `set` is a read-modify-write -- pop,
|
|
213
|
-
# assign, sweep, then evict while over the cap -- so two children
|
|
214
|
-
# storing at once could lose one entry and evict in an order
|
|
215
|
-
# neither asked for. Mirrors `GrantBudget.take()`, which is the
|
|
216
|
-
# pattern this module should have followed from the start.
|
|
217
|
-
#
|
|
218
|
-
# NOT reentrant, and `_sweep` is the reason: it is called from
|
|
219
|
-
# `set` with the lock already held, so it must never take it.
|
|
220
|
-
self._lock = threading.Lock()
|
|
221
|
-
|
|
222
|
-
def get(self, key: str) -> Optional[Any]:
|
|
223
|
-
with self._lock:
|
|
224
|
-
entry = self._entries.get(key)
|
|
225
|
-
if not entry:
|
|
226
|
-
return None
|
|
227
|
-
stored_at, value = entry
|
|
228
|
-
if time.time() - stored_at > self.ttl_s:
|
|
229
|
-
self._entries.pop(key, None)
|
|
230
|
-
return None
|
|
231
|
-
return value
|
|
232
|
-
|
|
233
|
-
def _sweep(self, now: float) -> None:
|
|
234
|
-
"""Drop everything already expired. Write-side only (§52, RB5).
|
|
235
|
-
|
|
236
|
-
CALLED WITH `self._lock` HELD, and it must not take it: the lock is
|
|
237
|
-
a plain `Lock`, so re-entering would deadlock the caller.
|
|
238
|
-
"""
|
|
239
|
-
stale = [key for key, (stored_at, _) in self._entries.items()
|
|
240
|
-
if now - stored_at > self.ttl_s]
|
|
241
|
-
for key in stale:
|
|
242
|
-
del self._entries[key]
|
|
243
|
-
|
|
244
|
-
def set(self, key: str, value: Any) -> None:
|
|
245
|
-
with self._lock:
|
|
246
|
-
self._set_locked(key, value)
|
|
247
|
-
|
|
248
|
-
def _set_locked(self, key: str, value: Any) -> None:
|
|
249
|
-
now = time.time()
|
|
250
|
-
# POPPED FIRST so a re-store moves the key to the END of the dict.
|
|
251
|
-
# Assigning to an existing key keeps its original insertion position,
|
|
252
|
-
# which would make the freshest entry the next one evicted -- and the
|
|
253
|
-
# freshest entry is the one a caller is most likely to read next.
|
|
254
|
-
self._entries.pop(key, None)
|
|
255
|
-
self._entries[key] = (now, value)
|
|
256
|
-
self._sweep(now)
|
|
257
|
-
# Insertion order IS age order, since every entry is stamped as it is
|
|
258
|
-
# written and the TTL is uniform. So the oldest key is simply the
|
|
259
|
-
# first one, and no second ordering has to be kept consistent.
|
|
260
|
-
while len(self._entries) > self.max_entries:
|
|
261
|
-
self._entries.pop(next(iter(self._entries)))
|
|
262
|
-
|
|
263
|
-
def clear(self) -> None:
|
|
264
|
-
"""Drop everything. For tests, which must not inherit a previous
|
|
265
|
-
test's answers -- the same reason conftest isolates the stores."""
|
|
266
|
-
with self._lock:
|
|
267
|
-
self._entries.clear()
|
|
1
|
+
"""
|
|
2
|
+
tools/builtin/_net_common.py
|
|
3
|
+
|
|
4
|
+
What the two network-backed search tools share. `_math_common.py`'s
|
|
5
|
+
precedent, one directory over: a private module the built-in tools import
|
|
6
|
+
from, never registered and never advertised.
|
|
7
|
+
|
|
8
|
+
TWO THINGS NOW. The first is the in-process response cache: `web_search.py`
|
|
9
|
+
and `arxiv.py` each carried a `_cache_get` / `_cache_set` pair that was
|
|
10
|
+
identical apart from the type of the value being stored, plus a
|
|
11
|
+
`CACHE_TTL_S = 300` each, plus a bare dict each. Three copies of an expiry
|
|
12
|
+
rule is how one of them comes to be measured in minutes.
|
|
13
|
+
|
|
14
|
+
The second is `ThirdPartyText` (batch 156, ROADMAP_v3 §73), which is where
|
|
15
|
+
the two search tools now bound and redact the strings they did not write.
|
|
16
|
+
It is here rather than in either tool for the reason the cache is: both
|
|
17
|
+
were doing the same thing to the same kind of value, and the one that got
|
|
18
|
+
it wrong got it wrong in the half the other had already fixed.
|
|
19
|
+
|
|
20
|
+
There are THREE callers now -- `fetch_url.py` joined them in ROADMAP_v3 §51 --
|
|
21
|
+
and the third is why §52 gave this class a size bound as well as a time one.
|
|
22
|
+
|
|
23
|
+
WHAT IS DELIBERATELY *NOT* SHARED. Each tool keeps its OWN instance, so the
|
|
24
|
+
two key spaces stay separate -- a `TTLCache` shared between them would let
|
|
25
|
+
an arXiv query and a web query collide on an equal key string, which is a
|
|
26
|
+
correctness bug rather than a tidiness one. Their `@field_validator`s
|
|
27
|
+
(`strip_keywords`, `strip_query`) are also left alone: they are three lines
|
|
28
|
+
each, they differ in the message the model is shown, and folding them into
|
|
29
|
+
a factory would make the params class harder to read to save two lines.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import logging
|
|
35
|
+
import threading
|
|
36
|
+
import time
|
|
37
|
+
from typing import Any, Optional
|
|
38
|
+
|
|
39
|
+
from safety.policy_enforcement import redact_output_text
|
|
40
|
+
|
|
41
|
+
logger = logging.getLogger(__name__)
|
|
42
|
+
|
|
43
|
+
# ROADMAP_v3 §52 (RB6). How many entries any one of these may hold.
|
|
44
|
+
#
|
|
45
|
+
# Derived rather than picked: `config.MAX_ITERATIONS` is 50, so one pass can
|
|
46
|
+
# make at most 50 tool calls, and a `fetch_url` whose URL redirects stores
|
|
47
|
+
# TWO keys for one body (the requested spelling and the answering one). 100
|
|
48
|
+
# keys is therefore the most a single pass can reach, and this sits above it
|
|
49
|
+
# so that the cap bounds a PROCESS -- which runs many passes and many runs --
|
|
50
|
+
# without ever biting inside one.
|
|
51
|
+
#
|
|
52
|
+
# That "without ever biting" is the requirement, not a nicety. A `fetch_url`
|
|
53
|
+
# page-2 request that misses degrades to a re-fetch, which is exactly the
|
|
54
|
+
# spliced-document failure §51 measured and built PG5 to prevent, so an
|
|
55
|
+
# eviction that fires in ordinary use would trade a bounded memory cost for
|
|
56
|
+
# an unbounded correctness one.
|
|
57
|
+
#
|
|
58
|
+
# In entries and not in bytes, because a generic cache cannot measure the
|
|
59
|
+
# size of what a caller hands it. For the caller that made this necessary the
|
|
60
|
+
# two are interchangeable anyway: `fetch_url`'s MAX_CONTENT_BYTES holds each
|
|
61
|
+
# body to ~64 KB whatever the encoding, so PG6's refusal to raise that cap is
|
|
62
|
+
# what turns this count into a memory bound at all.
|
|
63
|
+
MAX_ENTRIES = 128
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
# ROADMAP_v3 §73 (A58). How long a URL this harness is willing to SHOW the
|
|
67
|
+
# model may be.
|
|
68
|
+
#
|
|
69
|
+
# DERIVED, NOT PICKED, and the derivation is the argument for dropping an
|
|
70
|
+
# over-long URL rather than cutting it: `fetch_url` already bounds the URL
|
|
71
|
+
# the model may ASK for, at `max_length=400` on `FetchURLParams.url`.
|
|
72
|
+
# Measured (batch 156), a 401-character URL comes back from fetch_url's own
|
|
73
|
+
# gate as an "Invalid arguments" refusal while a 400-character one is
|
|
74
|
+
# accepted. So a longer URL is one THIS HARNESS refuses to take, and
|
|
75
|
+
# offering it in a search result is offering a route to nowhere.
|
|
76
|
+
#
|
|
77
|
+
# ONE DEFINITION: `fetch_url` reads it from here rather than spelling 400
|
|
78
|
+
# again. The two were the same number in two files, which is the state a
|
|
79
|
+
# pair of numbers is in just before it stops being a pair.
|
|
80
|
+
MAX_URL_CHARS = 400
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class ThirdPartyText:
|
|
84
|
+
"""One tool call's worth of strings nobody here wrote, REDACTED and
|
|
85
|
+
THEN cut.
|
|
86
|
+
|
|
87
|
+
ORDER IS THE WHOLE POINT, and it is RB1's point restated for the two
|
|
88
|
+
tools RB1 did not reach. `fetch_url` redacts its whole body above both
|
|
89
|
+
the hold and the slice, because a pattern only matches what it can see
|
|
90
|
+
whole; `web_search` and `arxiv_search` cut first and let
|
|
91
|
+
`registry.dispatch`'s `check_output_policy` redact the piece that was
|
|
92
|
+
kept. A fixed-length credential straddling such a cut is then invisible
|
|
93
|
+
to both halves -- the kept prefix is not the token and the dropped
|
|
94
|
+
suffix is gone. Measured (batch 156) at all three of their cut points:
|
|
95
|
+
a `ghp_` token survived post-hoc redaction at 36 of the 39 offsets it
|
|
96
|
+
could straddle, and redacting the whole value first caught every one.
|
|
97
|
+
|
|
98
|
+
TWO KINDS OF VALUE, because a cut means different things to them (A58).
|
|
99
|
+
`text()` cuts and flags: half an abstract is still an abstract, and PG4
|
|
100
|
+
says the flag is owed. `url()` DROPS and flags: half a URL is not a
|
|
101
|
+
URL, it is a fetchable-looking string that is not the resource, and
|
|
102
|
+
`pdf_url` and arXiv's `full_*_url` pair exist precisely to be the route
|
|
103
|
+
the model takes to recover what was cut. A broken route offered as a
|
|
104
|
+
route is worse than an absent one, which the flag makes visible.
|
|
105
|
+
|
|
106
|
+
TOTAL OVER WHATEVER THE PROVIDER EMITTED, for the reason `tools/base.py`
|
|
107
|
+
gives about tool arguments: a key that is present with a wrong-typed
|
|
108
|
+
value is not an absent key, and `r.get("title", "(untitled)")` returns
|
|
109
|
+
None for `{"title": null}` rather than the default. Both accessors
|
|
110
|
+
answer None for a non-string instead of raising, and the caller decides
|
|
111
|
+
what an absent value means -- which is a decision the two tools make
|
|
112
|
+
differently and should (A61).
|
|
113
|
+
|
|
114
|
+
ONE WARNING PER CALL, not per value (#49, RB2). Redacting here means
|
|
115
|
+
`check_output_policy` finds nothing left to alter, so the obligation to
|
|
116
|
+
say that something WAS altered moves here with the redaction; and a
|
|
117
|
+
feed with twenty-five authors must not produce twenty-five warning
|
|
118
|
+
lines. The count is carried on the instance and `report()` spends it.
|
|
119
|
+
The match is never echoed, for the reason `check_output_policy` and
|
|
120
|
+
`fetch_url._redacted` both state: this function exists because
|
|
121
|
+
something credential-shaped was found, so quoting it to say so would
|
|
122
|
+
put it in app.log and every sink a WARNING reaches.
|
|
123
|
+
"""
|
|
124
|
+
|
|
125
|
+
def __init__(self, tool: str) -> None:
|
|
126
|
+
self._tool = tool
|
|
127
|
+
self._redacted = 0
|
|
128
|
+
|
|
129
|
+
def text(self, value: Any, cap: int) -> tuple:
|
|
130
|
+
"""`(text, truncated)` -- `value` redacted and THEN cut to `cap`.
|
|
131
|
+
|
|
132
|
+
`(None, False)` for anything that is not a string, so a caller can
|
|
133
|
+
tell "the provider did not give me this" from "it gave me an empty
|
|
134
|
+
one" and choose its own substitute.
|
|
135
|
+
|
|
136
|
+
THE FLAG COUNTS REDACTED CHARACTERS, which is RB3's consequence
|
|
137
|
+
arriving here: redaction can SHRINK a value past the cap, and when
|
|
138
|
+
it does there is nothing further the cut is withholding. A flag
|
|
139
|
+
measured on the raw length would claim there is.
|
|
140
|
+
"""
|
|
141
|
+
if not isinstance(value, str):
|
|
142
|
+
return None, False
|
|
143
|
+
clean = self._clean(value)
|
|
144
|
+
return clean[:cap], len(clean) > cap
|
|
145
|
+
|
|
146
|
+
def url(self, value: Any, cap: int = MAX_URL_CHARS) -> tuple:
|
|
147
|
+
"""`(url, withheld)` -- a URL past `cap` is DROPPED, not cut (A58).
|
|
148
|
+
|
|
149
|
+
`(None, False)` for a non-string or an empty one: nothing was
|
|
150
|
+
withheld, there was nothing there. `(None, True)` says a URL was
|
|
151
|
+
found and is not being shown, which is the fact the flag carries.
|
|
152
|
+
"""
|
|
153
|
+
if not isinstance(value, str) or not value:
|
|
154
|
+
return None, False
|
|
155
|
+
clean = self._clean(value)
|
|
156
|
+
if len(clean) > cap:
|
|
157
|
+
return None, True
|
|
158
|
+
return clean, False
|
|
159
|
+
|
|
160
|
+
def _clean(self, value: str) -> str:
|
|
161
|
+
clean = redact_output_text(value)
|
|
162
|
+
if clean != value:
|
|
163
|
+
self._redacted += 1
|
|
164
|
+
return clean
|
|
165
|
+
|
|
166
|
+
def report(self) -> None:
|
|
167
|
+
"""The one warning this call owes (#49), or silence."""
|
|
168
|
+
if self._redacted:
|
|
169
|
+
logger.warning(
|
|
170
|
+
"Output policy altered %s result: credentials replaced in "
|
|
171
|
+
"%d value(s) before they were cut.",
|
|
172
|
+
self._tool, self._redacted)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
class TTLCache:
|
|
176
|
+
"""A tiny time-boxed cache, keyed by whatever string a caller builds.
|
|
177
|
+
|
|
178
|
+
TWO BOUNDS, ANSWERING DIFFERENT QUESTIONS. The TTL is about FRESHNESS: a
|
|
179
|
+
stale entry is dropped as it is read, which is what makes a repeated
|
|
180
|
+
query after the TTL a real request again rather than a permanent miss
|
|
181
|
+
against a growing dict. `max_entries` is about SIZE, and it is the only
|
|
182
|
+
thing that bounds a cache nothing reads a second time.
|
|
183
|
+
|
|
184
|
+
THE SWEEP ON WRITE IS §52 (RB5), AND THE REASON THIS DOCSTRING USED TO
|
|
185
|
+
GIVE FOR NOT HAVING ONE IS NOW FALSE. It said a sweep "would be machinery
|
|
186
|
+
for a problem neither tool has" -- true of two callers holding result
|
|
187
|
+
lists, and not of the third, which holds bodies. Measured (batch 106):
|
|
188
|
+
100 fetches of a capped page held 6.3 MB and not one entry was ever
|
|
189
|
+
dropped, because expiry ran only in `get` and nothing read those keys
|
|
190
|
+
again. Expiring on read alone means the TTL bounds staleness and nothing
|
|
191
|
+
whatever bounds memory.
|
|
192
|
+
|
|
193
|
+
READ IS DELIBERATELY UNCHANGED. `get` still expires the one key it
|
|
194
|
+
touches, because that path is what makes a post-TTL query a real request
|
|
195
|
+
again; the sweep is additional, not a replacement, and a cache that only
|
|
196
|
+
swept would keep serving a stale entry until someone else wrote.
|
|
197
|
+
"""
|
|
198
|
+
|
|
199
|
+
def __init__(self, ttl_s: float, max_entries: int = MAX_ENTRIES) -> None:
|
|
200
|
+
self.ttl_s = ttl_s
|
|
201
|
+
self.max_entries = max_entries
|
|
202
|
+
self._entries: dict[str, tuple[float, Any]] = {}
|
|
203
|
+
# LOCKED BECAUSE THE REGISTRY ALREADY SAYS IT IS (batch 117).
|
|
204
|
+
# `spawn_subagent` is the one tool with `parallel=True`, and the
|
|
205
|
+
# comment beside that field claims "everything the child's run
|
|
206
|
+
# touches was made concurrency-safe first -- the budget's lock, the
|
|
207
|
+
# serialised question, the per-future authorization". The child
|
|
208
|
+
# reaches this cache through `arxiv_search` and `fetch_url`, and it
|
|
209
|
+
# was the one item on that list without a lock.
|
|
210
|
+
#
|
|
211
|
+
# Single dict operations are atomic under CPython, so a corrupt
|
|
212
|
+
# dict was never the risk. `set` is a read-modify-write -- pop,
|
|
213
|
+
# assign, sweep, then evict while over the cap -- so two children
|
|
214
|
+
# storing at once could lose one entry and evict in an order
|
|
215
|
+
# neither asked for. Mirrors `GrantBudget.take()`, which is the
|
|
216
|
+
# pattern this module should have followed from the start.
|
|
217
|
+
#
|
|
218
|
+
# NOT reentrant, and `_sweep` is the reason: it is called from
|
|
219
|
+
# `set` with the lock already held, so it must never take it.
|
|
220
|
+
self._lock = threading.Lock()
|
|
221
|
+
|
|
222
|
+
def get(self, key: str) -> Optional[Any]:
|
|
223
|
+
with self._lock:
|
|
224
|
+
entry = self._entries.get(key)
|
|
225
|
+
if not entry:
|
|
226
|
+
return None
|
|
227
|
+
stored_at, value = entry
|
|
228
|
+
if time.time() - stored_at > self.ttl_s:
|
|
229
|
+
self._entries.pop(key, None)
|
|
230
|
+
return None
|
|
231
|
+
return value
|
|
232
|
+
|
|
233
|
+
def _sweep(self, now: float) -> None:
|
|
234
|
+
"""Drop everything already expired. Write-side only (§52, RB5).
|
|
235
|
+
|
|
236
|
+
CALLED WITH `self._lock` HELD, and it must not take it: the lock is
|
|
237
|
+
a plain `Lock`, so re-entering would deadlock the caller.
|
|
238
|
+
"""
|
|
239
|
+
stale = [key for key, (stored_at, _) in self._entries.items()
|
|
240
|
+
if now - stored_at > self.ttl_s]
|
|
241
|
+
for key in stale:
|
|
242
|
+
del self._entries[key]
|
|
243
|
+
|
|
244
|
+
def set(self, key: str, value: Any) -> None:
|
|
245
|
+
with self._lock:
|
|
246
|
+
self._set_locked(key, value)
|
|
247
|
+
|
|
248
|
+
def _set_locked(self, key: str, value: Any) -> None:
|
|
249
|
+
now = time.time()
|
|
250
|
+
# POPPED FIRST so a re-store moves the key to the END of the dict.
|
|
251
|
+
# Assigning to an existing key keeps its original insertion position,
|
|
252
|
+
# which would make the freshest entry the next one evicted -- and the
|
|
253
|
+
# freshest entry is the one a caller is most likely to read next.
|
|
254
|
+
self._entries.pop(key, None)
|
|
255
|
+
self._entries[key] = (now, value)
|
|
256
|
+
self._sweep(now)
|
|
257
|
+
# Insertion order IS age order, since every entry is stamped as it is
|
|
258
|
+
# written and the TTL is uniform. So the oldest key is simply the
|
|
259
|
+
# first one, and no second ordering has to be kept consistent.
|
|
260
|
+
while len(self._entries) > self.max_entries:
|
|
261
|
+
self._entries.pop(next(iter(self._entries)))
|
|
262
|
+
|
|
263
|
+
def clear(self) -> None:
|
|
264
|
+
"""Drop everything. For tests, which must not inherit a previous
|
|
265
|
+
test's answers -- the same reason conftest isolates the stores."""
|
|
266
|
+
with self._lock:
|
|
267
|
+
self._entries.clear()
|