@yahyahammad/venastine 0.3.8 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/.env.example +19 -19
  2. package/CONFIG_ARCHITECTURE.md +2030 -2002
  3. package/LICENSE +201 -201
  4. package/README.md +1034 -1034
  5. package/agents/builtin/compactor.md +75 -75
  6. package/agents/builtin/explore.md +77 -77
  7. package/agents/builtin/pipeline-reviewer.md +100 -100
  8. package/agents/builtin/plan.md +106 -106
  9. package/agents/builtin/review.md +74 -74
  10. package/agents/manager.py +261 -261
  11. package/agents/subagent_tool.py +472 -472
  12. package/agents/tui_commands.py +238 -146
  13. package/bin/venastine.mjs +528 -528
  14. package/bin/venastine.py +71 -71
  15. package/cli_commands.py +838 -835
  16. package/config.yaml +13 -0
  17. package/config_edit.py +1301 -1301
  18. package/config_schema.py +971 -966
  19. package/config_update.py +749 -749
  20. package/core/agent_activity.py +256 -256
  21. package/core/ansi.py +452 -452
  22. package/core/approval.py +137 -137
  23. package/core/call_view.py +364 -364
  24. package/core/cancel.py +121 -121
  25. package/core/chatgpt_auth.py +747 -747
  26. package/core/claims_view.py +107 -107
  27. package/core/client.py +2715 -2715
  28. package/core/commands.py +460 -454
  29. package/core/compaction.py +1735 -1735
  30. package/core/config_loader.py +1982 -1982
  31. package/core/connect.py +686 -686
  32. package/core/events.py +123 -123
  33. package/core/file_mentions.py +280 -280
  34. package/core/interaction.py +411 -411
  35. package/core/loop.py +2566 -2566
  36. package/core/memory.py +482 -482
  37. package/core/pipeline_models.py +190 -190
  38. package/core/provider_errors.py +234 -234
  39. package/core/reasoning/authorization.py +130 -130
  40. package/core/reasoning/base.py +219 -219
  41. package/core/reasoning/confidence_scoring.py +484 -484
  42. package/core/reasoning/events.py +149 -149
  43. package/core/reasoning/orchestrator.py +1561 -1561
  44. package/core/reasoning/output_writer.py +345 -345
  45. package/core/reasoning/payload_validation.py +524 -524
  46. package/core/reasoning/pipeline_storage.py +427 -427
  47. package/core/reasoning/scholar.py +458 -458
  48. package/core/reasoning/source_corpus.py +433 -433
  49. package/core/reasoning/source_scoring.py +855 -855
  50. package/core/replay.py +315 -315
  51. package/core/shell_sessions.py +1553 -1553
  52. package/credentials.py +307 -307
  53. package/database.py +333 -333
  54. package/docs/PRIVACY.md +78 -78
  55. package/docs/SECURITY.md +108 -108
  56. package/docs/THIRD_PARTY_NOTICES.md +148 -148
  57. package/env_secrets.py +118 -118
  58. package/json_store.py +343 -343
  59. package/logging_setup.py +295 -295
  60. package/main.py +3433 -3433
  61. package/mcp_client/client.py +733 -733
  62. package/mcp_client/config.py +484 -484
  63. package/mcp_client/registration.py +343 -343
  64. package/memories/manager.py +147 -147
  65. package/memories/tui_commands.py +319 -239
  66. package/package.json +303 -303
  67. package/project_init/config_files.py +282 -282
  68. package/project_init/generator.py +728 -728
  69. package/project_init/manifest.py +418 -418
  70. package/project_init/tui_commands.py +147 -147
  71. package/prompts/critic_pass.md +24 -24
  72. package/prompts/revalidate.md +25 -25
  73. package/prompts/revise.md +17 -17
  74. package/prompts/source_grounding.md +66 -66
  75. package/prompts/system_prompts.py +316 -316
  76. package/providers.json.example +94 -94
  77. package/pyproject.toml +149 -149
  78. package/requirements.txt +295 -295
  79. package/safety/policy_enforcement.py +1082 -1082
  80. package/security/askpass.py +419 -419
  81. package/security/capability.py +207 -207
  82. package/security/confine.py +484 -484
  83. package/security/hosted.py +182 -182
  84. package/security/permissions.py +237 -237
  85. package/security/protected_paths.py +501 -501
  86. package/security/sandbox.py +4108 -4108
  87. package/skills/builtin/math/formal-specification.md +67 -67
  88. package/skills/builtin/math/numerical-methods.md +66 -66
  89. package/skills/builtin/research/experiment-design.md +62 -62
  90. package/skills/builtin/research/reproducibility.md +55 -55
  91. package/skills/builtin/research/source-evaluation.md +70 -70
  92. package/skills/builtin/research/statistical-inference.md +63 -63
  93. package/skills/builtin/research/technical-writing.md +59 -59
  94. package/skills/builtin/software/code-review.md +61 -61
  95. package/skills/builtin/software/debugging.md +61 -61
  96. package/skills/builtin/software/test-design.md +67 -67
  97. package/skills/tui_commands.py +110 -110
  98. package/storage.py +1338 -1338
  99. package/tools/base.py +465 -465
  100. package/tools/builtin/_math_common.py +667 -667
  101. package/tools/builtin/_net_common.py +267 -267
  102. package/tools/builtin/arxiv.py +681 -681
  103. package/tools/builtin/fetch_url.py +371 -371
  104. package/tools/builtin/file_ops.py +720 -720
  105. package/tools/builtin/geometry.py +184 -184
  106. package/tools/builtin/load_skill.py +76 -76
  107. package/tools/builtin/pin.py +227 -227
  108. package/tools/builtin/probability_stats.py +224 -224
  109. package/tools/builtin/project_docs.py +464 -464
  110. package/tools/builtin/remember.py +175 -175
  111. package/tools/builtin/shell.py +980 -980
  112. package/tools/builtin/todo.py +149 -149
  113. package/tools/builtin/web_search.py +328 -328
  114. package/tools/context.py +159 -159
  115. package/tools/isolation.py +387 -387
  116. package/tools/registry.py +1242 -1242
  117. package/tui/app.py +8318 -8307
  118. package/tui/app.tcss +747 -739
  119. package/tui/commands.py +26 -26
  120. package/tui/diffs.py +286 -286
  121. package/tui/history.py +128 -128
  122. package/tui/hosted_driver.py +102 -102
  123. package/tui/markdown.py +1329 -1329
  124. package/tui/meters.py +364 -364
  125. package/tui/ravens.py +146 -146
  126. package/tui/screens.py +1255 -1197
  127. package/tui/themes.py +599 -599
  128. package/tui/widgets.py +3813 -3796
@@ -1,267 +1,267 @@
1
- """
2
- tools/builtin/_net_common.py
3
-
4
- What the two network-backed search tools share. `_math_common.py`'s
5
- precedent, one directory over: a private module the built-in tools import
6
- from, never registered and never advertised.
7
-
8
- TWO THINGS NOW. The first is the in-process response cache: `web_search.py`
9
- and `arxiv.py` each carried a `_cache_get` / `_cache_set` pair that was
10
- identical apart from the type of the value being stored, plus a
11
- `CACHE_TTL_S = 300` each, plus a bare dict each. Three copies of an expiry
12
- rule is how one of them comes to be measured in minutes.
13
-
14
- The second is `ThirdPartyText` (batch 156, ROADMAP_v3 §73), which is where
15
- the two search tools now bound and redact the strings they did not write.
16
- It is here rather than in either tool for the reason the cache is: both
17
- were doing the same thing to the same kind of value, and the one that got
18
- it wrong got it wrong in the half the other had already fixed.
19
-
20
- There are THREE callers now -- `fetch_url.py` joined them in ROADMAP_v3 §51 --
21
- and the third is why §52 gave this class a size bound as well as a time one.
22
-
23
- WHAT IS DELIBERATELY *NOT* SHARED. Each tool keeps its OWN instance, so the
24
- two key spaces stay separate -- a `TTLCache` shared between them would let
25
- an arXiv query and a web query collide on an equal key string, which is a
26
- correctness bug rather than a tidiness one. Their `@field_validator`s
27
- (`strip_keywords`, `strip_query`) are also left alone: they are three lines
28
- each, they differ in the message the model is shown, and folding them into
29
- a factory would make the params class harder to read to save two lines.
30
- """
31
-
32
- from __future__ import annotations
33
-
34
- import logging
35
- import threading
36
- import time
37
- from typing import Any, Optional
38
-
39
- from safety.policy_enforcement import redact_output_text
40
-
41
- logger = logging.getLogger(__name__)
42
-
43
- # ROADMAP_v3 §52 (RB6). How many entries any one of these may hold.
44
- #
45
- # Derived rather than picked: `config.MAX_ITERATIONS` is 50, so one pass can
46
- # make at most 50 tool calls, and a `fetch_url` whose URL redirects stores
47
- # TWO keys for one body (the requested spelling and the answering one). 100
48
- # keys is therefore the most a single pass can reach, and this sits above it
49
- # so that the cap bounds a PROCESS -- which runs many passes and many runs --
50
- # without ever biting inside one.
51
- #
52
- # That "without ever biting" is the requirement, not a nicety. A `fetch_url`
53
- # page-2 request that misses degrades to a re-fetch, which is exactly the
54
- # spliced-document failure §51 measured and built PG5 to prevent, so an
55
- # eviction that fires in ordinary use would trade a bounded memory cost for
56
- # an unbounded correctness one.
57
- #
58
- # In entries and not in bytes, because a generic cache cannot measure the
59
- # size of what a caller hands it. For the caller that made this necessary the
60
- # two are interchangeable anyway: `fetch_url`'s MAX_CONTENT_BYTES holds each
61
- # body to ~64 KB whatever the encoding, so PG6's refusal to raise that cap is
62
- # what turns this count into a memory bound at all.
63
- MAX_ENTRIES = 128
64
-
65
-
66
- # ROADMAP_v3 §73 (A58). How long a URL this harness is willing to SHOW the
67
- # model may be.
68
- #
69
- # DERIVED, NOT PICKED, and the derivation is the argument for dropping an
70
- # over-long URL rather than cutting it: `fetch_url` already bounds the URL
71
- # the model may ASK for, at `max_length=400` on `FetchURLParams.url`.
72
- # Measured (batch 156), a 401-character URL comes back from fetch_url's own
73
- # gate as an "Invalid arguments" refusal while a 400-character one is
74
- # accepted. So a longer URL is one THIS HARNESS refuses to take, and
75
- # offering it in a search result is offering a route to nowhere.
76
- #
77
- # ONE DEFINITION: `fetch_url` reads it from here rather than spelling 400
78
- # again. The two were the same number in two files, which is the state a
79
- # pair of numbers is in just before it stops being a pair.
80
- MAX_URL_CHARS = 400
81
-
82
-
83
- class ThirdPartyText:
84
- """One tool call's worth of strings nobody here wrote, REDACTED and
85
- THEN cut.
86
-
87
- ORDER IS THE WHOLE POINT, and it is RB1's point restated for the two
88
- tools RB1 did not reach. `fetch_url` redacts its whole body above both
89
- the hold and the slice, because a pattern only matches what it can see
90
- whole; `web_search` and `arxiv_search` cut first and let
91
- `registry.dispatch`'s `check_output_policy` redact the piece that was
92
- kept. A fixed-length credential straddling such a cut is then invisible
93
- to both halves -- the kept prefix is not the token and the dropped
94
- suffix is gone. Measured (batch 156) at all three of their cut points:
95
- a `ghp_` token survived post-hoc redaction at 36 of the 39 offsets it
96
- could straddle, and redacting the whole value first caught every one.
97
-
98
- TWO KINDS OF VALUE, because a cut means different things to them (A58).
99
- `text()` cuts and flags: half an abstract is still an abstract, and PG4
100
- says the flag is owed. `url()` DROPS and flags: half a URL is not a
101
- URL, it is a fetchable-looking string that is not the resource, and
102
- `pdf_url` and arXiv's `full_*_url` pair exist precisely to be the route
103
- the model takes to recover what was cut. A broken route offered as a
104
- route is worse than an absent one, which the flag makes visible.
105
-
106
- TOTAL OVER WHATEVER THE PROVIDER EMITTED, for the reason `tools/base.py`
107
- gives about tool arguments: a key that is present with a wrong-typed
108
- value is not an absent key, and `r.get("title", "(untitled)")` returns
109
- None for `{"title": null}` rather than the default. Both accessors
110
- answer None for a non-string instead of raising, and the caller decides
111
- what an absent value means -- which is a decision the two tools make
112
- differently and should (A61).
113
-
114
- ONE WARNING PER CALL, not per value (#49, RB2). Redacting here means
115
- `check_output_policy` finds nothing left to alter, so the obligation to
116
- say that something WAS altered moves here with the redaction; and a
117
- feed with twenty-five authors must not produce twenty-five warning
118
- lines. The count is carried on the instance and `report()` spends it.
119
- The match is never echoed, for the reason `check_output_policy` and
120
- `fetch_url._redacted` both state: this function exists because
121
- something credential-shaped was found, so quoting it to say so would
122
- put it in app.log and every sink a WARNING reaches.
123
- """
124
-
125
- def __init__(self, tool: str) -> None:
126
- self._tool = tool
127
- self._redacted = 0
128
-
129
- def text(self, value: Any, cap: int) -> tuple:
130
- """`(text, truncated)` -- `value` redacted and THEN cut to `cap`.
131
-
132
- `(None, False)` for anything that is not a string, so a caller can
133
- tell "the provider did not give me this" from "it gave me an empty
134
- one" and choose its own substitute.
135
-
136
- THE FLAG COUNTS REDACTED CHARACTERS, which is RB3's consequence
137
- arriving here: redaction can SHRINK a value past the cap, and when
138
- it does there is nothing further the cut is withholding. A flag
139
- measured on the raw length would claim there is.
140
- """
141
- if not isinstance(value, str):
142
- return None, False
143
- clean = self._clean(value)
144
- return clean[:cap], len(clean) > cap
145
-
146
- def url(self, value: Any, cap: int = MAX_URL_CHARS) -> tuple:
147
- """`(url, withheld)` -- a URL past `cap` is DROPPED, not cut (A58).
148
-
149
- `(None, False)` for a non-string or an empty one: nothing was
150
- withheld, there was nothing there. `(None, True)` says a URL was
151
- found and is not being shown, which is the fact the flag carries.
152
- """
153
- if not isinstance(value, str) or not value:
154
- return None, False
155
- clean = self._clean(value)
156
- if len(clean) > cap:
157
- return None, True
158
- return clean, False
159
-
160
- def _clean(self, value: str) -> str:
161
- clean = redact_output_text(value)
162
- if clean != value:
163
- self._redacted += 1
164
- return clean
165
-
166
- def report(self) -> None:
167
- """The one warning this call owes (#49), or silence."""
168
- if self._redacted:
169
- logger.warning(
170
- "Output policy altered %s result: credentials replaced in "
171
- "%d value(s) before they were cut.",
172
- self._tool, self._redacted)
173
-
174
-
175
- class TTLCache:
176
- """A tiny time-boxed cache, keyed by whatever string a caller builds.
177
-
178
- TWO BOUNDS, ANSWERING DIFFERENT QUESTIONS. The TTL is about FRESHNESS: a
179
- stale entry is dropped as it is read, which is what makes a repeated
180
- query after the TTL a real request again rather than a permanent miss
181
- against a growing dict. `max_entries` is about SIZE, and it is the only
182
- thing that bounds a cache nothing reads a second time.
183
-
184
- THE SWEEP ON WRITE IS §52 (RB5), AND THE REASON THIS DOCSTRING USED TO
185
- GIVE FOR NOT HAVING ONE IS NOW FALSE. It said a sweep "would be machinery
186
- for a problem neither tool has" -- true of two callers holding result
187
- lists, and not of the third, which holds bodies. Measured (batch 106):
188
- 100 fetches of a capped page held 6.3 MB and not one entry was ever
189
- dropped, because expiry ran only in `get` and nothing read those keys
190
- again. Expiring on read alone means the TTL bounds staleness and nothing
191
- whatever bounds memory.
192
-
193
- READ IS DELIBERATELY UNCHANGED. `get` still expires the one key it
194
- touches, because that path is what makes a post-TTL query a real request
195
- again; the sweep is additional, not a replacement, and a cache that only
196
- swept would keep serving a stale entry until someone else wrote.
197
- """
198
-
199
- def __init__(self, ttl_s: float, max_entries: int = MAX_ENTRIES) -> None:
200
- self.ttl_s = ttl_s
201
- self.max_entries = max_entries
202
- self._entries: dict[str, tuple[float, Any]] = {}
203
- # LOCKED BECAUSE THE REGISTRY ALREADY SAYS IT IS (batch 117).
204
- # `spawn_subagent` is the one tool with `parallel=True`, and the
205
- # comment beside that field claims "everything the child's run
206
- # touches was made concurrency-safe first -- the budget's lock, the
207
- # serialised question, the per-future authorization". The child
208
- # reaches this cache through `arxiv_search` and `fetch_url`, and it
209
- # was the one item on that list without a lock.
210
- #
211
- # Single dict operations are atomic under CPython, so a corrupt
212
- # dict was never the risk. `set` is a read-modify-write -- pop,
213
- # assign, sweep, then evict while over the cap -- so two children
214
- # storing at once could lose one entry and evict in an order
215
- # neither asked for. Mirrors `GrantBudget.take()`, which is the
216
- # pattern this module should have followed from the start.
217
- #
218
- # NOT reentrant, and `_sweep` is the reason: it is called from
219
- # `set` with the lock already held, so it must never take it.
220
- self._lock = threading.Lock()
221
-
222
- def get(self, key: str) -> Optional[Any]:
223
- with self._lock:
224
- entry = self._entries.get(key)
225
- if not entry:
226
- return None
227
- stored_at, value = entry
228
- if time.time() - stored_at > self.ttl_s:
229
- self._entries.pop(key, None)
230
- return None
231
- return value
232
-
233
- def _sweep(self, now: float) -> None:
234
- """Drop everything already expired. Write-side only (§52, RB5).
235
-
236
- CALLED WITH `self._lock` HELD, and it must not take it: the lock is
237
- a plain `Lock`, so re-entering would deadlock the caller.
238
- """
239
- stale = [key for key, (stored_at, _) in self._entries.items()
240
- if now - stored_at > self.ttl_s]
241
- for key in stale:
242
- del self._entries[key]
243
-
244
- def set(self, key: str, value: Any) -> None:
245
- with self._lock:
246
- self._set_locked(key, value)
247
-
248
- def _set_locked(self, key: str, value: Any) -> None:
249
- now = time.time()
250
- # POPPED FIRST so a re-store moves the key to the END of the dict.
251
- # Assigning to an existing key keeps its original insertion position,
252
- # which would make the freshest entry the next one evicted -- and the
253
- # freshest entry is the one a caller is most likely to read next.
254
- self._entries.pop(key, None)
255
- self._entries[key] = (now, value)
256
- self._sweep(now)
257
- # Insertion order IS age order, since every entry is stamped as it is
258
- # written and the TTL is uniform. So the oldest key is simply the
259
- # first one, and no second ordering has to be kept consistent.
260
- while len(self._entries) > self.max_entries:
261
- self._entries.pop(next(iter(self._entries)))
262
-
263
- def clear(self) -> None:
264
- """Drop everything. For tests, which must not inherit a previous
265
- test's answers -- the same reason conftest isolates the stores."""
266
- with self._lock:
267
- self._entries.clear()
1
+ """
2
+ tools/builtin/_net_common.py
3
+
4
+ What the two network-backed search tools share. `_math_common.py`'s
5
+ precedent, one directory over: a private module the built-in tools import
6
+ from, never registered and never advertised.
7
+
8
+ TWO THINGS NOW. The first is the in-process response cache: `web_search.py`
9
+ and `arxiv.py` each carried a `_cache_get` / `_cache_set` pair that was
10
+ identical apart from the type of the value being stored, plus a
11
+ `CACHE_TTL_S = 300` each, plus a bare dict each. Three copies of an expiry
12
+ rule is how one of them comes to be measured in minutes.
13
+
14
+ The second is `ThirdPartyText` (batch 156, ROADMAP_v3 §73), which is where
15
+ the two search tools now bound and redact the strings they did not write.
16
+ It is here rather than in either tool for the reason the cache is: both
17
+ were doing the same thing to the same kind of value, and the one that got
18
+ it wrong got it wrong in the half the other had already fixed.
19
+
20
+ There are THREE callers now -- `fetch_url.py` joined them in ROADMAP_v3 §51 --
21
+ and the third is why §52 gave this class a size bound as well as a time one.
22
+
23
+ WHAT IS DELIBERATELY *NOT* SHARED. Each tool keeps its OWN instance, so the
24
+ two key spaces stay separate -- a `TTLCache` shared between them would let
25
+ an arXiv query and a web query collide on an equal key string, which is a
26
+ correctness bug rather than a tidiness one. Their `@field_validator`s
27
+ (`strip_keywords`, `strip_query`) are also left alone: they are three lines
28
+ each, they differ in the message the model is shown, and folding them into
29
+ a factory would make the params class harder to read to save two lines.
30
+ """
31
+
32
+ from __future__ import annotations
33
+
34
+ import logging
35
+ import threading
36
+ import time
37
+ from typing import Any, Optional
38
+
39
+ from safety.policy_enforcement import redact_output_text
40
+
41
+ logger = logging.getLogger(__name__)
42
+
43
+ # ROADMAP_v3 §52 (RB6). How many entries any one of these may hold.
44
+ #
45
+ # Derived rather than picked: `config.MAX_ITERATIONS` is 50, so one pass can
46
+ # make at most 50 tool calls, and a `fetch_url` whose URL redirects stores
47
+ # TWO keys for one body (the requested spelling and the answering one). 100
48
+ # keys is therefore the most a single pass can reach, and this sits above it
49
+ # so that the cap bounds a PROCESS -- which runs many passes and many runs --
50
+ # without ever biting inside one.
51
+ #
52
+ # That "without ever biting" is the requirement, not a nicety. A `fetch_url`
53
+ # page-2 request that misses degrades to a re-fetch, which is exactly the
54
+ # spliced-document failure §51 measured and built PG5 to prevent, so an
55
+ # eviction that fires in ordinary use would trade a bounded memory cost for
56
+ # an unbounded correctness one.
57
+ #
58
+ # In entries and not in bytes, because a generic cache cannot measure the
59
+ # size of what a caller hands it. For the caller that made this necessary the
60
+ # two are interchangeable anyway: `fetch_url`'s MAX_CONTENT_BYTES holds each
61
+ # body to ~64 KB whatever the encoding, so PG6's refusal to raise that cap is
62
+ # what turns this count into a memory bound at all.
63
+ MAX_ENTRIES = 128
64
+
65
+
66
+ # ROADMAP_v3 §73 (A58). How long a URL this harness is willing to SHOW the
67
+ # model may be.
68
+ #
69
+ # DERIVED, NOT PICKED, and the derivation is the argument for dropping an
70
+ # over-long URL rather than cutting it: `fetch_url` already bounds the URL
71
+ # the model may ASK for, at `max_length=400` on `FetchURLParams.url`.
72
+ # Measured (batch 156), a 401-character URL comes back from fetch_url's own
73
+ # gate as an "Invalid arguments" refusal while a 400-character one is
74
+ # accepted. So a longer URL is one THIS HARNESS refuses to take, and
75
+ # offering it in a search result is offering a route to nowhere.
76
+ #
77
+ # ONE DEFINITION: `fetch_url` reads it from here rather than spelling 400
78
+ # again. The two were the same number in two files, which is the state a
79
+ # pair of numbers is in just before it stops being a pair.
80
+ MAX_URL_CHARS = 400
81
+
82
+
83
+ class ThirdPartyText:
84
+ """One tool call's worth of strings nobody here wrote, REDACTED and
85
+ THEN cut.
86
+
87
+ ORDER IS THE WHOLE POINT, and it is RB1's point restated for the two
88
+ tools RB1 did not reach. `fetch_url` redacts its whole body above both
89
+ the hold and the slice, because a pattern only matches what it can see
90
+ whole; `web_search` and `arxiv_search` cut first and let
91
+ `registry.dispatch`'s `check_output_policy` redact the piece that was
92
+ kept. A fixed-length credential straddling such a cut is then invisible
93
+ to both halves -- the kept prefix is not the token and the dropped
94
+ suffix is gone. Measured (batch 156) at all three of their cut points:
95
+ a `ghp_` token survived post-hoc redaction at 36 of the 39 offsets it
96
+ could straddle, and redacting the whole value first caught every one.
97
+
98
+ TWO KINDS OF VALUE, because a cut means different things to them (A58).
99
+ `text()` cuts and flags: half an abstract is still an abstract, and PG4
100
+ says the flag is owed. `url()` DROPS and flags: half a URL is not a
101
+ URL, it is a fetchable-looking string that is not the resource, and
102
+ `pdf_url` and arXiv's `full_*_url` pair exist precisely to be the route
103
+ the model takes to recover what was cut. A broken route offered as a
104
+ route is worse than an absent one, which the flag makes visible.
105
+
106
+ TOTAL OVER WHATEVER THE PROVIDER EMITTED, for the reason `tools/base.py`
107
+ gives about tool arguments: a key that is present with a wrong-typed
108
+ value is not an absent key, and `r.get("title", "(untitled)")` returns
109
+ None for `{"title": null}` rather than the default. Both accessors
110
+ answer None for a non-string instead of raising, and the caller decides
111
+ what an absent value means -- which is a decision the two tools make
112
+ differently and should (A61).
113
+
114
+ ONE WARNING PER CALL, not per value (#49, RB2). Redacting here means
115
+ `check_output_policy` finds nothing left to alter, so the obligation to
116
+ say that something WAS altered moves here with the redaction; and a
117
+ feed with twenty-five authors must not produce twenty-five warning
118
+ lines. The count is carried on the instance and `report()` spends it.
119
+ The match is never echoed, for the reason `check_output_policy` and
120
+ `fetch_url._redacted` both state: this function exists because
121
+ something credential-shaped was found, so quoting it to say so would
122
+ put it in app.log and every sink a WARNING reaches.
123
+ """
124
+
125
+ def __init__(self, tool: str) -> None:
126
+ self._tool = tool
127
+ self._redacted = 0
128
+
129
+ def text(self, value: Any, cap: int) -> tuple:
130
+ """`(text, truncated)` -- `value` redacted and THEN cut to `cap`.
131
+
132
+ `(None, False)` for anything that is not a string, so a caller can
133
+ tell "the provider did not give me this" from "it gave me an empty
134
+ one" and choose its own substitute.
135
+
136
+ THE FLAG COUNTS REDACTED CHARACTERS, which is RB3's consequence
137
+ arriving here: redaction can SHRINK a value past the cap, and when
138
+ it does there is nothing further the cut is withholding. A flag
139
+ measured on the raw length would claim there is.
140
+ """
141
+ if not isinstance(value, str):
142
+ return None, False
143
+ clean = self._clean(value)
144
+ return clean[:cap], len(clean) > cap
145
+
146
+ def url(self, value: Any, cap: int = MAX_URL_CHARS) -> tuple:
147
+ """`(url, withheld)` -- a URL past `cap` is DROPPED, not cut (A58).
148
+
149
+ `(None, False)` for a non-string or an empty one: nothing was
150
+ withheld, there was nothing there. `(None, True)` says a URL was
151
+ found and is not being shown, which is the fact the flag carries.
152
+ """
153
+ if not isinstance(value, str) or not value:
154
+ return None, False
155
+ clean = self._clean(value)
156
+ if len(clean) > cap:
157
+ return None, True
158
+ return clean, False
159
+
160
+ def _clean(self, value: str) -> str:
161
+ clean = redact_output_text(value)
162
+ if clean != value:
163
+ self._redacted += 1
164
+ return clean
165
+
166
+ def report(self) -> None:
167
+ """The one warning this call owes (#49), or silence."""
168
+ if self._redacted:
169
+ logger.warning(
170
+ "Output policy altered %s result: credentials replaced in "
171
+ "%d value(s) before they were cut.",
172
+ self._tool, self._redacted)
173
+
174
+
175
+ class TTLCache:
176
+ """A tiny time-boxed cache, keyed by whatever string a caller builds.
177
+
178
+ TWO BOUNDS, ANSWERING DIFFERENT QUESTIONS. The TTL is about FRESHNESS: a
179
+ stale entry is dropped as it is read, which is what makes a repeated
180
+ query after the TTL a real request again rather than a permanent miss
181
+ against a growing dict. `max_entries` is about SIZE, and it is the only
182
+ thing that bounds a cache nothing reads a second time.
183
+
184
+ THE SWEEP ON WRITE IS §52 (RB5), AND THE REASON THIS DOCSTRING USED TO
185
+ GIVE FOR NOT HAVING ONE IS NOW FALSE. It said a sweep "would be machinery
186
+ for a problem neither tool has" -- true of two callers holding result
187
+ lists, and not of the third, which holds bodies. Measured (batch 106):
188
+ 100 fetches of a capped page held 6.3 MB and not one entry was ever
189
+ dropped, because expiry ran only in `get` and nothing read those keys
190
+ again. Expiring on read alone means the TTL bounds staleness and nothing
191
+ whatever bounds memory.
192
+
193
+ READ IS DELIBERATELY UNCHANGED. `get` still expires the one key it
194
+ touches, because that path is what makes a post-TTL query a real request
195
+ again; the sweep is additional, not a replacement, and a cache that only
196
+ swept would keep serving a stale entry until someone else wrote.
197
+ """
198
+
199
+ def __init__(self, ttl_s: float, max_entries: int = MAX_ENTRIES) -> None:
200
+ self.ttl_s = ttl_s
201
+ self.max_entries = max_entries
202
+ self._entries: dict[str, tuple[float, Any]] = {}
203
+ # LOCKED BECAUSE THE REGISTRY ALREADY SAYS IT IS (batch 117).
204
+ # `spawn_subagent` is the one tool with `parallel=True`, and the
205
+ # comment beside that field claims "everything the child's run
206
+ # touches was made concurrency-safe first -- the budget's lock, the
207
+ # serialised question, the per-future authorization". The child
208
+ # reaches this cache through `arxiv_search` and `fetch_url`, and it
209
+ # was the one item on that list without a lock.
210
+ #
211
+ # Single dict operations are atomic under CPython, so a corrupt
212
+ # dict was never the risk. `set` is a read-modify-write -- pop,
213
+ # assign, sweep, then evict while over the cap -- so two children
214
+ # storing at once could lose one entry and evict in an order
215
+ # neither asked for. Mirrors `GrantBudget.take()`, which is the
216
+ # pattern this module should have followed from the start.
217
+ #
218
+ # NOT reentrant, and `_sweep` is the reason: it is called from
219
+ # `set` with the lock already held, so it must never take it.
220
+ self._lock = threading.Lock()
221
+
222
+ def get(self, key: str) -> Optional[Any]:
223
+ with self._lock:
224
+ entry = self._entries.get(key)
225
+ if not entry:
226
+ return None
227
+ stored_at, value = entry
228
+ if time.time() - stored_at > self.ttl_s:
229
+ self._entries.pop(key, None)
230
+ return None
231
+ return value
232
+
233
+ def _sweep(self, now: float) -> None:
234
+ """Drop everything already expired. Write-side only (§52, RB5).
235
+
236
+ CALLED WITH `self._lock` HELD, and it must not take it: the lock is
237
+ a plain `Lock`, so re-entering would deadlock the caller.
238
+ """
239
+ stale = [key for key, (stored_at, _) in self._entries.items()
240
+ if now - stored_at > self.ttl_s]
241
+ for key in stale:
242
+ del self._entries[key]
243
+
244
+ def set(self, key: str, value: Any) -> None:
245
+ with self._lock:
246
+ self._set_locked(key, value)
247
+
248
+ def _set_locked(self, key: str, value: Any) -> None:
249
+ now = time.time()
250
+ # POPPED FIRST so a re-store moves the key to the END of the dict.
251
+ # Assigning to an existing key keeps its original insertion position,
252
+ # which would make the freshest entry the next one evicted -- and the
253
+ # freshest entry is the one a caller is most likely to read next.
254
+ self._entries.pop(key, None)
255
+ self._entries[key] = (now, value)
256
+ self._sweep(now)
257
+ # Insertion order IS age order, since every entry is stamped as it is
258
+ # written and the TTL is uniform. So the oldest key is simply the
259
+ # first one, and no second ordering has to be kept consistent.
260
+ while len(self._entries) > self.max_entries:
261
+ self._entries.pop(next(iter(self._entries)))
262
+
263
+ def clear(self) -> None:
264
+ """Drop everything. For tests, which must not inherit a previous
265
+ test's answers -- the same reason conftest isolates the stores."""
266
+ with self._lock:
267
+ self._entries.clear()