@yahyahammad/venastine 0.3.8 → 0.3.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/.env.example +19 -19
  2. package/CONFIG_ARCHITECTURE.md +2139 -2002
  3. package/LICENSE +201 -201
  4. package/README.md +1043 -1034
  5. package/agents/builtin/compactor.md +75 -75
  6. package/agents/builtin/explore.md +77 -77
  7. package/agents/builtin/pipeline-reviewer.md +100 -100
  8. package/agents/builtin/plan.md +106 -106
  9. package/agents/builtin/review.md +74 -74
  10. package/agents/manager.py +261 -261
  11. package/agents/subagent_tool.py +472 -472
  12. package/agents/tui_commands.py +238 -146
  13. package/bin/venastine.mjs +584 -528
  14. package/bin/venastine.py +73 -71
  15. package/cli_commands.py +848 -835
  16. package/config.yaml +64 -0
  17. package/config_edit.py +1301 -1301
  18. package/config_schema.py +987 -966
  19. package/config_update.py +749 -749
  20. package/core/agent_activity.py +256 -256
  21. package/core/ansi.py +452 -452
  22. package/core/approval.py +137 -137
  23. package/core/call_view.py +364 -364
  24. package/core/cancel.py +121 -121
  25. package/core/chatgpt_auth.py +747 -747
  26. package/core/claims_view.py +107 -107
  27. package/core/client.py +2756 -2715
  28. package/core/commands.py +464 -454
  29. package/core/compaction.py +1735 -1735
  30. package/core/config_loader.py +1982 -1982
  31. package/core/connect.py +686 -686
  32. package/core/events.py +123 -123
  33. package/core/file_mentions.py +280 -280
  34. package/core/interaction.py +411 -411
  35. package/core/loop.py +2566 -2566
  36. package/core/memory.py +482 -482
  37. package/core/pipeline_models.py +190 -190
  38. package/core/provider_errors.py +274 -234
  39. package/core/reasoning/authorization.py +130 -130
  40. package/core/reasoning/base.py +219 -219
  41. package/core/reasoning/confidence_scoring.py +484 -484
  42. package/core/reasoning/events.py +149 -149
  43. package/core/reasoning/orchestrator.py +1561 -1561
  44. package/core/reasoning/output_writer.py +345 -345
  45. package/core/reasoning/payload_validation.py +524 -524
  46. package/core/reasoning/pipeline_storage.py +427 -427
  47. package/core/reasoning/scholar.py +467 -458
  48. package/core/reasoning/source_corpus.py +433 -433
  49. package/core/reasoning/source_scoring.py +855 -855
  50. package/core/replay.py +315 -315
  51. package/core/shell_sessions.py +1553 -1553
  52. package/credentials.py +307 -307
  53. package/database.py +333 -333
  54. package/docs/PRIVACY.md +78 -78
  55. package/docs/SECURITY.md +114 -108
  56. package/docs/THIRD_PARTY_NOTICES.md +148 -148
  57. package/env_secrets.py +118 -118
  58. package/json_store.py +343 -343
  59. package/logging_setup.py +295 -295
  60. package/main.py +3433 -3433
  61. package/mcp_client/client.py +780 -733
  62. package/mcp_client/config.py +484 -484
  63. package/mcp_client/registration.py +343 -343
  64. package/memories/manager.py +147 -147
  65. package/memories/tui_commands.py +319 -239
  66. package/package.json +303 -303
  67. package/project_init/config_files.py +282 -282
  68. package/project_init/generator.py +728 -728
  69. package/project_init/manifest.py +418 -418
  70. package/project_init/tui_commands.py +147 -147
  71. package/prompts/critic_pass.md +24 -24
  72. package/prompts/revalidate.md +25 -25
  73. package/prompts/revise.md +17 -17
  74. package/prompts/source_grounding.md +66 -66
  75. package/prompts/system_prompts.py +316 -316
  76. package/providers.json.example +94 -94
  77. package/pyproject.toml +149 -149
  78. package/requirements.txt +314 -295
  79. package/safety/policy_enforcement.py +1082 -1082
  80. package/security/askpass.py +419 -419
  81. package/security/capability.py +207 -207
  82. package/security/confine.py +484 -484
  83. package/security/hosted.py +182 -182
  84. package/security/permissions.py +237 -237
  85. package/security/protected_paths.py +501 -501
  86. package/security/sandbox.py +4108 -4108
  87. package/skills/builtin/math/formal-specification.md +67 -67
  88. package/skills/builtin/math/numerical-methods.md +66 -66
  89. package/skills/builtin/research/experiment-design.md +62 -62
  90. package/skills/builtin/research/reproducibility.md +55 -55
  91. package/skills/builtin/research/source-evaluation.md +70 -70
  92. package/skills/builtin/research/statistical-inference.md +63 -63
  93. package/skills/builtin/research/technical-writing.md +59 -59
  94. package/skills/builtin/software/code-review.md +61 -61
  95. package/skills/builtin/software/debugging.md +61 -61
  96. package/skills/builtin/software/test-design.md +67 -67
  97. package/skills/tui_commands.py +110 -110
  98. package/storage.py +1338 -1338
  99. package/tools/base.py +465 -465
  100. package/tools/builtin/_math_common.py +667 -667
  101. package/tools/builtin/_net_common.py +1145 -267
  102. package/tools/builtin/_page_text.py +490 -0
  103. package/tools/builtin/_pdf_text.py +113 -0
  104. package/tools/builtin/arxiv.py +685 -681
  105. package/tools/builtin/fetch_url.py +639 -371
  106. package/tools/builtin/file_ops.py +720 -720
  107. package/tools/builtin/geometry.py +184 -184
  108. package/tools/builtin/load_skill.py +76 -76
  109. package/tools/builtin/pin.py +227 -227
  110. package/tools/builtin/probability_stats.py +224 -224
  111. package/tools/builtin/project_docs.py +464 -464
  112. package/tools/builtin/remember.py +175 -175
  113. package/tools/builtin/shell.py +980 -980
  114. package/tools/builtin/todo.py +149 -149
  115. package/tools/builtin/web_search.py +402 -328
  116. package/tools/context.py +159 -159
  117. package/tools/isolation.py +387 -387
  118. package/tools/registry.py +1252 -1242
  119. package/tui/app.py +8372 -8307
  120. package/tui/app.tcss +754 -739
  121. package/tui/commands.py +26 -26
  122. package/tui/diffs.py +286 -286
  123. package/tui/history.py +128 -128
  124. package/tui/hosted_driver.py +102 -102
  125. package/tui/markdown.py +1329 -1329
  126. package/tui/meters.py +364 -364
  127. package/tui/ravens.py +146 -146
  128. package/tui/screens.py +1255 -1197
  129. package/tui/themes.py +599 -599
  130. package/tui/widgets.py +3861 -3796
@@ -1,267 +1,1145 @@
1
- """
2
- tools/builtin/_net_common.py
3
-
4
- What the two network-backed search tools share. `_math_common.py`'s
5
- precedent, one directory over: a private module the built-in tools import
6
- from, never registered and never advertised.
7
-
8
- TWO THINGS NOW. The first is the in-process response cache: `web_search.py`
9
- and `arxiv.py` each carried a `_cache_get` / `_cache_set` pair that was
10
- identical apart from the type of the value being stored, plus a
11
- `CACHE_TTL_S = 300` each, plus a bare dict each. Three copies of an expiry
12
- rule is how one of them comes to be measured in minutes.
13
-
14
- The second is `ThirdPartyText` (batch 156, ROADMAP_v3 §73), which is where
15
- the two search tools now bound and redact the strings they did not write.
16
- It is here rather than in either tool for the reason the cache is: both
17
- were doing the same thing to the same kind of value, and the one that got
18
- it wrong got it wrong in the half the other had already fixed.
19
-
20
- There are THREE callers now -- `fetch_url.py` joined them in ROADMAP_v3 §51 --
21
- and the third is why §52 gave this class a size bound as well as a time one.
22
-
23
- WHAT IS DELIBERATELY *NOT* SHARED. Each tool keeps its OWN instance, so the
24
- two key spaces stay separate -- a `TTLCache` shared between them would let
25
- an arXiv query and a web query collide on an equal key string, which is a
26
- correctness bug rather than a tidiness one. Their `@field_validator`s
27
- (`strip_keywords`, `strip_query`) are also left alone: they are three lines
28
- each, they differ in the message the model is shown, and folding them into
29
- a factory would make the params class harder to read to save two lines.
30
- """
31
-
32
- from __future__ import annotations
33
-
34
- import logging
35
- import threading
36
- import time
37
- from typing import Any, Optional
38
-
39
- from safety.policy_enforcement import redact_output_text
40
-
41
- logger = logging.getLogger(__name__)
42
-
43
- # ROADMAP_v3 §52 (RB6). How many entries any one of these may hold.
44
- #
45
- # Derived rather than picked: `config.MAX_ITERATIONS` is 50, so one pass can
46
- # make at most 50 tool calls, and a `fetch_url` whose URL redirects stores
47
- # TWO keys for one body (the requested spelling and the answering one). 100
48
- # keys is therefore the most a single pass can reach, and this sits above it
49
- # so that the cap bounds a PROCESS -- which runs many passes and many runs --
50
- # without ever biting inside one.
51
- #
52
- # That "without ever biting" is the requirement, not a nicety. A `fetch_url`
53
- # page-2 request that misses degrades to a re-fetch, which is exactly the
54
- # spliced-document failure §51 measured and built PG5 to prevent, so an
55
- # eviction that fires in ordinary use would trade a bounded memory cost for
56
- # an unbounded correctness one.
57
- #
58
- # In entries and not in bytes, because a generic cache cannot measure the
59
- # size of what a caller hands it. For the caller that made this necessary the
60
- # two are interchangeable anyway: `fetch_url`'s MAX_CONTENT_BYTES holds each
61
- # body to ~64 KB whatever the encoding, so PG6's refusal to raise that cap is
62
- # what turns this count into a memory bound at all.
63
- MAX_ENTRIES = 128
64
-
65
-
66
- # ROADMAP_v3 §73 (A58). How long a URL this harness is willing to SHOW the
67
- # model may be.
68
- #
69
- # DERIVED, NOT PICKED, and the derivation is the argument for dropping an
70
- # over-long URL rather than cutting it: `fetch_url` already bounds the URL
71
- # the model may ASK for, at `max_length=400` on `FetchURLParams.url`.
72
- # Measured (batch 156), a 401-character URL comes back from fetch_url's own
73
- # gate as an "Invalid arguments" refusal while a 400-character one is
74
- # accepted. So a longer URL is one THIS HARNESS refuses to take, and
75
- # offering it in a search result is offering a route to nowhere.
76
- #
77
- # ONE DEFINITION: `fetch_url` reads it from here rather than spelling 400
78
- # again. The two were the same number in two files, which is the state a
79
- # pair of numbers is in just before it stops being a pair.
80
- MAX_URL_CHARS = 400
81
-
82
-
83
- class ThirdPartyText:
84
- """One tool call's worth of strings nobody here wrote, REDACTED and
85
- THEN cut.
86
-
87
- ORDER IS THE WHOLE POINT, and it is RB1's point restated for the two
88
- tools RB1 did not reach. `fetch_url` redacts its whole body above both
89
- the hold and the slice, because a pattern only matches what it can see
90
- whole; `web_search` and `arxiv_search` cut first and let
91
- `registry.dispatch`'s `check_output_policy` redact the piece that was
92
- kept. A fixed-length credential straddling such a cut is then invisible
93
- to both halves -- the kept prefix is not the token and the dropped
94
- suffix is gone. Measured (batch 156) at all three of their cut points:
95
- a `ghp_` token survived post-hoc redaction at 36 of the 39 offsets it
96
- could straddle, and redacting the whole value first caught every one.
97
-
98
- TWO KINDS OF VALUE, because a cut means different things to them (A58).
99
- `text()` cuts and flags: half an abstract is still an abstract, and PG4
100
- says the flag is owed. `url()` DROPS and flags: half a URL is not a
101
- URL, it is a fetchable-looking string that is not the resource, and
102
- `pdf_url` and arXiv's `full_*_url` pair exist precisely to be the route
103
- the model takes to recover what was cut. A broken route offered as a
104
- route is worse than an absent one, which the flag makes visible.
105
-
106
- TOTAL OVER WHATEVER THE PROVIDER EMITTED, for the reason `tools/base.py`
107
- gives about tool arguments: a key that is present with a wrong-typed
108
- value is not an absent key, and `r.get("title", "(untitled)")` returns
109
- None for `{"title": null}` rather than the default. Both accessors
110
- answer None for a non-string instead of raising, and the caller decides
111
- what an absent value means -- which is a decision the two tools make
112
- differently and should (A61).
113
-
114
- ONE WARNING PER CALL, not per value (#49, RB2). Redacting here means
115
- `check_output_policy` finds nothing left to alter, so the obligation to
116
- say that something WAS altered moves here with the redaction; and a
117
- feed with twenty-five authors must not produce twenty-five warning
118
- lines. The count is carried on the instance and `report()` spends it.
119
- The match is never echoed, for the reason `check_output_policy` and
120
- `fetch_url._redacted` both state: this function exists because
121
- something credential-shaped was found, so quoting it to say so would
122
- put it in app.log and every sink a WARNING reaches.
123
- """
124
-
125
- def __init__(self, tool: str) -> None:
126
- self._tool = tool
127
- self._redacted = 0
128
-
129
- def text(self, value: Any, cap: int) -> tuple:
130
- """`(text, truncated)` -- `value` redacted and THEN cut to `cap`.
131
-
132
- `(None, False)` for anything that is not a string, so a caller can
133
- tell "the provider did not give me this" from "it gave me an empty
134
- one" and choose its own substitute.
135
-
136
- THE FLAG COUNTS REDACTED CHARACTERS, which is RB3's consequence
137
- arriving here: redaction can SHRINK a value past the cap, and when
138
- it does there is nothing further the cut is withholding. A flag
139
- measured on the raw length would claim there is.
140
- """
141
- if not isinstance(value, str):
142
- return None, False
143
- clean = self._clean(value)
144
- return clean[:cap], len(clean) > cap
145
-
146
- def url(self, value: Any, cap: int = MAX_URL_CHARS) -> tuple:
147
- """`(url, withheld)` -- a URL past `cap` is DROPPED, not cut (A58).
148
-
149
- `(None, False)` for a non-string or an empty one: nothing was
150
- withheld, there was nothing there. `(None, True)` says a URL was
151
- found and is not being shown, which is the fact the flag carries.
152
- """
153
- if not isinstance(value, str) or not value:
154
- return None, False
155
- clean = self._clean(value)
156
- if len(clean) > cap:
157
- return None, True
158
- return clean, False
159
-
160
- def _clean(self, value: str) -> str:
161
- clean = redact_output_text(value)
162
- if clean != value:
163
- self._redacted += 1
164
- return clean
165
-
166
- def report(self) -> None:
167
- """The one warning this call owes (#49), or silence."""
168
- if self._redacted:
169
- logger.warning(
170
- "Output policy altered %s result: credentials replaced in "
171
- "%d value(s) before they were cut.",
172
- self._tool, self._redacted)
173
-
174
-
175
- class TTLCache:
176
- """A tiny time-boxed cache, keyed by whatever string a caller builds.
177
-
178
- TWO BOUNDS, ANSWERING DIFFERENT QUESTIONS. The TTL is about FRESHNESS: a
179
- stale entry is dropped as it is read, which is what makes a repeated
180
- query after the TTL a real request again rather than a permanent miss
181
- against a growing dict. `max_entries` is about SIZE, and it is the only
182
- thing that bounds a cache nothing reads a second time.
183
-
184
- THE SWEEP ON WRITE IS §52 (RB5), AND THE REASON THIS DOCSTRING USED TO
185
- GIVE FOR NOT HAVING ONE IS NOW FALSE. It said a sweep "would be machinery
186
- for a problem neither tool has" -- true of two callers holding result
187
- lists, and not of the third, which holds bodies. Measured (batch 106):
188
- 100 fetches of a capped page held 6.3 MB and not one entry was ever
189
- dropped, because expiry ran only in `get` and nothing read those keys
190
- again. Expiring on read alone means the TTL bounds staleness and nothing
191
- whatever bounds memory.
192
-
193
- READ IS DELIBERATELY UNCHANGED. `get` still expires the one key it
194
- touches, because that path is what makes a post-TTL query a real request
195
- again; the sweep is additional, not a replacement, and a cache that only
196
- swept would keep serving a stale entry until someone else wrote.
197
- """
198
-
199
- def __init__(self, ttl_s: float, max_entries: int = MAX_ENTRIES) -> None:
200
- self.ttl_s = ttl_s
201
- self.max_entries = max_entries
202
- self._entries: dict[str, tuple[float, Any]] = {}
203
- # LOCKED BECAUSE THE REGISTRY ALREADY SAYS IT IS (batch 117).
204
- # `spawn_subagent` is the one tool with `parallel=True`, and the
205
- # comment beside that field claims "everything the child's run
206
- # touches was made concurrency-safe first -- the budget's lock, the
207
- # serialised question, the per-future authorization". The child
208
- # reaches this cache through `arxiv_search` and `fetch_url`, and it
209
- # was the one item on that list without a lock.
210
- #
211
- # Single dict operations are atomic under CPython, so a corrupt
212
- # dict was never the risk. `set` is a read-modify-write -- pop,
213
- # assign, sweep, then evict while over the cap -- so two children
214
- # storing at once could lose one entry and evict in an order
215
- # neither asked for. Mirrors `GrantBudget.take()`, which is the
216
- # pattern this module should have followed from the start.
217
- #
218
- # NOT reentrant, and `_sweep` is the reason: it is called from
219
- # `set` with the lock already held, so it must never take it.
220
- self._lock = threading.Lock()
221
-
222
- def get(self, key: str) -> Optional[Any]:
223
- with self._lock:
224
- entry = self._entries.get(key)
225
- if not entry:
226
- return None
227
- stored_at, value = entry
228
- if time.time() - stored_at > self.ttl_s:
229
- self._entries.pop(key, None)
230
- return None
231
- return value
232
-
233
- def _sweep(self, now: float) -> None:
234
- """Drop everything already expired. Write-side only (§52, RB5).
235
-
236
- CALLED WITH `self._lock` HELD, and it must not take it: the lock is
237
- a plain `Lock`, so re-entering would deadlock the caller.
238
- """
239
- stale = [key for key, (stored_at, _) in self._entries.items()
240
- if now - stored_at > self.ttl_s]
241
- for key in stale:
242
- del self._entries[key]
243
-
244
- def set(self, key: str, value: Any) -> None:
245
- with self._lock:
246
- self._set_locked(key, value)
247
-
248
- def _set_locked(self, key: str, value: Any) -> None:
249
- now = time.time()
250
- # POPPED FIRST so a re-store moves the key to the END of the dict.
251
- # Assigning to an existing key keeps its original insertion position,
252
- # which would make the freshest entry the next one evicted -- and the
253
- # freshest entry is the one a caller is most likely to read next.
254
- self._entries.pop(key, None)
255
- self._entries[key] = (now, value)
256
- self._sweep(now)
257
- # Insertion order IS age order, since every entry is stamped as it is
258
- # written and the TTL is uniform. So the oldest key is simply the
259
- # first one, and no second ordering has to be kept consistent.
260
- while len(self._entries) > self.max_entries:
261
- self._entries.pop(next(iter(self._entries)))
262
-
263
- def clear(self) -> None:
264
- """Drop everything. For tests, which must not inherit a previous
265
- test's answers -- the same reason conftest isolates the stores."""
266
- with self._lock:
267
- self._entries.clear()
1
+ """
2
+ tools/builtin/_net_common.py
3
+
4
+ What the two network-backed search tools share. `_math_common.py`'s
5
+ precedent, one directory over: a private module the built-in tools import
6
+ from, never registered and never advertised.
7
+
8
+ TWO THINGS NOW. The first is the in-process response cache: `web_search.py`
9
+ and `arxiv.py` each carried a `_cache_get` / `_cache_set` pair that was
10
+ identical apart from the type of the value being stored, plus a
11
+ `CACHE_TTL_S = 300` each, plus a bare dict each. Three copies of an expiry
12
+ rule is how one of them comes to be measured in minutes.
13
+
14
+ The second is `ThirdPartyText` (batch 156, ROADMAP_v3 §73), which is where
15
+ the two search tools now bound and redact the strings they did not write.
16
+ It is here rather than in either tool for the reason the cache is: both
17
+ were doing the same thing to the same kind of value, and the one that got
18
+ it wrong got it wrong in the half the other had already fixed.
19
+
20
+ There are THREE callers now -- `fetch_url.py` joined them in ROADMAP_v3 §51 --
21
+ and the third is why §52 gave this class a size bound as well as a time one.
22
+
23
+ THE THIRD THING IS THE NETWORK METER (ROADMAP_v3 §53, batch 239), and it is
24
+ here for IO2's reason: this is the one module every network-backed tool
25
+ already imports, `core/reasoning/scholar.py` included, so a second home
26
+ would be a second place to forget. Three meters -- tools, the provider API
27
+ and MCP over HTTP -- each with its own ceiling over one rolling window,
28
+ counted into a small machine-wide SQLite ledger. Its section, at the end of
29
+ this file, says what is counted exactly, what is an estimate, and what
30
+ nothing here can see.
31
+
32
+ WHAT IS DELIBERATELY *NOT* SHARED. Each tool keeps its OWN instance, so the
33
+ two key spaces stay separate -- a `TTLCache` shared between them would let
34
+ an arXiv query and a web query collide on an equal key string, which is a
35
+ correctness bug rather than a tidiness one. Their `@field_validator`s
36
+ (`strip_keywords`, `strip_query`) are also left alone: they are three lines
37
+ each, they differ in the message the model is shown, and folding them into
38
+ a factory would make the params class harder to read to save two lines.
39
+ """
40
+
41
+ from __future__ import annotations
42
+
43
+ import atexit
44
+ import contextlib
45
+ import json
46
+ import logging
47
+ import os
48
+ import sqlite3
49
+ import threading
50
+ import time
51
+ from dataclasses import dataclass
52
+ from functools import cache
53
+ from typing import Any, Optional
54
+
55
+ import config
56
+ from safety.policy_enforcement import redact_output_text
57
+ from security import protected_paths
58
+
59
+ logger = logging.getLogger(__name__)
60
+
61
+ # ROADMAP_v3 §52 (RB6). How many entries any one of these may hold.
62
+ #
63
+ # Derived rather than picked: `config.MAX_ITERATIONS` is 50, so one pass can
64
+ # make at most 50 tool calls, and a `fetch_url` whose URL redirects stores
65
+ # TWO keys for one body (the requested spelling and the answering one). 100
66
+ # keys is therefore the most a single pass can reach, and this sits above it
67
+ # so that the cap bounds a PROCESS -- which runs many passes and many runs --
68
+ # without ever biting inside one.
69
+ #
70
+ # That "without ever biting" is the requirement, not a nicety. A `fetch_url`
71
+ # page-2 request that misses degrades to a re-fetch, which is exactly the
72
+ # spliced-document failure §51 measured and built PG5 to prevent, so an
73
+ # eviction that fires in ordinary use would trade a bounded memory cost for
74
+ # an unbounded correctness one.
75
+ #
76
+ # In entries and not in bytes, because a generic cache cannot measure the
77
+ # size of what a caller hands it. For the caller that made this necessary the
78
+ # two are interchangeable anyway: `fetch_url`'s MAX_CONTENT_BYTES holds each
79
+ # body to ~64 KB whatever the encoding, so PG6's refusal to raise that cap is
80
+ # what turns this count into a memory bound at all.
81
+ MAX_ENTRIES = 128
82
+
83
+
84
+ # ROADMAP_v3 §73 (A58). How long a URL this harness is willing to SHOW the
85
+ # model may be.
86
+ #
87
+ # DERIVED, NOT PICKED, and the derivation is the argument for dropping an
88
+ # over-long URL rather than cutting it: `fetch_url` already bounds the URL
89
+ # the model may ASK for, at `max_length=400` on `FetchURLParams.url`.
90
+ # Measured (batch 156), a 401-character URL comes back from fetch_url's own
91
+ # gate as an "Invalid arguments" refusal while a 400-character one is
92
+ # accepted. So a longer URL is one THIS HARNESS refuses to take, and
93
+ # offering it in a search result is offering a route to nowhere.
94
+ #
95
+ # ONE DEFINITION: `fetch_url` reads it from here rather than spelling 400
96
+ # again. The two were the same number in two files, which is the state a
97
+ # pair of numbers is in just before it stops being a pair.
98
+ MAX_URL_CHARS = 400
99
+
100
+
101
+ # Batch 240. Who this harness says it is when it fetches a page itself.
102
+ # Wikimedia answers httpx's default `python-httpx/<version>` with 403 --
103
+ # measured on en.wikipedia.org, where a descriptive agent got 200 -- and its
104
+ # policy asks a client to name itself and a way to reach its maintainers.
105
+ # The project's own public repository is that way; nothing about the user
106
+ # is in it. `fetch_url` and the scholar lookup send it; `arxiv_search` keeps
107
+ # the agent arXiv's API was already given.
108
+ USER_AGENT = ("venastine-research-harness "
109
+ "(+https://github.com/YahyaHammad/Venastine-Research-Harness)")
110
+
111
+
112
+ class ThirdPartyText:
113
+ """One tool call's worth of strings nobody here wrote, REDACTED and
114
+ THEN cut.
115
+
116
+ ORDER IS THE WHOLE POINT, and it is RB1's point restated for the two
117
+ tools RB1 did not reach. `fetch_url` redacts its whole body above both
118
+ the hold and the slice, because a pattern only matches what it can see
119
+ whole; `web_search` and `arxiv_search` cut first and let
120
+ `registry.dispatch`'s `check_output_policy` redact the piece that was
121
+ kept. A fixed-length credential straddling such a cut is then invisible
122
+ to both halves -- the kept prefix is not the token and the dropped
123
+ suffix is gone. Measured (batch 156) at all three of their cut points:
124
+ a `ghp_` token survived post-hoc redaction at 36 of the 39 offsets it
125
+ could straddle, and redacting the whole value first caught every one.
126
+
127
+ TWO KINDS OF VALUE, because a cut means different things to them (A58).
128
+ `text()` cuts and flags: half an abstract is still an abstract, and PG4
129
+ says the flag is owed. `url()` DROPS and flags: half a URL is not a
130
+ URL, it is a fetchable-looking string that is not the resource, and
131
+ `pdf_url` and arXiv's `full_*_url` pair exist precisely to be the route
132
+ the model takes to recover what was cut. A broken route offered as a
133
+ route is worse than an absent one, which the flag makes visible.
134
+
135
+ TOTAL OVER WHATEVER THE PROVIDER EMITTED, for the reason `tools/base.py`
136
+ gives about tool arguments: a key that is present with a wrong-typed
137
+ value is not an absent key, and `r.get("title", "(untitled)")` returns
138
+ None for `{"title": null}` rather than the default. Both accessors
139
+ answer None for a non-string instead of raising, and the caller decides
140
+ what an absent value means -- which is a decision the two tools make
141
+ differently and should (A61).
142
+
143
+ ONE WARNING PER CALL, not per value (#49, RB2). Redacting here means
144
+ `check_output_policy` finds nothing left to alter, so the obligation to
145
+ say that something WAS altered moves here with the redaction; and a
146
+ feed with twenty-five authors must not produce twenty-five warning
147
+ lines. The count is carried on the instance and `report()` spends it.
148
+ The match is never echoed, for the reason `check_output_policy` and
149
+ `fetch_url._redacted` both state: this function exists because
150
+ something credential-shaped was found, so quoting it to say so would
151
+ put it in app.log and every sink a WARNING reaches.
152
+ """
153
+
154
+ def __init__(self, tool: str) -> None:
155
+ self._tool = tool
156
+ self._redacted = 0
157
+
158
+ def text(self, value: Any, cap: int) -> tuple:
159
+ """`(text, truncated)` -- `value` redacted and THEN cut to `cap`.
160
+
161
+ `(None, False)` for anything that is not a string, so a caller can
162
+ tell "the provider did not give me this" from "it gave me an empty
163
+ one" and choose its own substitute.
164
+
165
+ THE FLAG COUNTS REDACTED CHARACTERS, which is RB3's consequence
166
+ arriving here: redaction can SHRINK a value past the cap, and when
167
+ it does there is nothing further the cut is withholding. A flag
168
+ measured on the raw length would claim there is.
169
+ """
170
+ if not isinstance(value, str):
171
+ return None, False
172
+ clean = self._clean(value)
173
+ return clean[:cap], len(clean) > cap
174
+
175
+ def url(self, value: Any, cap: int = MAX_URL_CHARS) -> tuple:
176
+ """`(url, withheld)` -- a URL past `cap` is DROPPED, not cut (A58).
177
+
178
+ `(None, False)` for a non-string or an empty one: nothing was
179
+ withheld, there was nothing there. `(None, True)` says a URL was
180
+ found and is not being shown, which is the fact the flag carries.
181
+ """
182
+ if not isinstance(value, str) or not value:
183
+ return None, False
184
+ clean = self._clean(value)
185
+ if len(clean) > cap:
186
+ return None, True
187
+ return clean, False
188
+
189
+ def _clean(self, value: str) -> str:
190
+ clean = redact_output_text(value)
191
+ if clean != value:
192
+ self._redacted += 1
193
+ return clean
194
+
195
+ def report(self) -> None:
196
+ """The one warning this call owes (#49), or silence."""
197
+ if self._redacted:
198
+ logger.warning(
199
+ "Output policy altered %s result: credentials replaced in "
200
+ "%d value(s) before they were cut.",
201
+ self._tool, self._redacted)
202
+
203
+
204
+ class TTLCache:
205
+ """A tiny time-boxed cache, keyed by whatever string a caller builds.
206
+
207
+ TWO BOUNDS, ANSWERING DIFFERENT QUESTIONS. The TTL is about FRESHNESS: a
208
+ stale entry is dropped as it is read, which is what makes a repeated
209
+ query after the TTL a real request again rather than a permanent miss
210
+ against a growing dict. `max_entries` is about SIZE, and it is the only
211
+ thing that bounds a cache nothing reads a second time.
212
+
213
+ THE SWEEP ON WRITE IS §52 (RB5), AND THE REASON THIS DOCSTRING USED TO
214
+ GIVE FOR NOT HAVING ONE IS NOW FALSE. It said a sweep "would be machinery
215
+ for a problem neither tool has" -- true of two callers holding result
216
+ lists, and not of the third, which holds bodies. Measured (batch 106):
217
+ 100 fetches of a capped page held 6.3 MB and not one entry was ever
218
+ dropped, because expiry ran only in `get` and nothing read those keys
219
+ again. Expiring on read alone means the TTL bounds staleness and nothing
220
+ whatever bounds memory.
221
+
222
+ READ IS DELIBERATELY UNCHANGED. `get` still expires the one key it
223
+ touches, because that path is what makes a post-TTL query a real request
224
+ again; the sweep is additional, not a replacement, and a cache that only
225
+ swept would keep serving a stale entry until someone else wrote.
226
+ """
227
+
228
+ def __init__(self, ttl_s: float, max_entries: int = MAX_ENTRIES) -> None:
229
+ self.ttl_s = ttl_s
230
+ self.max_entries = max_entries
231
+ self._entries: dict[str, tuple[float, Any]] = {}
232
+ # LOCKED BECAUSE THE REGISTRY ALREADY SAYS IT IS (batch 117).
233
+ # `spawn_subagent` is the one tool with `parallel=True`, and the
234
+ # comment beside that field claims "everything the child's run
235
+ # touches was made concurrency-safe first -- the budget's lock, the
236
+ # serialised question, the per-future authorization". The child
237
+ # reaches this cache through `arxiv_search` and `fetch_url`, and it
238
+ # was the one item on that list without a lock.
239
+ #
240
+ # Single dict operations are atomic under CPython, so a corrupt
241
+ # dict was never the risk. `set` is a read-modify-write -- pop,
242
+ # assign, sweep, then evict while over the cap -- so two children
243
+ # storing at once could lose one entry and evict in an order
244
+ # neither asked for. Mirrors `GrantBudget.take()`, which is the
245
+ # pattern this module should have followed from the start.
246
+ #
247
+ # NOT reentrant, and `_sweep` is the reason: it is called from
248
+ # `set` with the lock already held, so it must never take it.
249
+ self._lock = threading.Lock()
250
+
251
+ def get(self, key: str) -> Optional[Any]:
252
+ with self._lock:
253
+ entry = self._entries.get(key)
254
+ if not entry:
255
+ return None
256
+ stored_at, value = entry
257
+ if time.time() - stored_at > self.ttl_s:
258
+ self._entries.pop(key, None)
259
+ return None
260
+ return value
261
+
262
+ def _sweep(self, now: float) -> None:
263
+ """Drop everything already expired. Write-side only (§52, RB5).
264
+
265
+ CALLED WITH `self._lock` HELD, and it must not take it: the lock is
266
+ a plain `Lock`, so re-entering would deadlock the caller.
267
+ """
268
+ stale = [key for key, (stored_at, _) in self._entries.items()
269
+ if now - stored_at > self.ttl_s]
270
+ for key in stale:
271
+ del self._entries[key]
272
+
273
+ def set(self, key: str, value: Any) -> None:
274
+ with self._lock:
275
+ self._set_locked(key, value)
276
+
277
+ def _set_locked(self, key: str, value: Any) -> None:
278
+ now = time.time()
279
+ # POPPED FIRST so a re-store moves the key to the END of the dict.
280
+ # Assigning to an existing key keeps its original insertion position,
281
+ # which would make the freshest entry the next one evicted -- and the
282
+ # freshest entry is the one a caller is most likely to read next.
283
+ self._entries.pop(key, None)
284
+ self._entries[key] = (now, value)
285
+ self._sweep(now)
286
+ # Insertion order IS age order, since every entry is stamped as it is
287
+ # written and the TTL is uniform. So the oldest key is simply the
288
+ # first one, and no second ordering has to be kept consistent.
289
+ while len(self._entries) > self.max_entries:
290
+ self._entries.pop(next(iter(self._entries)))
291
+
292
+ def clear(self) -> None:
293
+ """Drop everything. For tests, which must not inherit a previous
294
+ test's answers -- the same reason conftest isolates the stores."""
295
+ with self._lock:
296
+ self._entries.clear()
297
+
298
+
299
+ # ===========================================================================
300
+ # ---- The network meter (ROADMAP_v3 §53, batch 239) ------------------------
301
+ # ===========================================================================
302
+ #
303
+ # UNLIMITED BY DEFAULT, AND THEN NOTHING HERE DOES ANYTHING (IO1). Every
304
+ # ceiling in config.yaml ships at 0. A meter with no ceiling admits every
305
+ # call, counts nothing and never opens the ledger, so a harness nobody has
306
+ # configured behaves exactly as it did before this section existed and
307
+ # writes no file for it.
308
+ #
309
+ # WHAT IS COUNTED, AND HOW EXACTLY (IO3, IO4). A request and its response as
310
+ # HTTP/1.1 frames them: the request line, the headers, the body, the status
311
+ # line, the response headers and the body AS IT CAME OFF THE WIRE --
312
+ # compressed, and without chunk framing. Measured (batch 239) against a
313
+ # local server: a 50,000-byte page served gzipped counts as its 143 wire
314
+ # bytes, not its 50,000 decoded ones, because that is what an allowance is
315
+ # charged. Every redirect hop is counted, its body too.
316
+ #
317
+ # WHAT IS NOT (IO5), so a ceiling is never read as more than it is: TLS and
318
+ # TCP overhead, HTTP/2's header compression, bytes the kernel had already
319
+ # received when a read stopped early, sign-in and token-refresh traffic
320
+ # (`/connect` and the ChatGPT plan's refresh), a `stdio` MCP server's own
321
+ # sockets, and everything a shell command does. `web_search` is an
322
+ # ESTIMATE (IO15), kept apart so every figure that includes it can say so.
323
+
324
+ #: The three meters, in the order every listing shows them (IO1).
325
+ METERS = ("tools", "provider", "mcp")
326
+
327
+ #: What a person reads for each, in a refusal and in `/network`.
328
+ METER_LABELS = {"tools": "tool", "provider": "provider API", "mcp": "MCP"}
329
+
330
+ #: A ceiling is in DECIMAL megabytes (IO9), which is how a data allowance
331
+ #: is sold and how a carrier counts it.
332
+ MEGABYTE = 1_000_000
333
+
334
+ #: The ledger is written at most this often (IO12), and only when something
335
+ #: was counted. It is also how stale another process's figures may get
336
+ #: before this one reads the file again.
337
+ FLUSH_INTERVAL_S = 60.0
338
+
339
+ #: The ledger holds at most this many buckets per meter (IO10). A bucket is
340
+ #: the window divided by this, never narrower than MIN_BUCKET_S, so the
341
+ #: file's size is bounded by the bucket count and not by the traffic: 50 MB
342
+ #: a day and 50 GB a day both add to one integer per bucket.
343
+ LEDGER_BUCKETS = 1440
344
+ MIN_BUCKET_S = 60
345
+
346
+ LEDGER_FILENAME = "network.db"
347
+
348
+ #: IO15, amending IO3, the owner's choice. What one `web_search` costs on
349
+ #: the wire, per result requested. Measured (batch 239) through a counting
350
+ #: CONNECT proxy on two live queries: 85,746 bytes for five results and
351
+ #: 206,637 for ten, against 1,967 and 2,445 bytes of rows returned -- so
352
+ #: IO3's "the size of what it returned" would have undercounted the tool
353
+ #: by 44x and 85x. The higher ratio, rounded up, so the estimate leans
354
+ #: high rather than low. Re-measure when `ddgs` moves: which backends it
355
+ #: queries decides this number.
356
+ WEB_SEARCH_BYTES_PER_RESULT = 21_000
357
+
358
+ #: How long a ledger write waits on another process's lock. Short, because
359
+ #: a writer holds it for one tiny transaction.
360
+ _BUSY_TIMEOUT_S = 2.0
361
+
362
+ #: The WAL file is cut back to this after a checkpoint, so it cannot keep
363
+ #: the size of the busiest minute it ever saw.
364
+ _JOURNAL_LIMIT_BYTES = 256 * 1024
365
+
366
+ _SCHEMA = (
367
+ "CREATE TABLE IF NOT EXISTS usage ("
368
+ " meter TEXT NOT NULL,"
369
+ " bucket INTEGER NOT NULL,"
370
+ " bytes INTEGER NOT NULL,"
371
+ " estimated INTEGER NOT NULL DEFAULT 0,"
372
+ " PRIMARY KEY (meter, bucket)) WITHOUT ROWID")
373
+
374
+ # An ADD, not a replace: two processes flushing the same bucket must both
375
+ # count. This is why the ledger is SQLite rather than a JSON file written
376
+ # atomically -- an atomic replace of a read-modify-write still loses the
377
+ # other writer's increment.
378
+ _UPSERT = (
379
+ "INSERT INTO usage (meter, bucket, bytes, estimated) VALUES (?, ?, ?, ?)"
380
+ " ON CONFLICT (meter, bucket) DO UPDATE SET"
381
+ " bytes = bytes + excluded.bytes,"
382
+ " estimated = estimated + excluded.estimated")
383
+
384
+
385
+ def window_s() -> float:
386
+ """The rolling window, in seconds, read at call time (IO7, IO8)."""
387
+ return float(config.NETWORK_WINDOW_HOURS) * 3600.0
388
+
389
+
390
+ def bucket_width_s() -> int:
391
+ """How wide one ledger bucket is for the current window (IO10)."""
392
+ return max(MIN_BUCKET_S, int(window_s() // LEDGER_BUCKETS))
393
+
394
+
395
+ class NetworkLimitReached(Exception):
396
+ """A call refused because its meter has no room for it (IO6).
397
+
398
+ RAISED, NOT RETURNED, so that one boundary per consumer decides what it
399
+ becomes, and no caller has to remember to check: `registry.dispatch`
400
+ turns it into the tool result the model reads, `core/provider_errors`
401
+ makes it a failure that is never retried and is described in these
402
+ words even under an SDK that wrapped it, and `mcp_client` returns it in
403
+ its one error shape. `reason` is the sentence all three show.
404
+ """
405
+
406
+ def __init__(self, reason: str, meter: str) -> None:
407
+ super().__init__(reason)
408
+ self.reason = reason
409
+ self.meter = meter
410
+
411
+
412
+ @dataclass(frozen=True)
413
+ class MeterState:
414
+ """One meter, as `/network`, the sidebar and a refusal read it."""
415
+
416
+ name: str
417
+ label: str
418
+ ceiling: int # bytes; 0 means this meter is off
419
+ used: int # bytes in the window, estimate included
420
+ estimated: int # the part of `used` that is an estimate
421
+ window_s: float
422
+ room_at: Optional[float] # when room returns, if the meter is full
423
+ ledger_note: Optional[str] # why the ledger is not in use, if it is not
424
+
425
+ @property
426
+ def enabled(self) -> bool:
427
+ return self.ceiling > 0
428
+
429
+ @property
430
+ def full(self) -> bool:
431
+ return self.enabled and self.used >= self.ceiling
432
+
433
+
434
+ def format_bytes(n: int) -> str:
435
+ """`12.3 MB` -- decimal units, to match the ceiling's (IO9)."""
436
+ if n < 1000:
437
+ return f"{n} B"
438
+ if n < MEGABYTE:
439
+ return f"{n / 1000:.1f} kB"
440
+ if n < 1000 * MEGABYTE:
441
+ return f"{n / MEGABYTE:.1f} MB"
442
+ return f"{n / (1000 * MEGABYTE):.2f} GB"
443
+
444
+
445
+ def format_bytes_short(n: int) -> str:
446
+ """`12M` -- the sidebar's spelling, which has 22 columns to work in."""
447
+ if n < 1000:
448
+ return f"{n}"
449
+ if n < MEGABYTE:
450
+ return f"{n / 1000:.0f}k"
451
+ if n < 10 * MEGABYTE:
452
+ return f"{n / MEGABYTE:.1f}M"
453
+ if n < 1000 * MEGABYTE:
454
+ return f"{n / MEGABYTE:.0f}M"
455
+ return f"{n / (1000 * MEGABYTE):.1f}G"
456
+
457
+
458
+ def describe_window(seconds: float) -> str:
459
+ """`24 h`, or `7 days` for a whole number of days past two."""
460
+ hours = seconds / 3600.0
461
+ if hours >= 48 and hours % 24 == 0:
462
+ return f"{hours / 24:g} days"
463
+ return f"{hours:g} h"
464
+
465
+
466
+ def _clock(at: float) -> str:
467
+ """A local time, with the date only when it is not today."""
468
+ when = time.localtime(at)
469
+ if time.strftime("%Y-%m-%d", when) == time.strftime("%Y-%m-%d"):
470
+ return time.strftime("%H:%M", when)
471
+ return time.strftime("%Y-%m-%d %H:%M", when)
472
+
473
+
474
+ def _room_at(buckets: list, used: int, ceiling: int, outgoing: int,
475
+ window: float, width: int) -> Optional[float]:
476
+ """When a request sending `outgoing` bytes would next be admitted.
477
+
478
+ The oldest bytes leave first, so walk the buckets oldest to newest and
479
+ find the first whose departure brings the meter under the line. None
480
+ when the request is larger than the whole ceiling, which no wait fixes.
481
+ """
482
+ need = max(outgoing, 0)
483
+ if need > ceiling:
484
+ return None
485
+ remaining = used
486
+ for bucket, nbytes, _ in buckets:
487
+ remaining -= nbytes
488
+ if remaining < ceiling and remaining + need <= ceiling:
489
+ return bucket + width + window
490
+ return None
491
+
492
+
493
+ class Ledger:
494
+ """The machine-wide record the meters count into (IO10, IO12).
495
+
496
+ ONE FILE FOR EVERY HARNESS PROCESS, `~/.config/venastine/network.db`,
497
+ because IO7 rejects a reset per run and a restart is a run: a counter
498
+ that lived in memory would be emptied by quitting, which is the
499
+ short-run loophole IO7 names. Every subagent shares it for the same
500
+ reason -- they run in this process and count into this instance.
501
+
502
+ COUNTS, NEVER TRAFFIC, AND BOUNDED BY TIME RATHER THAN BY VOLUME. A row
503
+ is (meter, bucket start, bytes, estimated bytes). Rows older than the
504
+ window are deleted on every write and SQLite reuses the freed pages, so
505
+ the file stops at about `LEDGER_BUCKETS` rows per meter whatever the
506
+ traffic was.
507
+
508
+ WRITTEN AT MOST ONCE A MINUTE, AND NEVER WHILE IDLE (IO12, the owner's
509
+ concern about write wear). Counts gather in memory; the first one after
510
+ a flush starts a one-shot timer, and the flush it fires is one
511
+ transaction of a few rows. With nothing counted there is no timer and no
512
+ write, and with every meter off there is no file at all. A refusal
513
+ decision reads memory, never the disk; the file is read again only when
514
+ the last read is more than a minute old.
515
+
516
+ THE COSTS, STATED: another process's traffic can be up to a minute late
517
+ here, and a crash loses at most the last minute of counts -- an
518
+ UNDER-count, which the gap register records. A file that cannot be
519
+ read or written does not turn the meter off: this process's own counts
520
+ still enforce, and `note` says the ledger is not in use.
521
+ """
522
+
523
+ def __init__(self, path: Optional[str] = None, clock=time.time,
524
+ timers: bool = True) -> None:
525
+ self._path = path
526
+ self._clock = clock
527
+ self._timers = timers
528
+ # (meter, bucket) -> [bytes, estimated]. `_stored` is what the file
529
+ # said at the last read, every process's flushed counts included;
530
+ # `_pending` is this process's, not yet written. Disjoint by
531
+ # construction: a flush moves pending into the file and reads the
532
+ # file back, so nothing is ever in both.
533
+ self._stored: dict = {}
534
+ self._pending: dict = {}
535
+ self._read_at: Optional[float] = None
536
+ self._timer: Optional[threading.Timer] = None
537
+ self._note: Optional[str] = None
538
+ # NOT reentrant, for TTLCache's reason: every `_locked` method is
539
+ # called with it held and must not take it again.
540
+ self._lock = threading.Lock()
541
+
542
+ # -- reading ---------------------------------------------------------
543
+
544
+ def now(self) -> float:
545
+ return self._clock()
546
+
547
+ def path(self) -> str:
548
+ """Resolved at CALL time, so a redirected HOME is honoured."""
549
+ return self._path or os.path.join(
550
+ protected_paths.user_config_dir(), LEDGER_FILENAME)
551
+
552
+ @property
553
+ def note(self) -> Optional[str]:
554
+ return self._note
555
+
556
+ def usage(self, meter: str, now: Optional[float] = None) -> list:
557
+ """`[(bucket, bytes, estimated)]` inside the window, oldest first.
558
+
559
+ A bucket STRADDLING the window's old edge is counted whole, so the
560
+ figure errs high by at most one bucket (1/1440 of the window) and
561
+ never low -- a ceiling should refuse a moment early rather than
562
+ admit a moment late.
563
+ """
564
+ now = self._clock() if now is None else now
565
+ with self._lock:
566
+ if self._read_at is None or now - self._read_at >= FLUSH_INTERVAL_S:
567
+ self._refresh_locked(now)
568
+ edge = now - window_s() - bucket_width_s()
569
+ merged: dict = {}
570
+ for source in (self._stored, self._pending):
571
+ for (name, bucket), (nbytes, estimated) in source.items():
572
+ if name == meter and bucket > edge:
573
+ slot = merged.setdefault(bucket, [0, 0])
574
+ slot[0] += nbytes
575
+ slot[1] += estimated
576
+ return sorted((bucket, n, e) for bucket, (n, e) in merged.items())
577
+
578
+ # -- counting --------------------------------------------------------
579
+
580
+ def add(self, meter: str, nbytes: int, estimated: bool = False,
581
+ now: Optional[float] = None) -> None:
582
+ now = self._clock() if now is None else now
583
+ width = bucket_width_s()
584
+ bucket = int(now // width * width)
585
+ with self._lock:
586
+ slot = self._pending.setdefault((meter, bucket), [0, 0])
587
+ slot[0] += nbytes
588
+ if estimated:
589
+ slot[1] += nbytes
590
+ self._schedule_locked()
591
+
592
+ def flush(self) -> None:
593
+ """Write what this process counted, then read every process's back.
594
+
595
+ Called by the timer, at exit, and by tests. A failure keeps the
596
+ counts pending for the next attempt and says why, once.
597
+ """
598
+ with self._lock:
599
+ self._timer = None
600
+ if not self._pending:
601
+ return
602
+ now = self._clock()
603
+ try:
604
+ self._write_locked(now)
605
+ except (sqlite3.Error, OSError) as exc:
606
+ self._fail_locked("written", exc)
607
+ # Bounded even while the file is failing: a bucket that has
608
+ # left the window is no use to anyone.
609
+ edge = now - window_s() - 2 * bucket_width_s()
610
+ self._pending = {key: value
611
+ for key, value in self._pending.items()
612
+ if key[1] > edge}
613
+
614
+ def cancel(self) -> None:
615
+ """Stop a scheduled flush. For tests, which replace the ledger."""
616
+ with self._lock:
617
+ if self._timer is not None:
618
+ self._timer.cancel()
619
+ self._timer = None
620
+
621
+ # -- the file --------------------------------------------------------
622
+
623
+ def _schedule_locked(self) -> None:
624
+ if self._timers and self._timer is None:
625
+ timer = threading.Timer(FLUSH_INTERVAL_S, self.flush)
626
+ timer.daemon = True
627
+ self._timer = timer
628
+ timer.start()
629
+
630
+ def _connect(self, create: bool) -> Optional[sqlite3.Connection]:
631
+ path = self.path()
632
+ if not create and not os.path.exists(path):
633
+ return None
634
+ if create:
635
+ os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
636
+ # Autocommit, so the one transaction below is spelled out rather
637
+ # than opened implicitly by the first statement.
638
+ db = sqlite3.connect(path, timeout=_BUSY_TIMEOUT_S,
639
+ isolation_level=None)
640
+ try:
641
+ if create:
642
+ # WAL with synchronous=NORMAL: a commit appends to the log
643
+ # and does not wait on fsync; only a checkpoint does. The
644
+ # last connection to close checkpoints and removes the log.
645
+ db.execute("PRAGMA journal_mode=WAL")
646
+ db.execute("PRAGMA synchronous=NORMAL")
647
+ db.execute(f"PRAGMA journal_size_limit={_JOURNAL_LIMIT_BYTES}")
648
+ db.execute(_SCHEMA)
649
+ except BaseException:
650
+ db.close()
651
+ raise
652
+ return db
653
+
654
+ def _select_locked(self, db: sqlite3.Connection, edge: float) -> dict:
655
+ rows = db.execute(
656
+ "SELECT meter, bucket, bytes, estimated FROM usage"
657
+ " WHERE bucket > ?", (int(edge),)).fetchall()
658
+ return {(meter, bucket): [nbytes, estimated]
659
+ for meter, bucket, nbytes, estimated in rows}
660
+
661
+ def _refresh_locked(self, now: float) -> None:
662
+ # Stamped even when the read fails, so a broken file is tried once
663
+ # a minute rather than on every call.
664
+ self._read_at = now
665
+ edge = now - window_s() - bucket_width_s()
666
+ try:
667
+ db = self._connect(create=False)
668
+ if db is None:
669
+ self._stored = {}
670
+ return
671
+ try:
672
+ self._stored = self._select_locked(db, edge)
673
+ finally:
674
+ db.close()
675
+ except (sqlite3.Error, OSError) as exc:
676
+ self._fail_locked("read", exc)
677
+
678
+ def _write_locked(self, now: float) -> None:
679
+ width = bucket_width_s()
680
+ # Two buckets of margin past the window's edge: the straddling one
681
+ # `usage` still counts, and one more so a clock that has moved on
682
+ # between processes cannot delete a row another still reads.
683
+ horizon = int(now - window_s()) - 2 * width
684
+ rows = [(meter, bucket, nbytes, estimated)
685
+ for (meter, bucket), (nbytes, estimated)
686
+ in self._pending.items() if bucket > horizon]
687
+ db = self._connect(create=True)
688
+ try:
689
+ db.execute("BEGIN IMMEDIATE")
690
+ try:
691
+ db.executemany(_UPSERT, rows)
692
+ db.execute("DELETE FROM usage WHERE bucket <= ?", (horizon,))
693
+ db.execute("COMMIT")
694
+ except BaseException:
695
+ with contextlib.suppress(sqlite3.Error):
696
+ db.execute("ROLLBACK")
697
+ raise
698
+ stored = self._select_locked(db, now - window_s() - width)
699
+ finally:
700
+ db.close()
701
+ self._pending.clear()
702
+ self._stored = stored
703
+ self._read_at = now
704
+ if self._note is not None:
705
+ logger.warning("The network ledger at %s is in use again.",
706
+ self.path())
707
+ self._note = None
708
+
709
+ def _fail_locked(self, verb: str, exc: BaseException) -> None:
710
+ note = (f"the ledger at {self.path()} could not be {verb} "
711
+ f"({type(exc).__name__}: {exc}); counting this process only")
712
+ if note != self._note:
713
+ logger.warning("Network meter: %s.", note)
714
+ self._note = note
715
+
716
+
717
+ #: The one ledger every meter in this process counts into. Replaced, never
718
+ #: mutated, by the tests that turn a meter on (tests/conftest.py).
719
+ LEDGER = Ledger()
720
+
721
+
722
+ def _ledger() -> Ledger:
723
+ """The CURRENT ledger, looked up at call time so a test's swap is seen."""
724
+ return LEDGER
725
+
726
+
727
+ @atexit.register
728
+ def _flush_at_exit() -> None:
729
+ """The last minute's counts, written as the process leaves (IO12)."""
730
+ with contextlib.suppress(Exception):
731
+ _ledger().flush()
732
+
733
+
734
+ class Meter:
735
+ """One of the three (IO1): a ceiling, and the arithmetic against it.
736
+
737
+ THE CEILING IS READ AT CALL TIME from `config`, like every other
738
+ setting a test may monkeypatch; nothing here caches it.
739
+
740
+ ADMISSION, NEVER TRUNCATION (IO6, IO11). `admit` is asked before a call
741
+ starts, with the size of what the call will SEND when that is known,
742
+ and refuses when the meter is full or the send would pass the ceiling.
743
+ A response already in flight is never cut: it completes and is counted,
744
+ so a meter can end a window over its ceiling by at most one response --
745
+ and a half-read body is the spliced-document failure PG5 exists to
746
+ prevent.
747
+ """
748
+
749
+ def __init__(self, name: str) -> None:
750
+ self.name = name
751
+ self.label = METER_LABELS[name]
752
+ # Whether the last decision was a refusal, so a run of them warns
753
+ # once (the transcript line IO6 owes) and not once per call.
754
+ self._refusing = False
755
+ self._lock = threading.Lock()
756
+
757
+ def ceiling(self) -> int:
758
+ megabytes = getattr(config, f"NETWORK_LIMIT_{self.name.upper()}_MB", 0)
759
+ # Rounded, not truncated: 2.01 MB is 2,009,999.9999999998 bytes in
760
+ # binary floating point, and a ceiling must not shrink in the
761
+ # conversion.
762
+ return int(round(float(megabytes or 0) * MEGABYTE))
763
+
764
+ def enabled(self) -> bool:
765
+ return self.ceiling() > 0
766
+
767
+ def _measure(self) -> tuple:
768
+ ceiling = self.ceiling()
769
+ window = window_s()
770
+ if not ceiling:
771
+ return MeterState(self.name, self.label, 0, 0, 0, window,
772
+ None, None), []
773
+ ledger = _ledger()
774
+ buckets = ledger.usage(self.name)
775
+ used = sum(nbytes for _, nbytes, _ in buckets)
776
+ estimated = sum(part for _, _, part in buckets)
777
+ room = (_room_at(buckets, used, ceiling, 0, window, bucket_width_s())
778
+ if used >= ceiling else None)
779
+ return MeterState(self.name, self.label, ceiling, used, estimated,
780
+ window, room, ledger.note), buckets
781
+
782
+ def state(self) -> MeterState:
783
+ return self._measure()[0]
784
+
785
+ def admit(self, outgoing: int = 0) -> None:
786
+ """Refuse, by raising NetworkLimitReached, or return and admit."""
787
+ if not self.enabled():
788
+ return
789
+ state, buckets = self._measure()
790
+ if state.used < state.ceiling and state.used + outgoing <= state.ceiling:
791
+ with self._lock:
792
+ self._refusing = False
793
+ return
794
+ reason = self._refusal(state, buckets, outgoing)
795
+ with self._lock:
796
+ first, self._refusing = not self._refusing, True
797
+ if first:
798
+ logger.warning("Network limit: %s", reason)
799
+ raise NetworkLimitReached(reason, self.name)
800
+
801
+ def record(self, nbytes: int, estimated: bool = False) -> None:
802
+ if nbytes > 0 and self.enabled():
803
+ _ledger().add(self.name, int(nbytes), estimated)
804
+
805
+ def _refusal(self, state: MeterState, buckets: list, outgoing: int) -> str:
806
+ ceiling, window = state.ceiling, describe_window(state.window_s)
807
+ if outgoing > ceiling:
808
+ text = (f"The {self.label} network limit cannot admit this "
809
+ f"request: it sends {format_bytes(outgoing)}, more than "
810
+ f"the whole {format_bytes(ceiling)} allowed per {window}.")
811
+ elif state.used >= ceiling:
812
+ text = (f"The {self.label} network limit is reached: "
813
+ f"{format_bytes(state.used)} of {format_bytes(ceiling)} "
814
+ f"used in the last {window}.")
815
+ else:
816
+ text = (f"The {self.label} network limit would be passed: this "
817
+ f"request sends {format_bytes(outgoing)} and "
818
+ f"{format_bytes(ceiling - state.used)} of "
819
+ f"{format_bytes(ceiling)} is left for the last {window}.")
820
+ if state.estimated:
821
+ text += (f" {format_bytes(state.estimated)} of the use is an "
822
+ f"estimate (web_search).")
823
+ room = _room_at(buckets, state.used, ceiling, outgoing,
824
+ state.window_s, bucket_width_s())
825
+ if room is not None:
826
+ text += f" Room returns at about {_clock(room)}."
827
+ text += (f" The ceiling is network_limit_{self.name}_mb in "
828
+ f"config.yaml.")
829
+ if self.name != "provider":
830
+ # The model reads this one. The limit is the user's, and the
831
+ # shell is where an agent would reach next (gap register).
832
+ text += " It is the user's limit: do not work around it."
833
+ return text
834
+
835
+
836
+ TOOLS = Meter("tools")
837
+ PROVIDER = Meter("provider")
838
+ MCP = Meter("mcp")
839
+ _BY_NAME = {meter.name: meter for meter in (TOOLS, PROVIDER, MCP)}
840
+
841
+
842
+ def meter(name: str) -> Meter:
843
+ return _BY_NAME[name]
844
+
845
+
846
+ def states() -> list:
847
+ """Every meter's state, in METERS order."""
848
+ return [_BY_NAME[name].state() for name in METERS]
849
+
850
+
851
+ # ---- Sizing an exchange ----------------------------------------------------
852
+ #
853
+ # DUCK-TYPED over httpx and httpx2, which share these attribute names: the
854
+ # tools' own calls are httpx, the provider SDKs and MCP are httpx2. Total
855
+ # over a wrong shape -- every accessor is guarded, and a part that cannot
856
+ # be read counts as zero -- because sizing a request must never be the
857
+ # reason it fails.
858
+
859
+ _HTTP11_REQUEST_TAIL = len(" HTTP/1.1\r\n")
860
+
861
+
862
+ def _headers_bytes(headers) -> int:
863
+ raw = getattr(headers, "raw", None)
864
+ if raw is not None:
865
+ pairs = list(raw)
866
+ elif hasattr(headers, "items"):
867
+ pairs = list(headers.items())
868
+ else:
869
+ return 0
870
+ total = 2 # the blank line
871
+ for key, value in pairs:
872
+ total += len(key) + len(value) + 4 # ": " and CRLF
873
+ return total
874
+
875
+
876
+ def request_bytes(request) -> int:
877
+ """One request as HTTP/1.1 frames it: line, headers and body."""
878
+ if request is None:
879
+ return 0
880
+ try:
881
+ target = getattr(request.url, "raw_path", b"/")
882
+ total = len(request.method) + 1 + len(target) + _HTTP11_REQUEST_TAIL
883
+ total += _headers_bytes(request.headers)
884
+ length = request.headers.get("content-length")
885
+ if length is not None:
886
+ total += int(length)
887
+ else:
888
+ with contextlib.suppress(Exception):
889
+ total += len(request.content)
890
+ return total
891
+ except Exception: # noqa: BLE001 -- see the section comment
892
+ return 0
893
+
894
+
895
+ def response_head_bytes(response) -> int:
896
+ """The status line and headers. The body is counted as it is read."""
897
+ try:
898
+ reason = getattr(response, "reason_phrase", "") or ""
899
+ line = len("HTTP/1.1 000 \r\n") + len(reason)
900
+ return line + _headers_bytes(response.headers)
901
+ except Exception: # noqa: BLE001
902
+ return 0
903
+
904
+
905
+ def _request_of(response):
906
+ """The request a response answered, or None.
907
+
908
+ A PROPERTY THAT RAISES rather than an attribute that may be missing:
909
+ httpx's `Response.request` raises RuntimeError when no request was set,
910
+ so `getattr` with a default does not cover it.
911
+ """
912
+ try:
913
+ return response.request
914
+ except Exception: # noqa: BLE001 -- see the section comment
915
+ return None
916
+
917
+
918
+ def exchange_bytes(response) -> int:
919
+ """Every hop of a response the CALLER read: request, head and body.
920
+
921
+ `num_bytes_downloaded` is httpx's count of body bytes as received --
922
+ before decompression, after chunk framing -- which is the wire figure
923
+ (measured, above). `history` holds the redirect hops httpx followed.
924
+ """
925
+ total = 0
926
+ for hop in [*getattr(response, "history", ()), response]:
927
+ total += request_bytes(_request_of(hop))
928
+ total += response_head_bytes(hop)
929
+ total += int(getattr(hop, "num_bytes_downloaded", 0) or 0)
930
+ return total
931
+
932
+
933
+ # ---- The tools' calls (IO3) ------------------------------------------------
934
+ #
935
+ # `httpx` is imported INSIDE each call, for two reasons: scholar's lookup
936
+ # imports it lazily so a run with `scholar_lookup` off never loads it, and
937
+ # the suite installs a fake `httpx` module these must reach (the root
938
+ # conftest). The module function is looked up per call, so a test's fake is
939
+ # what runs.
940
+
941
+ @contextlib.contextmanager
942
+ def stream(method: str, url: str, **kwargs):
943
+ """`httpx.stream`, admitted and counted against the tool meter."""
944
+ import httpx
945
+
946
+ TOOLS.admit()
947
+ with httpx.stream(method, url, **kwargs) as response:
948
+ try:
949
+ yield response
950
+ finally:
951
+ if TOOLS.enabled():
952
+ TOOLS.record(exchange_bytes(response))
953
+
954
+
955
+ def get(url: str, **kwargs):
956
+ """`httpx.get`, admitted and counted against the tool meter."""
957
+ import httpx
958
+
959
+ TOOLS.admit()
960
+ response = httpx.get(url, **kwargs)
961
+ if TOOLS.enabled():
962
+ TOOLS.record(exchange_bytes(response))
963
+ return response
964
+
965
+
966
+ def record_web_search(results_requested: int) -> None:
967
+ """IO15's estimate for one search, which has no client to count."""
968
+ if TOOLS.enabled():
969
+ TOOLS.record(max(int(results_requested), 1)
970
+ * WEB_SEARCH_BYTES_PER_RESULT, estimated=True)
971
+
972
+
973
+ # ---- The provider SDKs and MCP (IO4) ---------------------------------------
974
+ #
975
+ # EVENT HOOKS ON AN httpx2 CLIENT, not a wrapped transport, and both halves
976
+ # of the reason were measured (batch 239, httpx2 2.16.0). With HTTPS_PROXY
977
+ # set and no transport passed, the client MOUNTS a proxy transport of its
978
+ # own and a proxied request goes through that, which a wrapper on the
979
+ # default transport never sees. Pass a transport instead, and the client
980
+ # drops the environment's proxy altogether -- a meter that silently breaks
981
+ # the user's proxy. Hooks run for every hop, whatever carries it. Measured
982
+ # the same day against all three SDKs: anthropic, openai and google-genai
983
+ # each call both hooks for an ordinary request, a stream and (google) an
984
+ # embedding.
985
+ #
986
+ # THE BODY IS COUNTED BY REPLACING `response.stream` in the response hook,
987
+ # which runs before anything reads the body, with a stream that counts each
988
+ # raw chunk as it passes.
989
+
990
+ @cache
991
+ def _stream_classes() -> tuple:
992
+ """The two counting streams, built on first use so `httpx2` is only
993
+ imported by a process that has a meter on."""
994
+ import httpx2
995
+
996
+ class CountingStream(httpx2.SyncByteStream):
997
+ def __init__(self, inner, meter: Meter) -> None:
998
+ self._inner = inner
999
+ self._meter = meter
1000
+
1001
+ def __iter__(self):
1002
+ for chunk in self._inner:
1003
+ self._meter.record(len(chunk))
1004
+ yield chunk
1005
+
1006
+ def close(self) -> None:
1007
+ self._inner.close()
1008
+
1009
+ class AsyncCountingStream(httpx2.AsyncByteStream):
1010
+ def __init__(self, inner, meter: Meter) -> None:
1011
+ self._inner = inner
1012
+ self._meter = meter
1013
+
1014
+ async def __aiter__(self):
1015
+ async for chunk in self._inner:
1016
+ self._meter.record(len(chunk))
1017
+ yield chunk
1018
+
1019
+ async def aclose(self) -> None:
1020
+ await self._inner.aclose()
1021
+
1022
+ return CountingStream, AsyncCountingStream
1023
+
1024
+
1025
+ def provider_event_hooks() -> dict:
1026
+ """Hooks for a provider SDK's client: admit, then count (IO4, IO11).
1027
+
1028
+ THE REQUEST IS SIZED EXACTLY BEFORE IT IS SENT, so an upload that would
1029
+ pass the ceiling is refused whole. The provider's upload is the big
1030
+ one -- every step re-sends the thread -- which is why this meter
1031
+ admits on the outgoing size and not only on what is already used.
1032
+ """
1033
+ counting, _ = _stream_classes()
1034
+ meter = PROVIDER
1035
+
1036
+ def on_request(request) -> None:
1037
+ outgoing = request_bytes(request)
1038
+ meter.admit(outgoing)
1039
+ meter.record(outgoing)
1040
+
1041
+ def on_response(response) -> None:
1042
+ meter.record(response_head_bytes(response))
1043
+ response.stream = counting(response.stream, meter)
1044
+
1045
+ return {"request": [on_request], "response": [on_response]}
1046
+
1047
+
1048
+ def mcp_event_hooks() -> dict:
1049
+ """Hooks for an MCP server's async client: count, never refuse.
1050
+
1051
+ MCP's admission is `mcp_client.call_tool`'s, not this hook's. A refusal
1052
+ raised here would surface inside the SDK's background writer task,
1053
+ where it reads as a timeout rather than as a reason; refused at the
1054
+ call, it is the one error shape `call_tool` already returns.
1055
+ """
1056
+ _, counting = _stream_classes()
1057
+ meter = MCP
1058
+
1059
+ async def on_request(request) -> None:
1060
+ meter.record(request_bytes(request))
1061
+
1062
+ async def on_response(response) -> None:
1063
+ meter.record(response_head_bytes(response))
1064
+ response.stream = counting(response.stream, meter)
1065
+
1066
+ return {"request": [on_request], "response": [on_response]}
1067
+
1068
+
1069
+ def mcp_call_bytes(params: Any) -> int:
1070
+ """What a tool call will send, near enough to admit on."""
1071
+ try:
1072
+ return len(json.dumps(params, default=str).encode("utf-8"))
1073
+ except (TypeError, ValueError):
1074
+ return 0
1075
+
1076
+
1077
+ # ---- What a person reads (IO13) --------------------------------------------
1078
+
1079
+ #: IO5's list, said once and shown by `/network` whatever the meters say.
1080
+ NOT_COUNTED = (
1081
+ "Not counted: shell commands (containers, WSL, SSH), stdio MCP servers, "
1082
+ "sign-in and token-refresh traffic, and TLS/TCP overhead. web_search is "
1083
+ "an estimate, about 21 kB per result asked for.")
1084
+
1085
+ #: The sidebar's names, which have 22 columns to share.
1086
+ SHORT_LABELS = {"tools": "tools", "provider": "api", "mcp": "mcp"}
1087
+
1088
+
1089
+ def network_report() -> list:
1090
+ """`/network`'s lines, the same in both shells (ROADMAP_v3 §89's rule:
1091
+ the shells bind handlers, and what they print is decided here)."""
1092
+ rows = states()
1093
+ window = describe_window(window_s())
1094
+ if not any(row.enabled for row in rows):
1095
+ return [
1096
+ "No network limit is set: every meter is off and nothing is "
1097
+ "counted.",
1098
+ "To set one, give network_limit_tools_mb, "
1099
+ "network_limit_provider_mb or network_limit_mcp_mb a value in "
1100
+ f"config.yaml (decimal MB per {window}, network_window_hours).",
1101
+ NOT_COUNTED,
1102
+ ]
1103
+ width = max(len(row.label) for row in rows)
1104
+ lines = [f"Network use over the last {window}:"]
1105
+ for row in rows:
1106
+ if not row.enabled:
1107
+ lines.append(f" {row.label:<{width}} off")
1108
+ continue
1109
+ line = (f" {row.label:<{width}} {format_bytes(row.used)} of "
1110
+ f"{format_bytes(row.ceiling)}")
1111
+ if row.estimated:
1112
+ line += f" ({format_bytes(row.estimated)} of it estimated)"
1113
+ if row.full:
1114
+ line += " -- full"
1115
+ if row.room_at is not None:
1116
+ line += f", room returns at about {_clock(row.room_at)}"
1117
+ lines.append(line)
1118
+ note = next((row.ledger_note for row in rows if row.ledger_note), None)
1119
+ if note:
1120
+ lines.append(f"Ledger: {note}.")
1121
+ else:
1122
+ lines.append(f"Ledger: {_ledger().path()}, shared by every harness "
1123
+ "process and written at most once a minute.")
1124
+ lines.append(NOT_COUNTED)
1125
+ return lines
1126
+
1127
+
1128
+ def sidebar_text(rows: list) -> Optional[str]:
1129
+ """The sidebar line for these states, or None while every meter is off.
1130
+
1131
+ `~` marks a figure that includes an estimate, IO3's rule that the
1132
+ weaker number says so wherever it is shown.
1133
+ """
1134
+ on = [row for row in rows if row.enabled]
1135
+ if not on:
1136
+ return None
1137
+ parts = []
1138
+ for row in on:
1139
+ mark = "~" if row.estimated else ""
1140
+ full = " full" if row.full else ""
1141
+ parts.append(f"{SHORT_LABELS[row.name]} {mark}"
1142
+ f"{format_bytes_short(row.used)}/"
1143
+ f"{format_bytes_short(row.ceiling)}{full}")
1144
+ return ("net · " + " · ".join(parts)
1145
+ + f" · per {describe_window(on[0].window_s)}")