@yahyahammad/venastine 0.3.16 → 0.3.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONFIG_ARCHITECTURE.md +14 -8
- package/README.md +5 -5
- package/cli_commands.py +82 -1
- package/core/client.py +130 -11
- package/core/commands.py +7 -3
- package/core/loop.py +21 -1
- package/core/memory.py +13 -0
- package/core/reasoning/json_retry.py +148 -31
- package/core/reasoning/orchestrator.py +49 -3
- package/core/shell_sessions.py +67 -6
- package/core/usage_view.py +119 -0
- package/docs/THIRD_PARTY_NOTICES.md +1 -1
- package/logging_setup.py +35 -2
- package/main.py +184 -21
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/safety/policy_enforcement.py +79 -9
- package/security/sandbox.py +11 -2
- package/tools/builtin/arxiv.py +8 -2
- package/tools/builtin/fetch_url.py +8 -1
- package/tools/builtin/file_ops.py +74 -8
- package/tools/builtin/project_docs.py +10 -4
- package/tools/builtin/shell.py +2 -1
- package/tools/builtin/shell_sessions.py +6 -3
- package/tools/builtin/web_search.py +5 -1
- package/tools/isolation.py +9 -6
- package/tui/app.py +240 -12
- package/tui/meters.py +124 -40
- package/tui/widgets.py +8 -7
package/CONFIG_ARCHITECTURE.md
CHANGED
|
@@ -322,6 +322,10 @@ max corrective follow-up attempts when a pass returns malformed JSON
|
|
|
322
322
|
|
|
323
323
|
(total attempts per pass = MAX_JSON_RETRIES + 1). See ROADMAP §3.
|
|
324
324
|
|
|
325
|
+
Since batch 267 the same budget covers Pass 1 and Final synthesis coming back with no
|
|
326
|
+
text at all: each is asked again that many times, then the pass fails (ROADMAP_v3 §95,
|
|
327
|
+
RO6 amended).
|
|
328
|
+
|
|
325
329
|
## Spend cap (batch 27, #4)
|
|
326
330
|
|
|
327
331
|
There is NO default spend ceiling any more. The loop's cumulative
|
|
@@ -1361,11 +1365,13 @@ start at a time nobody chose.
|
|
|
1361
1365
|
### `shell_session_max_consecutive_wakes`
|
|
1362
1366
|
|
|
1363
1367
|
ROADMAP_v3 §49 (SS2). How many wake turns may run in a row with no message
|
|
1364
|
-
from the user. Every wake is a model call nobody typed,
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1368
|
+
from the user. Every wake is a model call nobody typed, so an agent that
|
|
1369
|
+
starts a new session on every wake would otherwise spend tokens
|
|
1370
|
+
indefinitely. Past the limit, waking stops, the commands the wait holds
|
|
1371
|
+
are released, and the results are delivered with the user's next message;
|
|
1372
|
+
a user message resets the count. (Since batch 265 a plain message is never
|
|
1373
|
+
held by live sessions: the user can type while they run, and each message
|
|
1374
|
+
gets a turn of its own -- ROADMAP_v3 §49, SS2 as amended.)
|
|
1369
1375
|
|
|
1370
1376
|
10 is a ceiling on an unattended loop, not an estimate of a normal one. A
|
|
1371
1377
|
subagent has no user to reset it, so it gets one final wake and returns
|
|
@@ -1985,9 +1991,9 @@ for `shell`'s reason and not a weaker one: a session IS a shell command, and
|
|
|
1985
1991
|
one that keeps running after the call that started it has been answered.
|
|
1986
1992
|
Enabling them without `shell` is allowed and coherent -- a session is
|
|
1987
1993
|
classified, approved and routed by exactly the same rules -- but a session
|
|
1988
|
-
also holds a container for as long as its timeout,
|
|
1989
|
-
while it runs, and can wake the agent,
|
|
1990
|
-
less.
|
|
1994
|
+
also holds a container for as long as its timeout, holds the user's
|
|
1995
|
+
thread- and model-changing commands while it runs, and can wake the agent,
|
|
1996
|
+
so it is more authority rather than less.
|
|
1991
1997
|
|
|
1992
1998
|
The two that START one are approval-gated by `shell_approval_mode` alone
|
|
1993
1999
|
(SS16) and have no `tool_approvals` key, exactly like `shell`; the three
|
package/README.md
CHANGED
|
@@ -774,7 +774,7 @@ paths are cached and 200 matches offered; narrow the query when a limit is shown
|
|
|
774
774
|
| `/threads` | Conversations-only picker — research runs' internal threads excluded — most recently active first, capped at 200 with a notice (**ctrl+t**) |
|
|
775
775
|
| `/resume <thread-id>` | Open any thread by id, however old |
|
|
776
776
|
| `/new` | Start a fresh thread; created lazily on your next message, so `/new` twice leaves nothing behind |
|
|
777
|
-
| `/kill [session-id]` | Stop a background session. Bare, it opens the picker **ctrl+b** opens; with an id it stops that one. Deliberately usable *while* sessions are
|
|
777
|
+
| `/kill [session-id]` | Stop a background session. Bare, it opens the picker **ctrl+b** opens; with an id it stops that one. Deliberately usable *while* sessions are running and holding the other commands — it is the way out of that, so it is the one command that never refuses for being busy. A plain message is not held: it gets a turn of its own while they run |
|
|
778
778
|
| `/agent [name\|default]` | Make an agent active for subsequent turns; `default` clears. Active skills are re-checked against the new agent's tool policy |
|
|
779
779
|
| `/goal [text\|clear]` | Persistent objective for this thread — injected into every turn in either shell, mirrored in a banner |
|
|
780
780
|
| `/grill-me` | One turn under the grill-me agent inside the current thread, reading the live history rather than a digest of it |
|
|
@@ -930,7 +930,7 @@ Subagents spawned in one model response run **concurrently**, up to `subagent_ma
|
|
|
930
930
|
|
|
931
931
|
### Logging
|
|
932
932
|
|
|
933
|
-
Two sinks from startup: stderr, and a rotating file — `logs/app.log` in the working directory, 1 MB per file, three backups kept. `AGENT_LOG_LEVEL` sets verbosity (a name like `INFO` or a number like `20`; anything else prints a warning naming the value and runs at `INFO`, as an unwritable `AGENT_LOG_FILE` does) and `AGENT_LOG_FILE` moves the file, creating the parent directory if needed. The file formatter redacts secret-shaped strings on the way out — the same patterns tool outputs get — because a failing tool's traceback routinely carries the request that produced it, and for an HTTP client that is a URL with the key in it.
|
|
933
|
+
Two sinks from startup: stderr, and a rotating file — `logs/app.log` in the working directory, 1 MB per file, three backups kept. `AGENT_LOG_LEVEL` sets verbosity (a name like `INFO` or a number like `20`; anything else prints a warning naming the value and runs at `INFO`, as an unwritable `AGENT_LOG_FILE` does) and `AGENT_LOG_FILE` moves the file, creating the parent directory if needed. The terminal shows WARNING and above, as the TUI's transcript does, and the file keeps INFO too; setting `AGENT_LOG_LEVEL` puts that level on the terminal as well. A tool failure the shell already prints as the call's result is logged at INFO, so it is said once. The file formatter redacts secret-shaped strings on the way out — the same patterns tool outputs get — because a failing tool's traceback routinely carries the request that produced it, and for an HTTP client that is a URL with the key in it.
|
|
934
934
|
|
|
935
935
|
In the TUI stderr is detached before the screen is taken (anything written there would paint over the rendered UI and vanish on the next repaint); the rotating file keeps everything, and WARNING-and-above is routed into the transcript so warnings stay visible where you are looking.
|
|
936
936
|
|
|
@@ -1009,7 +1009,7 @@ Deliberately not settings.json keys: editing these means editing the harness's o
|
|
|
1009
1009
|
| `attended_approval_timeout_s` | 600 | Attended prompt deadline; expiry denies that call while the run continues |
|
|
1010
1010
|
| `subagent_review` | `false` | Shipped default for the post-pipeline review |
|
|
1011
1011
|
| `max_review_findings` / `max_review_refinements` | 25 / 3 | Consent-fatigue caps on the review stage |
|
|
1012
|
-
| `max_pipeline_retries` / `max_json_retries` | 2 / 2 | Revise/re-validate rounds per claim; corrective attempts per malformed pass payload |
|
|
1012
|
+
| `max_pipeline_retries` / `max_json_retries` | 2 / 2 | Revise/re-validate rounds per claim; corrective attempts per malformed pass payload, or per empty Pass 1 or Final synthesis |
|
|
1013
1013
|
| `max_iterations` | 50 | Step ceiling for a turn or pass absent an agent's own `max_steps` |
|
|
1014
1014
|
| `subagent_max_depth` | 2 | `spawn_subagent` nesting limit |
|
|
1015
1015
|
| `subagent_max_parallel` | 3 | How many subagents of one model response run at once; 1 makes them sequential again |
|
|
@@ -1058,7 +1058,7 @@ Deliberately not settings.json keys: editing these means editing the harness's o
|
|
|
1058
1058
|
|
|
1059
1059
|
**Built:** ROADMAP.md §1–§12 (§10's ensemble mode rebuilt by its revisit — diversity now comes from a roster of different models) and ROADMAP_v2.md §13 (streaming loop), §14 (config loader + workspace trust), §15 (permissions), §16 (TUI), §17 (MCP), §18 (agents), §19 (skills), §20 (post-pipeline review), §21a (compaction + `pin`), §21b (durable memory), §22 (live pipeline progress), §25 (authorised tool use in the pipeline), §26 (research legibility: per-pass tool calls, code-stage announcements, the claims view, role colour, `/copy`), §27 (thread legibility: transcript replay on resume, a conversations-only thread picker), §21c (session summaries and cross-thread referencing — §21 is now complete), §24 (`/init`), §23 (interactive tools: the response channel, per-tool subagent sign-off, `ask_user`, and the todo list), §28 (the shell capability classifier).
|
|
1060
1060
|
|
|
1061
|
-
**Remaining:** nothing in `ROADMAP.md` or `ROADMAP_v2.md`, and §10's revisit is now closed. `ROADMAP_v3.md` §49–§59, §61–§93 and §
|
|
1061
|
+
**Remaining:** nothing in `ROADMAP.md` or `ROADMAP_v2.md`, and §10's revisit is now closed. `ROADMAP_v3.md` §49–§59, §61–§93, §95, §96 and §97 are built. The section still to build is §60 (Unified Harness Protocol compliance), specified and awaiting detailed design. §56 lets a decision model approve a single contained shell command under the opt-in `classified` mode; §54 completes project file/folder mentions without loading contents; §58 supplies a short harness identity that follows the current model; §83's final live checks are confirmed, and the ChatGPT header distinguishes a selected provider from an active sign-in. `TECHNICAL_DEBT.md` items 14, 17, 18, 21, 102, 116, 118, 119 and 120 are open, and so is the secrets half of item 97. The two checks recorded against §10 are answered from the providers' documentation in `ROADMAP_v3.md` §87: the default `temperature` on OpenAI and Google is 1.0, and OpenAI's reasoning models refuse one whenever reasoning effort is not `none`, which the guard now follows.
|
|
1062
1062
|
|
|
1063
1063
|
**And both audits are closed.** Both roadmaps being built is not the same as the code
|
|
1064
1064
|
being clean, and saying only the first would be the "built and runs are different claims"
|
|
@@ -1077,7 +1077,7 @@ note that shell approval is governed solely by `shell_approval_mode` -- `tool_ap
|
|
|
1077
1077
|
deliberately has no `shell` key, so an old file carrying one is refused at startup
|
|
1078
1078
|
naming the key — see `tests/BREAKING_CHANGES.md` §24.
|
|
1079
1079
|
|
|
1080
|
-
Run the test suite with `pytest` —
|
|
1080
|
+
Run the test suite with `pytest` — 9829 tests, fully offline, no API keys needed. Since batch 243 the suite holds itself to that: a test that asks DNS for a name, or writes into your home folder, fails by name, and every test process runs with HOME pointing at a throwaway folder of its own. The suite's own packages are `requirements-test.txt`, separate from `requirements.txt` since batch 136: they used to be in the shipped dependency list, so installing the harness also installed a test runner. It runs in parallel by default (`pytest-xdist`, 70% of your cores): **1:26 on a 16-core machine, against 9:06 single-process.** `pytest -n0` forces the old serial run, which is what you want under a debugger or when something is parsing the output. Eight further tests are marked `integration` and excluded by default: three spawn a real stdio MCP server, one runs an external program through the host fallback shell, one builds the hosted bundle's Docker image, and three run the installers' uv path against a real uv (`pytest -m integration`).
|
|
1081
1081
|
|
|
1082
1082
|
## Documentation
|
|
1083
1083
|
|
package/cli_commands.py
CHANGED
|
@@ -41,6 +41,7 @@ from uuid import UUID
|
|
|
41
41
|
|
|
42
42
|
from core import ansi
|
|
43
43
|
from core.commands import CATALOG, CLI, CommandRegistry, absent_reason, bind
|
|
44
|
+
from core.usage_view import UsageTally, report_lines
|
|
44
45
|
|
|
45
46
|
logger = logging.getLogger(__name__)
|
|
46
47
|
|
|
@@ -84,6 +85,12 @@ class ChatSession:
|
|
|
84
85
|
# Left for run_chat by /quit and /grill-me.
|
|
85
86
|
quit_requested: bool = False
|
|
86
87
|
one_shot: object = None
|
|
88
|
+
# ROADMAP_v3 §97 (UF4). `/usage`'s tally: the calls this shell made on
|
|
89
|
+
# `usage_thread` since it was opened here. The TUI's memory keeps one
|
|
90
|
+
# across turns; this shell builds a memory per turn, so the turns'
|
|
91
|
+
# tallies (`ModelResponse.turn_usage`) are added up here instead.
|
|
92
|
+
usage: object = field(default_factory=UsageTally)
|
|
93
|
+
usage_thread: Optional[UUID] = None
|
|
87
94
|
|
|
88
95
|
def memory(self):
|
|
89
96
|
"""The current thread, created if there is none yet.
|
|
@@ -99,6 +106,19 @@ class ChatSession:
|
|
|
99
106
|
self.thread_id = memory.thread_id
|
|
100
107
|
return memory
|
|
101
108
|
|
|
109
|
+
def count_usage(self, response) -> None:
|
|
110
|
+
"""Add a finished turn's calls to `/usage`'s tally (§97, UF4).
|
|
111
|
+
|
|
112
|
+
The tally starts again when the turn ran on another thread than
|
|
113
|
+
the one it was counting -- the first turn after /new, /resume or
|
|
114
|
+
/threads -- which is the TUI's span: a thread since it was opened.
|
|
115
|
+
"""
|
|
116
|
+
thread = getattr(response, "thread_id", None) or self.thread_id
|
|
117
|
+
if thread != self.usage_thread:
|
|
118
|
+
self.usage = UsageTally()
|
|
119
|
+
self.usage_thread = thread
|
|
120
|
+
self.usage.merge(getattr(response, "turn_usage", None))
|
|
121
|
+
|
|
102
122
|
|
|
103
123
|
def _out(text: str = "") -> None:
|
|
104
124
|
"""One line of a command's answer.
|
|
@@ -112,9 +132,57 @@ def _out(text: str = "") -> None:
|
|
|
112
132
|
|
|
113
133
|
# ---- the loop's entry point ---------------------------------------------------
|
|
114
134
|
|
|
115
|
-
|
|
135
|
+
#: The commands refused when they are sent while the chat waits on a turn's
|
|
136
|
+
#: background sessions, and what each would have done (ROADMAP_v3 §49, SS2
|
|
137
|
+
#: as amended in batch 265, the owner's choice). The CLI's half of the
|
|
138
|
+
#: TUI's `refuse_if_occupied`: the TUI refuses these while its wait runs,
|
|
139
|
+
#: because each changes what the wait is waiting FOR -- the thread, the
|
|
140
|
+
#: model, the history -- underneath it. `/grill-me` is one of them because
|
|
141
|
+
#: the TUI's refuses for its running turn, and its wait is one.
|
|
142
|
+
#: `/submodel` only when it would change something, as in the TUI.
|
|
143
|
+
REFUSED_WHILE_WAITING = {
|
|
144
|
+
"compact": "compact this thread",
|
|
145
|
+
"grill-me": "run /grill-me",
|
|
146
|
+
"model": "change the model",
|
|
147
|
+
"new": "start a new thread",
|
|
148
|
+
"resume": "switch threads",
|
|
149
|
+
"submodel": "change the submodel",
|
|
150
|
+
"threads": "switch threads",
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _refused_while_waiting(session: ChatSession, line: str) -> bool:
|
|
155
|
+
"""Whether `line`, sent during the session wait, is refused, saying why.
|
|
156
|
+
|
|
157
|
+
Asked only of a line the wait handed back: at the prompt, with nothing
|
|
158
|
+
waiting, there is no wait for a command to pull the ground from under,
|
|
159
|
+
and a refusal there would leave the user no way round it.
|
|
160
|
+
"""
|
|
161
|
+
from core import shell_sessions
|
|
162
|
+
|
|
163
|
+
name, _, args = line[1:].partition(" ")
|
|
164
|
+
action = REFUSED_WHILE_WAITING.get(name)
|
|
165
|
+
if action is None or (name == "submodel" and not args.strip()):
|
|
166
|
+
return False
|
|
167
|
+
thread_id = session.thread_id
|
|
168
|
+
sessions = shell_sessions.sessions
|
|
169
|
+
if thread_id is None or not sessions.blocks(thread_id):
|
|
170
|
+
return False
|
|
171
|
+
live = sessions.waited_on(thread_id)
|
|
172
|
+
plural = "" if live == 1 else "s"
|
|
173
|
+
_out(f"[{live} background session{plural} still running — {action} "
|
|
174
|
+
f"once they finish, or press Ctrl+C to stop them]")
|
|
175
|
+
return True
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def run_command(session: ChatSession, line: str, *,
|
|
179
|
+
waiting: bool = False) -> None:
|
|
116
180
|
"""Run one `/` line, and contain whatever the command does.
|
|
117
181
|
|
|
182
|
+
`waiting` (batch 265): the line was sent while the chat waited on
|
|
183
|
+
background sessions, and the commands in `REFUSED_WHILE_WAITING` are
|
|
184
|
+
refused rather than run.
|
|
185
|
+
|
|
118
186
|
CONTAINED HERE, as the TUI contains its own dispatch (batch 135): a
|
|
119
187
|
handler that raises must not end the conversation, and the traceback
|
|
120
188
|
goes to logs/app.log where a real bug stays findable. Ctrl+C during a
|
|
@@ -123,6 +191,8 @@ def run_command(session: ChatSession, line: str) -> None:
|
|
|
123
191
|
turn.
|
|
124
192
|
"""
|
|
125
193
|
name = line[1:].split(" ")[0]
|
|
194
|
+
if waiting and _refused_while_waiting(session, line):
|
|
195
|
+
return
|
|
126
196
|
try:
|
|
127
197
|
handled = registry.dispatch(session, line)
|
|
128
198
|
except KeyboardInterrupt:
|
|
@@ -689,6 +759,16 @@ def _cmd_network(session: ChatSession, args: str) -> None:
|
|
|
689
759
|
_out(line)
|
|
690
760
|
|
|
691
761
|
|
|
762
|
+
def _cmd_usage(session: ChatSession, args: str) -> None:
|
|
763
|
+
"""ROADMAP_v3 §97 (UF4). The TUI's /usage, line for line: both print
|
|
764
|
+
`core.usage_view.report_lines`. Empty until a turn has run on this
|
|
765
|
+
thread here, which is the span the TUI's memory covers."""
|
|
766
|
+
same = (session.thread_id is not None
|
|
767
|
+
and session.usage_thread == session.thread_id)
|
|
768
|
+
for line in report_lines(session.usage if same else None):
|
|
769
|
+
_out(line)
|
|
770
|
+
|
|
771
|
+
|
|
692
772
|
def _cmd_forget(session: ChatSession, args: str) -> None:
|
|
693
773
|
raw = args.strip()
|
|
694
774
|
if not raw:
|
|
@@ -852,6 +932,7 @@ def register_cli_commands() -> None:
|
|
|
852
932
|
bind("model", _cmd_model),
|
|
853
933
|
bind("network", _cmd_network),
|
|
854
934
|
bind("new", _cmd_new),
|
|
935
|
+
bind("usage", _cmd_usage),
|
|
855
936
|
bind("quit", _cmd_quit),
|
|
856
937
|
bind("ref", _cmd_ref),
|
|
857
938
|
bind("resume", _cmd_resume),
|
package/core/client.py
CHANGED
|
@@ -310,6 +310,17 @@ class ModelResponse:
|
|
|
310
310
|
turn_billed_tokens: Optional[int] = None
|
|
311
311
|
turn_new_tokens: Optional[int] = None
|
|
312
312
|
turn_output_tokens: Optional[int] = None
|
|
313
|
+
# ROADMAP_v3 §97 (UF4). The usage keys THIS call's provider did not
|
|
314
|
+
# send, which `_stream_usage` coerced to 0 so the loop's arithmetic
|
|
315
|
+
# stays integer (D21). `/usage` reads it so a coerced zero is never
|
|
316
|
+
# shown as a count. A cache count needs no entry here: it is put in
|
|
317
|
+
# `usage` only when reported, so its absence already says so.
|
|
318
|
+
usage_unreported: tuple = ()
|
|
319
|
+
# §97 (UF4). This run's calls as a `core.usage_view.UsageTally`, set by
|
|
320
|
+
# `run_agent_conversation` and `wake_conversation` -- whose memory is
|
|
321
|
+
# built for the one run, so its tally is this turn's -- for the CLI's
|
|
322
|
+
# `/usage`, which keeps no memory between turns. None elsewhere.
|
|
323
|
+
turn_usage: Optional[object] = None
|
|
313
324
|
# Batch 44 (§44). The turn's reasoning, as the PROVIDER'S OWN BLOCKS
|
|
314
325
|
# rather than as the text §38 streams for display:
|
|
315
326
|
#
|
|
@@ -497,6 +508,14 @@ def _stream_usage(provider_name: str, counts: dict,
|
|
|
497
508
|
With the flag FALSE the coercion is all that happens, and 0 is the
|
|
498
509
|
right answer there: the flag's whole purpose is to let a provider that
|
|
499
510
|
genuinely omits usage run anyway.
|
|
511
|
+
|
|
512
|
+
RETURNS THE COERCED KEYS TOO (ROADMAP_v3 §97, UF4): `(usage,
|
|
513
|
+
unreported)`, where `unreported` names each count this made a 0 of,
|
|
514
|
+
for `ModelResponse.usage_unreported`. The 0 is right for the loop's
|
|
515
|
+
sums and wrong for `/usage`, which shows only what the provider sent;
|
|
516
|
+
one return keeps the two from being decided in two places. A cache
|
|
517
|
+
count (`CACHE_COUNTS`) is carried through only when it was reported --
|
|
518
|
+
nothing adds it, so it needs no zero.
|
|
500
519
|
"""
|
|
501
520
|
reported_in = counts["input_tokens"]
|
|
502
521
|
reported_out = counts["output_tokens"]
|
|
@@ -518,8 +537,56 @@ def _stream_usage(provider_name: str, counts: dict,
|
|
|
518
537
|
f"supports_stream_usage=True, so this step's token "
|
|
519
538
|
f"accounting would be short by it."
|
|
520
539
|
)
|
|
521
|
-
|
|
522
|
-
|
|
540
|
+
usage = {"input_tokens": reported_in or 0,
|
|
541
|
+
"output_tokens": reported_out or 0}
|
|
542
|
+
for key in CACHE_COUNTS:
|
|
543
|
+
if counts.get(key) is not None:
|
|
544
|
+
usage[key] = counts[key]
|
|
545
|
+
unreported = tuple(name for name, value in
|
|
546
|
+
(("input_tokens", reported_in),
|
|
547
|
+
("output_tokens", reported_out)) if value is None)
|
|
548
|
+
return usage, unreported
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
#: The cache counts a usage dict may carry (ROADMAP_v3 §97, UF4/UF5), each
|
|
552
|
+
#: present only when the provider reported it. Spelled the same on every
|
|
553
|
+
#: branch, whatever the SDK calls it, so `core/usage_view.py` reads one key.
|
|
554
|
+
CACHE_COUNTS = ("cache_read_tokens", "cache_write_tokens")
|
|
555
|
+
|
|
556
|
+
#: The Google branch's usage fields, each read per chunk and kept per its
|
|
557
|
+
#: own last report (batch 147's rule), then combined at the end of the
|
|
558
|
+
#: stream (§97, UF2). `tests/test_sdk_conformance.py` asks the pinned SDK
|
|
559
|
+
#: for every one.
|
|
560
|
+
_GOOGLE_COUNTS = ("prompt_token_count", "candidates_token_count",
|
|
561
|
+
"thoughts_token_count", "cached_content_token_count")
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def _sum_reported(*values) -> Optional[int]:
|
|
565
|
+
"""The sum of the counts that were reported, or None when none was.
|
|
566
|
+
|
|
567
|
+
For a figure a provider splits across fields that are each nullable:
|
|
568
|
+
Google's output, which is its candidates PLUS its thoughts (§97 UF2).
|
|
569
|
+
Not `sum(v or 0 ...)`, which would turn "reported nothing" into a zero
|
|
570
|
+
before D21 has looked at it -- batch 147's defect, one helper over.
|
|
571
|
+
"""
|
|
572
|
+
reported = [value for value in values if value is not None]
|
|
573
|
+
return sum(reported) if reported else None
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def _whole_prompt(input_tokens, cache_read, cache_write) -> Optional[int]:
|
|
577
|
+
"""Input as the WHOLE prompt, on a branch that reports its cached part
|
|
578
|
+
apart (ROADMAP_v3 §97, UF5, the owner's choice).
|
|
579
|
+
|
|
580
|
+
The Anthropic SDK's `input_tokens` leaves out the tokens read from the
|
|
581
|
+
cache and the tokens written to it; the OpenAI-compatible and Google
|
|
582
|
+
counts include theirs. Adding them here, at the producer, is what makes
|
|
583
|
+
`billed`, `ctx` and the compaction trigger mean one thing on every
|
|
584
|
+
branch. None stays None: an input count the provider did not send is
|
|
585
|
+
D21's to judge, and a cache count alone is not an input count.
|
|
586
|
+
"""
|
|
587
|
+
if input_tokens is None:
|
|
588
|
+
return None
|
|
589
|
+
return input_tokens + (cache_read or 0) + (cache_write or 0)
|
|
523
590
|
|
|
524
591
|
|
|
525
592
|
# ============================================================================
|
|
@@ -2186,6 +2253,15 @@ def _chatgpt_stream(client, model: str, items: list, system_prompt: str,
|
|
|
2186
2253
|
reported = _token_count(usage_obj, key)
|
|
2187
2254
|
if reported is not None:
|
|
2188
2255
|
usage[key] = reported
|
|
2256
|
+
# §97 (UF4). Part of `input_tokens` already on this API,
|
|
2257
|
+
# so UF5 adds nothing here; only what was sent is kept.
|
|
2258
|
+
cached = getattr(usage_obj, "input_tokens_details", None)
|
|
2259
|
+
for key, attribute in (("cache_read_tokens", "cached_tokens"),
|
|
2260
|
+
("cache_write_tokens",
|
|
2261
|
+
"cache_write_tokens")):
|
|
2262
|
+
reported = _token_count(cached, attribute)
|
|
2263
|
+
if reported is not None:
|
|
2264
|
+
usage[key] = reported
|
|
2189
2265
|
details = getattr(response, "incomplete_details", None)
|
|
2190
2266
|
if getattr(details, "reason", None) == "max_output_tokens":
|
|
2191
2267
|
finish_reason = "length"
|
|
@@ -2206,7 +2282,7 @@ def _chatgpt_stream(client, model: str, items: list, system_prompt: str,
|
|
|
2206
2282
|
if callable(close):
|
|
2207
2283
|
close()
|
|
2208
2284
|
|
|
2209
|
-
usage = _stream_usage("CHATGPT", usage, supports_stream_usage)
|
|
2285
|
+
usage, unreported = _stream_usage("CHATGPT", usage, supports_stream_usage)
|
|
2210
2286
|
calls = []
|
|
2211
2287
|
for item in pending_calls:
|
|
2212
2288
|
name = getattr(item, "name", None) or ""
|
|
@@ -2221,6 +2297,7 @@ def _chatgpt_stream(client, model: str, items: list, system_prompt: str,
|
|
|
2221
2297
|
name=name, input=parsed, parse_error=parse_error))
|
|
2222
2298
|
yield StreamToken(final_response=ModelResponse(
|
|
2223
2299
|
text="".join(text_parts), tool_calls=calls, usage=usage,
|
|
2300
|
+
usage_unreported=unreported,
|
|
2224
2301
|
thinking=_thinking_record("CHATGPT", model, reasoning_items),
|
|
2225
2302
|
))
|
|
2226
2303
|
|
|
@@ -2396,9 +2473,19 @@ def call_model_stream(
|
|
|
2396
2473
|
if b.type in ("thinking", "redacted_thinking")
|
|
2397
2474
|
]
|
|
2398
2475
|
usage_obj = getattr(final_msg, "usage", None)
|
|
2476
|
+
# §97 (UF4, UF5). This SDK reports the cached part of the
|
|
2477
|
+
# prompt APART from `input_tokens`, so input is made the whole
|
|
2478
|
+
# prompt here, as every other branch's already is.
|
|
2479
|
+
cache_read = _token_count(usage_obj, "cache_read_input_tokens")
|
|
2480
|
+
cache_write = _token_count(usage_obj,
|
|
2481
|
+
"cache_creation_input_tokens")
|
|
2399
2482
|
usage = {
|
|
2400
|
-
"input_tokens":
|
|
2483
|
+
"input_tokens": _whole_prompt(
|
|
2484
|
+
_token_count(usage_obj, "input_tokens"),
|
|
2485
|
+
cache_read, cache_write),
|
|
2401
2486
|
"output_tokens": _token_count(usage_obj, "output_tokens"),
|
|
2487
|
+
"cache_read_tokens": cache_read,
|
|
2488
|
+
"cache_write_tokens": cache_write,
|
|
2402
2489
|
}
|
|
2403
2490
|
|
|
2404
2491
|
# D21 parity with the other two branches (audit #40). This branch
|
|
@@ -2421,10 +2508,12 @@ def call_model_stream(
|
|
|
2421
2508
|
# Batch 147: the rule itself, and the coercion that makes `usage`
|
|
2422
2509
|
# two ints, are in `_stream_usage` -- one copy for three branches
|
|
2423
2510
|
# that carried three.
|
|
2424
|
-
usage = _stream_usage(provider_name, usage,
|
|
2511
|
+
usage, unreported = _stream_usage(provider_name, usage,
|
|
2512
|
+
supports_stream_usage)
|
|
2425
2513
|
|
|
2426
2514
|
yield StreamToken(final_response=ModelResponse(
|
|
2427
2515
|
text=text, tool_calls=calls, usage=usage,
|
|
2516
|
+
usage_unreported=unreported,
|
|
2428
2517
|
thinking=_thinking_record(provider_name, model, thinking_blocks),
|
|
2429
2518
|
))
|
|
2430
2519
|
return
|
|
@@ -2450,6 +2539,13 @@ def call_model_stream(
|
|
|
2450
2539
|
# partial-report case the widened rule exists for arrives here
|
|
2451
2540
|
# already coerced and indistinguishable (batch 147).
|
|
2452
2541
|
usage = {"input_tokens": None, "output_tokens": None}
|
|
2542
|
+
# The provider's own fields, kept apart until the stream ends:
|
|
2543
|
+
# OUTPUT IS TWO OF THEM (§97, UF2). `candidates_token_count` leaves
|
|
2544
|
+
# the model's thoughts out -- the SDK's own `total_token_count` is
|
|
2545
|
+
# documented as prompt + candidates + tool-use prompt + thoughts --
|
|
2546
|
+
# and thoughts are generated and billed as output, so the rate and
|
|
2547
|
+
# `billed` read short by them on a thinking model until this batch.
|
|
2548
|
+
google_counts = dict.fromkeys(_GOOGLE_COUNTS)
|
|
2453
2549
|
|
|
2454
2550
|
for chunk in client.models.generate_content_stream(
|
|
2455
2551
|
model=model,
|
|
@@ -2489,22 +2585,33 @@ def call_model_stream(
|
|
|
2489
2585
|
# the provider chose to put the totals on.
|
|
2490
2586
|
usage_meta = getattr(chunk, "usage_metadata", None)
|
|
2491
2587
|
if usage_meta is not None:
|
|
2492
|
-
for
|
|
2493
|
-
("input_tokens", "prompt_token_count"),
|
|
2494
|
-
("output_tokens", "candidates_token_count")):
|
|
2588
|
+
for attribute in _GOOGLE_COUNTS:
|
|
2495
2589
|
reported = _token_count(usage_meta, attribute)
|
|
2496
2590
|
if reported is not None:
|
|
2497
|
-
|
|
2591
|
+
google_counts[attribute] = reported
|
|
2592
|
+
|
|
2593
|
+
# §97 (UF2, UF4). Each field kept per its own last report above,
|
|
2594
|
+
# then combined once: thoughts join the candidates in output
|
|
2595
|
+
# (`_sum_reported`, so neither alone being absent zeroes the other),
|
|
2596
|
+
# and the cached content is a cache read that `prompt_token_count`
|
|
2597
|
+
# already includes, so UF5 adds nothing to input on this branch.
|
|
2598
|
+
usage["input_tokens"] = google_counts["prompt_token_count"]
|
|
2599
|
+
usage["output_tokens"] = _sum_reported(
|
|
2600
|
+
google_counts["candidates_token_count"],
|
|
2601
|
+
google_counts["thoughts_token_count"])
|
|
2602
|
+
usage["cache_read_tokens"] = google_counts["cached_content_token_count"]
|
|
2498
2603
|
|
|
2499
2604
|
# D21, in the one place all three branches read it (batch 147).
|
|
2500
2605
|
# This is the branch the nullable-field defect was found in: the
|
|
2501
2606
|
# pinned google-genai declares every usage field Optional[int], so
|
|
2502
2607
|
# `usage` could leave here holding None and the raise could not
|
|
2503
2608
|
# fire -- `None != 0`.
|
|
2504
|
-
usage = _stream_usage(provider_name, usage,
|
|
2609
|
+
usage, unreported = _stream_usage(provider_name, usage,
|
|
2610
|
+
supports_stream_usage)
|
|
2505
2611
|
|
|
2506
2612
|
yield StreamToken(final_response=ModelResponse(
|
|
2507
2613
|
text="".join(text_parts), tool_calls=calls, usage=usage,
|
|
2614
|
+
usage_unreported=unreported,
|
|
2508
2615
|
thinking=_thinking_record(provider_name, model, signatures),
|
|
2509
2616
|
))
|
|
2510
2617
|
return
|
|
@@ -2605,12 +2712,23 @@ def call_model_stream(
|
|
|
2605
2712
|
reported = _token_count(usage_obj, attribute)
|
|
2606
2713
|
if reported is not None:
|
|
2607
2714
|
usage[key] = reported
|
|
2715
|
+
# §97 (UF4). Part of `prompt_tokens` already on this API, so
|
|
2716
|
+
# UF5 adds nothing here. A provider that spells its cache
|
|
2717
|
+
# counts some other way reports none, and /usage says so.
|
|
2718
|
+
cached = getattr(usage_obj, "prompt_tokens_details", None)
|
|
2719
|
+
for key, attribute in (("cache_read_tokens", "cached_tokens"),
|
|
2720
|
+
("cache_write_tokens",
|
|
2721
|
+
"cache_write_tokens")):
|
|
2722
|
+
reported = _token_count(cached, attribute)
|
|
2723
|
+
if reported is not None:
|
|
2724
|
+
usage[key] = reported
|
|
2608
2725
|
|
|
2609
2726
|
# D21: if the provider is marked as supporting stream usage but we
|
|
2610
2727
|
# got no usable usage, the stream was likely interrupted before the
|
|
2611
2728
|
# final usage chunk arrived. Raise rather than silently reporting zero.
|
|
2612
2729
|
# One copy for three branches since batch 147.
|
|
2613
|
-
usage = _stream_usage(provider_name, usage,
|
|
2730
|
+
usage, unreported = _stream_usage(provider_name, usage,
|
|
2731
|
+
supports_stream_usage)
|
|
2614
2732
|
|
|
2615
2733
|
calls = []
|
|
2616
2734
|
for idx in sorted(tool_fragments):
|
|
@@ -2651,6 +2769,7 @@ def call_model_stream(
|
|
|
2651
2769
|
|
|
2652
2770
|
yield StreamToken(final_response=ModelResponse(
|
|
2653
2771
|
text="".join(text_parts), tool_calls=calls, usage=usage,
|
|
2772
|
+
usage_unreported=unreported,
|
|
2654
2773
|
thinking=_thinking_record(provider_name, model, reasoning_blocks),
|
|
2655
2774
|
))
|
|
2656
2775
|
|
package/core/commands.py
CHANGED
|
@@ -119,9 +119,10 @@ _SPECS = (
|
|
|
119
119
|
"[--software|--research] [--config]",
|
|
120
120
|
absent={CLI: "not yet; run venastine --init."}),
|
|
121
121
|
CommandSpec("kill", "stop a background session", "[session-id]",
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
122
|
+
# Batch 265: the prompt no longer waits for them (SS2 as
|
|
123
|
+
# amended), so the reason is the key alone.
|
|
124
|
+
absent={CLI: "Ctrl+C while the CLI waits on background "
|
|
125
|
+
"sessions stops them."}),
|
|
125
126
|
CommandSpec("memories", "list durable memories visible here"),
|
|
126
127
|
CommandSpec("mentions", "show file mention status or include ignored "
|
|
127
128
|
"paths", "[ignored on|off]",
|
|
@@ -163,6 +164,9 @@ _SPECS = (
|
|
|
163
164
|
CommandSpec("trigger", "override when this chat compacts, for the "
|
|
164
165
|
"session", "[tokens|off]",
|
|
165
166
|
absent={CLI: _IN_THE_TUI}),
|
|
167
|
+
# ROADMAP_v3 §97 (UF4): both shells, one report (`core/usage_view.py`).
|
|
168
|
+
CommandSpec("usage", "token counts the provider reported for this "
|
|
169
|
+
"thread"),
|
|
166
170
|
CommandSpec("window", "override this model's context window for the "
|
|
167
171
|
"session", "[tokens|off]",
|
|
168
172
|
absent={CLI: _IN_THE_TUI}),
|
package/core/loop.py
CHANGED
|
@@ -683,11 +683,18 @@ def _already_said(notices, kind: str) -> bool:
|
|
|
683
683
|
# reasoning-only reply ends the turn with a notice. The row's first line is
|
|
684
684
|
# also the live notice's text, so the line a person sees while the turn
|
|
685
685
|
# runs is the line its replay draws (RO5).
|
|
686
|
+
#
|
|
687
|
+
# The first line says what did not run is THAT reply, and that results
|
|
688
|
+
# returned before it are real (RO2 as amended in batch 261, the owner's
|
|
689
|
+
# choice). It corrects a model that acts as though calls written in its
|
|
690
|
+
# reasoning took effect, and a bare "nothing ran" could also be read as
|
|
691
|
+
# "the calls earlier in this turn did not run either".
|
|
686
692
|
HARNESS_REASONING_ONLY = "reasoning_only"
|
|
687
693
|
REASONING_ONLY_END = "reasoning_only_end"
|
|
688
694
|
REASONING_ONLY_HEADER = (
|
|
689
695
|
"[harness] The last reply was reasoning only: no answer and no tool "
|
|
690
|
-
"call reached the harness, so nothing ran."
|
|
696
|
+
"call reached the harness, so nothing in that reply ran. Results "
|
|
697
|
+
"returned before it are real.")
|
|
691
698
|
_REASONING_ONLY_NUDGE = (
|
|
692
699
|
REASONING_ONLY_HEADER + "\n"
|
|
693
700
|
"A tool call written inside reasoning is not run. Make the call through "
|
|
@@ -2131,6 +2138,11 @@ class RunAgentLoop:
|
|
|
2131
2138
|
turn_output_tokens += usage_out
|
|
2132
2139
|
turn_prev_input = usage_in
|
|
2133
2140
|
memory.record_billed(usage_in + usage_out)
|
|
2141
|
+
# ROADMAP_v3 §97 (UF4). Beside billed, on the same object and so
|
|
2142
|
+
# over the same span: what `/usage` shows, the provider's own
|
|
2143
|
+
# counts with the coerced ones left out.
|
|
2144
|
+
memory.record_usage(response.usage,
|
|
2145
|
+
getattr(response, "usage_unreported", ()))
|
|
2134
2146
|
response.turn_billed_tokens = total_tokens_used
|
|
2135
2147
|
response.turn_new_tokens = turn_new_tokens
|
|
2136
2148
|
response.turn_output_tokens = turn_output_tokens
|
|
@@ -2395,6 +2407,10 @@ class RunAgentLoop:
|
|
|
2395
2407
|
**auth_kwargs,
|
|
2396
2408
|
), observe)
|
|
2397
2409
|
response.thread_id = memory.thread_id
|
|
2410
|
+
# ROADMAP_v3 §97 (UF4). This memory was built for this run, so its
|
|
2411
|
+
# tally is this turn's calls: the CLI's `/usage` adds them up. None
|
|
2412
|
+
# from a memory that keeps none, which the CLI's tally ignores.
|
|
2413
|
+
response.turn_usage = getattr(memory, "usage", None)
|
|
2398
2414
|
return response
|
|
2399
2415
|
|
|
2400
2416
|
@staticmethod
|
|
@@ -2663,4 +2679,8 @@ class RunAgentLoop:
|
|
|
2663
2679
|
**auth_kwargs,
|
|
2664
2680
|
), observe)
|
|
2665
2681
|
response.thread_id = memory.thread_id
|
|
2682
|
+
# ROADMAP_v3 §97 (UF4). This memory was built for this run, so its
|
|
2683
|
+
# tally is this turn's calls: the CLI's `/usage` adds them up. None
|
|
2684
|
+
# from a memory that keeps none, which the CLI's tally ignores.
|
|
2685
|
+
response.turn_usage = getattr(memory, "usage", None)
|
|
2666
2686
|
return response
|
package/core/memory.py
CHANGED
|
@@ -46,6 +46,7 @@ PERSISTENCE, AND WHY THERE'S NO NORMALIZATION STEP HERE:
|
|
|
46
46
|
from typing import Optional
|
|
47
47
|
from uuid import UUID
|
|
48
48
|
|
|
49
|
+
from core.usage_view import UsageTally
|
|
49
50
|
from storage import (
|
|
50
51
|
THREAD_KIND_CHAT,
|
|
51
52
|
create_thread,
|
|
@@ -127,6 +128,8 @@ class ConversationMemory:
|
|
|
127
128
|
# resumed -- see record_billed for why this one is deliberately not
|
|
128
129
|
# extra_data like its neighbour.
|
|
129
130
|
self.billed_tokens = 0
|
|
131
|
+
# ROADMAP_v3 §97 (UF4). Its neighbour's scope, for its reason.
|
|
132
|
+
self.usage = UsageTally()
|
|
130
133
|
# ROADMAP_v3 §81 (IM3). Images a tool attached for a call whose
|
|
131
134
|
# result has not been written yet, keyed by the call's id.
|
|
132
135
|
# add_tool_result takes them into the row it writes; nothing else
|
|
@@ -259,6 +262,16 @@ class ConversationMemory:
|
|
|
259
262
|
"""
|
|
260
263
|
self.billed_tokens += int(tokens or 0)
|
|
261
264
|
|
|
265
|
+
def record_usage(self, usage, unreported=()) -> None:
|
|
266
|
+
"""One model call's provider-reported counts, for `/usage`
|
|
267
|
+
(ROADMAP_v3 §97, UF4).
|
|
268
|
+
|
|
269
|
+
record_billed's scope and its reason: not persisted, zero on every
|
|
270
|
+
construction, so `/usage` covers exactly the span the usage line's
|
|
271
|
+
`billed` does. `unreported` is `ModelResponse.usage_unreported`.
|
|
272
|
+
"""
|
|
273
|
+
self.usage.add(usage, unreported)
|
|
274
|
+
|
|
262
275
|
def completed_turns(self) -> int:
|
|
263
276
|
"""How many turns of this thread are already finished (§21 M5/M11).
|
|
264
277
|
|