@yahyahammad/venastine 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/core/loop.py CHANGED
@@ -1,1353 +1,1490 @@
1
- """
2
- core/loop.py
3
-
4
- RunAgentLoop._run() is the shared ReAct control flow: call the model,
5
- dispatch any tool calls via the registry, feed results back, repeat until
6
- the model responds with plain text or max_steps is hit. Both public
7
- methods below configure and call this same shared method -- neither
8
- reimplements the loop, so there's exactly one place the mechanics live.
9
-
10
- §13 streaming refactor: _run() is now a generator yielding LoopEvent
11
- objects. run_to_completion() drains it for callers that only need the
12
- final ModelResponse (the three public wrappers). The generator shape
13
- lets the TUI observe token deltas, tool calls, and permission prompts
14
- in real time without callbacks.
15
-
16
- §15 permission refactor: _run()'s old `allowed_tools` list is replaced by
17
- an optional ToolContext, which carries the same restriction plus approval
18
- overrides and subagent depth. One representation, threaded to the one
19
- place that enforces it (registry -> security.permissions), instead of the
20
- loop keeping a parallel membership check of its own.
21
-
22
- D20 invariant: memory.add_assistant_message(response) executes
23
- immediately after every call_model_stream return and BEFORE any
24
- stop-condition branch. This ordering is load-bearing — persisting only
25
- on the tool-calling path silently drops plain-text final answers and
26
- budget-truncated responses.
27
- """
28
-
29
- import logging
30
- from typing import Optional
31
- from uuid import UUID
32
-
33
- import config
34
- import prompts.system_prompts as system_prompts
35
- from core.client import (
36
- api_initialization, call_model_stream, context_window_for, effort_for,
37
- ModelResponse,
38
- )
39
- from core.events import LoopEvent
40
- from core import interaction
41
- from core.memory import ConversationMemory
42
- # §27 (T1). The loop is where a thread's kind is DECIDED -- it knows which
43
- # of its three entry points is running -- and storage.create_thread is
44
- # where it is stored. Nothing in between learns what a kind means.
45
- from storage import (
46
- THREAD_KIND_CHAT, THREAD_KIND_RESEARCH_PASS, THREAD_KIND_SUBAGENT,
47
- )
48
- from tools.base import GRANT_NEVER
49
- from tools.context import ToolContext, RunInfo
50
- from tools.registry import registry, ToolCallDenied
51
-
52
- logger = logging.getLogger(__name__)
53
-
54
- # §32 A8 (#72). "You are a helpful assistant." was the whole of it,
55
- # and everything else in a chat prompt is generated or thread state --
56
- # the two catalogs, with_goal, with_memories, with_refs, with_todos,
57
- # and any active skill bodies. So the one paragraph a chat turn most
58
- # needs was the one part no tier supplied.
59
- #
60
- # Shared with the ten research passes rather than copied: see
61
- # prompts/untrusted_content and system_prompts.untrusted_content_paragraph.
62
- DEFAULT_SYSTEM_PROMPT = (
63
- "You are a helpful assistant.\n\n"
64
- + system_prompts.untrusted_content_paragraph(
65
- system_prompts.CHAT_INSTRUCTION_SOURCE))
66
- DEFAULT_PROVIDER = "ANTHROPIC"
67
-
68
- # The headless callability notice (§18) names tools hidden by
69
- # schemas(callable_only=True). Deduplicated on the HIDDEN SET, not once
70
- # per process: the set is context-dependent, so a once-per-process flag
71
- # reported only the first run's and left every later one silent. A
72
- # subagent whose definition tightens approval for another tool hides a
73
- # strictly larger set, and that difference was exactly what never got
74
- # logged -- the quiet invisibility this notice exists to prevent.
75
- _headless_notices_shown: set = set()
76
-
77
-
78
- def with_goal(system_prompt: str, memory: ConversationMemory) -> str:
79
- """Append a thread's persistent objective (§18 goal mode) to a system
80
- prompt. Single source for the '## Persistent objective' marker so the
81
- CLI wrapper and the TUI worker cannot drift apart. No-op when the
82
- thread has no goal."""
83
- goal = memory.extra.get("goal")
84
- if not goal:
85
- return system_prompt
86
- return f"{system_prompt}\n\n## Persistent objective\n{goal}"
87
-
88
-
89
- def with_todos(system_prompt: str, memory: ConversationMemory) -> str:
90
- """Append §23's todo list to a prompt. No-op when the thread has none.
91
-
92
- Beside with_goal and taking the same argument, for the same reason: a
93
- checklist is a property of the THREAD, not of the agent or the session.
94
- It survives a resume and applies whichever agent is active.
95
-
96
- MUST NOT move inside prompts.system_prompts.with_catalogs(). That
97
- function feeds pass_prompt(), so a checklist written in a chat session
98
- would land in all ten passes of a research run the user started
99
- separately -- §19's K6, FOURTH instance, after §21b's memories (M13) and
100
- §21c's refs (M19). Four copies of one warning is why the convergence of
101
- these four functions is now a recorded deferral rather than a docstring
102
- aside (J11).
103
-
104
- THE MODEL IS TOLD IT OWNS THIS. Unlike a memory or a reference, which
105
- are context the model reads, the todo list is state the model WRITES --
106
- so the fragment has to say that the way to change it is to call the
107
- tool, or the model reports progress in prose while the panel the user is
108
- watching stays stale.
109
- """
110
- todos = memory.extra.get("todos") or []
111
- if not todos:
112
- return system_prompt
113
-
114
- lines = ["## Your checklist for this conversation",
115
- "",
116
- "You maintain this with `todo_write`, which replaces the whole "
117
- "list. Keep it current as you work -- the user can see it.",
118
- ""]
119
- for item in todos:
120
- status = item.get("status", "pending")
121
- marker = {"completed": "[x]", "in_progress": "[>]"}.get(status, "[ ]")
122
- lines.append(f"{marker} {item.get('content', '')}")
123
- return f"{system_prompt}\n\n" + "\n".join(lines)
124
-
125
-
126
- def with_memories(system_prompt: str, agent=None) -> str:
127
- """Append §21b's durable-memory tier to a prompt. No-op when this run
128
- is opted out or nothing is in scope.
129
-
130
- Beside with_goal for the same reason: both shells assemble a plain-chat
131
- prompt themselves, so a single named function is what stops the CLI and
132
- the TUI drifting apart. §23's response channel is where all three of
133
- these (goal, skills, memories) should eventually converge into one
134
- assembly point rather than three parallel ones.
135
-
136
- MUST NOT move inside prompts.system_prompts.with_catalogs() -- that
137
- feeds pass_prompt(), so memories would land in all ten research passes.
138
- §19's K6, second instance.
139
- """
140
- from memories.manager import manager as memory_manager
141
-
142
- fragment = memory_manager.prompt_fragment(agent)
143
- if not fragment:
144
- return system_prompt
145
- return f"{system_prompt}\n\n{fragment}"
146
-
147
-
148
- def with_refs(system_prompt: str, memory: ConversationMemory) -> str:
149
- """Append §21c's referenced-conversation tier to a prompt. No-op when the
150
- thread has no references attached.
151
-
152
- Beside with_goal and taking the same argument, because a reference is a
153
- property of the THREAD rather than of the agent or the session: it was
154
- attached to this conversation by a person, it persists across a resume,
155
- and it applies whichever agent happens to be active. That is also why it
156
- is not routed through agents.manager.system_prompt_for() the way §21b's
157
- memories are -- that function has no memory object, and refs are not an
158
- agent's opt-in.
159
-
160
- MUST NOT move inside prompts.system_prompts.with_catalogs(). That function
161
- feeds pass_prompt(), so a reference attached in a chat session would land
162
- in all ten passes of a research run the user started separately. §19's K6,
163
- THIRD instance -- after §21b's memories, which is recorded as M13.
164
-
165
- THE SUMMARIES ARE ANNOUNCED AS SUMMARIES OF SOMETHING ELSE. A referenced
166
- thread is not this conversation's history, and a model that cannot tell
167
- the difference will answer as though the user said things they said
168
- elsewhere. Same reason §21a's compaction summary carries SUMMARY_PREFIX.
169
- """
170
- refs = memory.extra.get("refs") or []
171
- if not refs:
172
- return system_prompt
173
-
174
- # CAPPED, AND THE CAP IS STATED IN THE TEXT (M14). The attach path already
175
- # refuses to exceed it, so this is the second line of defence rather than
176
- # the first -- but MAX_INJECTED_REFS can be lowered between sessions, and a
177
- # thread carrying four refs under a cap of three must drop one visibly
178
- # rather than silently.
179
- cap = config.MAX_INJECTED_REFS
180
- shown, total = refs[:cap], len(refs)
181
- header = ("## Referenced conversations\n\n"
182
- "Distilled summaries of OTHER conversations, attached by the "
183
- "user for context. They are NOT part of this conversation's "
184
- "history -- nothing here was said to you in this thread.")
185
- if total > cap:
186
- logger.warning(
187
- "Injecting %s of %s attached references; the rest are not in this "
188
- "prompt. Raise config.MAX_INJECTED_REFS or drop one with "
189
- "/ref --clear.", cap, total)
190
- header = (f"{header}\n\nShowing {cap} of {total} attached references. "
191
- f"The others exist and are not included here.")
192
-
193
- lines = [header]
194
- for ref in shown:
195
- label = ref.get("label") or "(no preview)"
196
- lines.append(f"\n### {label}\n(thread {ref.get('thread_id')})\n"
197
- f"{ref.get('summary', '')}")
198
- return f"{system_prompt}\n\n" + "\n".join(lines)
199
-
200
-
201
- def attach_ref(memory: ConversationMemory, thread_id, summary: str,
202
- label: str = "") -> tuple:
203
- """Attach one thread's summary to `memory`'s thread. Returns (ok, message).
204
-
205
- ONE writer for both shells, like with_refs is one reader: the CLI's --ref
206
- and the TUI's /ref must agree about the cap, about re-attaching the same
207
- thread, and about the stored shape, or the two disagree about what a
208
- reference IS.
209
-
210
- REFUSES AT THE CAP rather than dropping the oldest. A reference is
211
- something a person chose by name; making room by discarding one silently
212
- is what §21b's "removal is by id, never by substring" rule exists to
213
- prevent. The message names the command that makes room.
214
-
215
- Re-attaching a thread already attached REPLACES its summary rather than
216
- adding a second entry -- the newer summary is strictly better information
217
- about the same source, and two entries for one thread would spend the cap
218
- on a duplicate.
219
- """
220
- from datetime import datetime, timezone
221
-
222
- refs = [dict(r) for r in (memory.extra.get("refs") or [])]
223
- key = str(thread_id)
224
- existing = next((r for r in refs if r.get("thread_id") == key), None)
225
- if existing is None and len(refs) >= config.MAX_INJECTED_REFS:
226
- return False, (
227
- f"Already referencing {len(refs)} conversations, which is the "
228
- f"limit (config.MAX_INJECTED_REFS). Drop one with /ref --clear "
229
- f"first -- none is removed automatically.")
230
-
231
- entry = {
232
- "thread_id": key,
233
- "summary": summary,
234
- "label": label,
235
- "attached_at": datetime.now(timezone.utc).isoformat(),
236
- }
237
- if existing is not None:
238
- refs[refs.index(existing)] = entry
239
- verb = "Updated the reference to"
240
- else:
241
- refs.append(entry)
242
- verb = "Now referencing"
243
- memory.set_extra("refs", refs)
244
- return True, f"{verb} thread {key}."
245
-
246
-
247
- def clear_refs(memory: ConversationMemory) -> int:
248
- """Drop every attached reference. Returns how many were dropped.
249
-
250
- All or nothing, deliberately: a per-reference removal needs a stable way
251
- for a person to name one, and the only handle a ref has is a thread uuid
252
- the user would have to retype. `/ref --list` then `/ref` to re-attach the
253
- ones still wanted is two steps with nothing to mistype.
254
- """
255
- refs = memory.extra.get("refs") or []
256
- if refs:
257
- memory.set_extra("refs", None)
258
- return len(refs)
259
-
260
-
261
- def _authorization_kwargs(authorization) -> dict:
262
- """Unpack a RunAuthorization into _run()'s primitives.
263
-
264
- ONE unpacking point, so the pipeline entry point and its JSON-retry
265
- twin cannot disagree about what a bundle means. The retry re-enters the
266
- SAME pass thread through continue_conversation(): passing the bundle
267
- there and not here (or the reverse) would strip a pass's tools on
268
- exactly the attempt where the model is already struggling, and the only
269
- symptom would be a second malformed answer.
270
- """
271
- if authorization is None:
272
- return {}
273
- return {
274
- "granted_tools": authorization.granted_tools or None,
275
- "grant_budget": authorization.budget,
276
- "response_channel": authorization.provider,
277
- }
278
-
279
-
280
- def advertisement_facts(authorization=None, response_channel=None,
281
- granted_tools=None) -> tuple:
282
- """(callable_only, granted) for a caller BUILDING A PROMPT (#68).
283
-
284
- Sibling of _authorization_kwargs above, and there for the same
285
- reason: one unpacking point, so the places that must agree cannot
286
- disagree about what a bundle means. The difference is only WHEN --
287
- _run() computes `headless = response_channel is None` after the
288
- bundle has been unpacked into its primitives, and a system prompt
289
- is assembled before _run() is called at all.
290
-
291
- Accepts either a bundle or the loose primitives, because
292
- run_agent_conversation takes both and rejects being given both.
293
- The bundle wins when present, which is the same precedence
294
- _authorization_kwargs applies.
295
-
296
- A NULL BUNDLE MEANS HEADLESS, not "unknown": _authorization_kwargs
297
- returns {} for None, so response_channel stays None and _run reads
298
- that as headless. Answering anything else here would advertise a
299
- catalog to exactly the runs that cannot use it.
300
- """
301
- if authorization is not None:
302
- return (authorization.provider is None,
303
- set(authorization.granted_tools or ()))
304
- return (response_channel is None, set(granted_tools or ()))
305
-
306
-
307
- def _denial_reason(tool_name: str, response_channel, context) -> str:
308
- """Why a gated call was refused, in terms the model can act on (#158).
309
-
310
- The loop already learned this lesson one branch over -- see the
311
- reachability comment above the approval block: headless, "requires
312
- approval and was not given" tells the model to retry WITH approval,
313
- which is the one thing that cannot happen there. It retries, spends a
314
- step against max_steps, and gets the same sentence back. That is the
315
- fetch_url damage class, which §15\\D24 fixed for undeclared permissions
316
- and left open on this axis.
317
-
318
- THE DISTINCTION IS WHETHER THE ARGUMENTS CAUSED THE GATE, and it is
319
- asked the same way schemas(callable_only=…) asks it -- with empty
320
- params, through the one function both dispatch() and this loop already
321
- share. A tool gated by NAME is uncallable in this run however it is
322
- called, so the useful thing to say is "stop". A tool gated by THESE
323
- PARAMS survived that filter and was advertised precisely because some
324
- other call shape is callable, so the useful thing to say is "vary the
325
- arguments" -- which is exactly the case #158 measured: `write` is
326
- advertised in a headless pass and denied only for paths outside the
327
- workspace.
328
-
329
- No tool name and no notion of a path appears here. Which argument to
330
- vary is the model's problem and the tool's schema already describes it;
331
- putting "try a path inside the workspace" in this file is the thing
332
- ToolSpec.grant_scope and request_kind exist to prevent.
333
- """
334
- if response_channel is not None:
335
- # Somebody was asked and declined. Nothing to reframe -- the answer
336
- # was a decision, not an absence, and telling the model the call
337
- # was unavailable would misreport a person's "no" as a limitation.
338
- return f"{tool_name} requires approval and was not given"
339
- if registry.approval_needed(tool_name, {}, context):
340
- return (f"{tool_name} requires approval and this run has no way to "
341
- f"ask, so it is unavailable however it is called. Do not "
342
- f"retry it.")
343
- return (f"{tool_name} requires approval for these arguments and this "
344
- f"run has no way to ask. It may not need approval with "
345
- f"different arguments.")
346
-
347
-
348
- def _obtain_approval(response_channel, tool_name: str, params: dict,
349
- notice, request_payload=None, rationale=None,
350
- headline=None) -> tuple:
351
- """Ask the human about one gated call. Returns (approved, grant).
352
-
353
- ONE route since §23. There were two -- a queue.Queue the TUI's worker
354
- parked on while its UI thread rendered a modal, and an
355
- ApprovalProvider callable, which existed only because
356
- run_to_completion() DISCARDS the LoopEvent carrying the question, so
357
- the CLI and the pipeline would have blocked on a queue with nothing
358
- displayed. Both were the same exchange with different plumbing, and
359
- the shells now supply one ResponseChannel whose `ask` does whichever
360
- of those two things that shell needs.
361
-
362
- TWO RETURN VALUES, because §23 AC1b made one question's answer richer
363
- than a yes/no. A subagent sign-off answers with the SUBSET of tools
364
- the child may then use unprompted -- None to refuse the spawn, an
365
- empty set to allow it with nothing granted, a non-empty set to grant
366
- those. `grant` is None for every other kind, which carries no subset.
367
-
368
- Which kind to ask comes from the REGISTRY, not from a name test here:
369
- the loop asks what shape of question a tool needs the same way it asks
370
- whether it needs one at all. `request_payload` is passed in already
371
- computed, because the caller needs it before this point to look up the
372
- sign-off memo -- and building a subagent's candidate list twice per
373
- call would be wasteful and could disagree with itself.
374
-
375
- `rationale` is the agent's own account of why it wants this call
376
- (§42, RA5), and it is passed in already resolved for the same
377
- reason: the caller needs it for the LoopEvent too, and looking it
378
- up twice invites the two from disagreeing. DISPLAY ONLY -- it
379
- reaches the prompt and nothing that decides anything.
380
-
381
- `headline` is the SUBJECT of the question (§46, EP2) -- for `shell`
382
- the command itself -- and travels for exactly the same reasons,
383
- resolved once by the caller and display-only. It is also already
384
- inside `params`, which every prompt renders whole; what carrying it
385
- separately buys is a shell's ability to place it where it cannot
386
- scroll out of view, without learning a parameter name.
387
- """
388
- kind = registry.request_kind(tool_name)
389
- payload = {"tool_name": tool_name, "params": params,
390
- "rationale": rationale, "headline": headline}
391
- payload.update(request_payload or {})
392
- answer = interaction.ask(response_channel, interaction.Request(
393
- kind=kind, payload=payload, notice=notice))
394
- if kind == interaction.SUBAGENT_SIGNOFF:
395
- # None denies the spawn; any set (including empty) allows it.
396
- return answer is not None, answer
397
- return bool(answer), None
398
-
399
-
400
- # ROADMAP_v2 §21. Notice kinds that describe a STANDING CONDITION rather
401
- # than an event: they stay true on every step until something changes, so
402
- # emitting them per step fills a transcript with the same line and trains
403
- # the reader past it -- which is exactly the reader you need when the one
404
- # that matters arrives. A compaction that actually happened is an event and
405
- # repeats legitimately, once per compaction.
406
- _ONCE_PER_RUN_NOTICES = (
407
- "compaction_warning", # still approaching the threshold
408
- "compaction_blocked", # the keep floors still protect everything
409
- "compaction_failed", # provider down, or the compactor returned
410
- # nothing usable (#136) -- and since batch 20
411
- # this kind is also the LATCH: seen once, no
412
- # further compact() calls this turn
413
- )
414
-
415
-
416
- def _derived_compaction_mode(memory) -> str:
417
- """The compaction mode for a caller that did not say (#43, batch 16).
418
-
419
- The mode is a property of WHAT THE THREAD IS, not of the call site:
420
- `research_pass` and `subagent` threads are machinery -- unattended,
421
- already returning a distillation, and (for a retry thread) holding
422
- exactly the turns §3 exists to keep in front of the model -- so M6's
423
- backstop applies WHEREVER they are re-entered. A chat thread is a
424
- conversation someone is present for; the ordinary working-set trigger
425
- is the feature working as designed there.
426
-
427
- This closes #43 by construction rather than by forwarding: json_retry
428
- and the §20 reviewer could not express a mode at all before, and a
429
- parameter they had to remember would have re-created the D24/R13
430
- failure shape -- correct today, one forgotten call site from wrong
431
- again. Unknown kinds fail toward the chat answer: routine compaction
432
- of an unforeseen thread kind is the conservative default, and a
433
- future kind states its own answer here when it arrives.
434
- """
435
- kind = getattr(memory, "kind", THREAD_KIND_CHAT)
436
- if kind in (THREAD_KIND_RESEARCH_PASS, THREAD_KIND_SUBAGENT):
437
- return "backstop"
438
- return "working_set"
439
-
440
-
441
- def _already_said(notices, kind: str) -> bool:
442
- return kind in _ONCE_PER_RUN_NOTICES and any(
443
- n["kind"] == kind for n in notices)
444
-
445
-
446
- def _maybe_compact(memory, model, provider_name, notices, mode):
447
- """Evaluate §21's trigger and act on it. A generator, so the caller
448
- yields from it and the notice reaches a live UI as it happens.
449
-
450
- Every notice goes BOTH ways -- appended to `notices`, which _run()
451
- attaches to the response, and yielded as a LoopEvent. The event alone
452
- would be invisible to the CLI and the pipeline, which drain the
453
- generator through run_to_completion() and discard everything that is
454
- not final. §20 and §25 each shipped that defect once.
455
-
456
- Outcome routing (batch 16, #44). compact() returns a status dict now,
457
- and this is where reportability is decided:
458
-
459
- folded notice every occurrence -- an event, and repeats
460
- legitimately, once per compaction.
461
- blocked /
462
- all-pinned /
463
- missing-agent STANDING CONDITIONS: notice AND WARNING together,
464
- ONCE PER RUN under the same dedup list. The two
465
- WARNINGs used to live inside compact() and fired on
466
- EVERY evaluation -- up to MAX_ITERATIONS + 1
467
- identical lines per turn on a thread that stays over
468
- trigger and fully floored, which is how a reader
469
- gets trained past the warnings that do not repeat.
470
- Same defect `context_limit()` had before it learned
471
- to warn once per model.
472
- no-progress /
473
- reentrant no notice at all: ordinary non-events (nothing new
474
- to fold) or already carried by _summarize's own
475
- per-call warnings.
476
-
477
- THE LATCH (#136, batch 20). compaction_failed is both a notice and a
478
- STOP for the run: once this turn has seen one -- a provider error, or
479
- an empty compactor response (compact() now reports that as
480
- status="failed" with the same kind) -- _maybe_compact returns before
481
- calling compact() again. Without the gate the dedup above only
482
- silenced the MESSAGE while every step still spent a full compactor
483
- model call on an attempt that had just failed; measured at seven
484
- calls in one six-step turn on the thread's first crossing. The latch
485
- is per-turn by construction -- `notices` dies with _run() -- so the
486
- next turn retries cleanly and a transient provider hiccup costs one
487
- wasted call, not one per step. Deliberately NOT latched: no-progress,
488
- reentrant, blocked and all-pinned exit before any model call, cost
489
- only storage reads, and are self-correcting if state shifts.
490
-
491
- Failure here is contained. Compaction is a model call, and a provider
492
- error during it must not take down a turn that was otherwise fine --
493
- the same call §20 made for its review stage. The thread simply stays
494
- uncompacted and the next call fails or succeeds on its own merits,
495
- which is strictly better than failing this one on its behalf.
496
-
497
- Takes no authorization, deliberately. The compactor declares
498
- `allowed_tools: []`, so there is nothing for a grant to apply to --
499
- a parameter here would read as security-relevant and could not be.
500
- """
501
- from core import compaction
502
-
503
- used = memory.last_input_tokens or compaction.estimated_tokens(memory.messages)
504
- action = compaction.should_compact(used, model, mode,
505
- provider_name=provider_name)
506
- if not action:
507
- return
508
- if action == "warn":
509
- if _already_said(notices, "compaction_warning"):
510
- return
511
- notice = {
512
- "kind": "compaction_warning",
513
- "text": ("Approaching the compaction threshold — older messages "
514
- "may be summarized soon."),
515
- }
516
- notices.append(notice)
517
- yield LoopEvent(notice=notice)
518
- return
519
- # THE LATCH (#136): a failed attempt already spent its model call this
520
- # turn. Everything below the warn branch re-enters compact() on every
521
- # step while the trigger stays open, so without this early return the
522
- # dedup below only hides the message and the spend repeats. See the
523
- # docstring's latch paragraph for why only compaction_failed gates.
524
- if _already_said(notices, "compaction_failed"):
525
- return
526
- try:
527
- # The current-turn floor (M5), computed HERE rather than at the
528
- # top of the turn. Archive-space makes it time-invariant within
529
- # a turn -- the archive gains assistant and tool rows as the
530
- # turn runs but no new USER row, so the answer is the same
531
- # whenever it is asked. Asking lazily keeps a storage read off
532
- # every turn that is nowhere near the threshold.
533
- outcome = compaction.compact(
534
- memory, model, provider_name,
535
- current_turn_start=memory.completed_turns())
536
- except Exception as e: # noqa: BLE001 -- contained on purpose, see above
537
- logger.exception("Compaction failed; continuing uncompacted.")
538
- notice = {
539
- "kind": "compaction_failed",
540
- "text": f"Could not compact this conversation: {e}",
541
- }
542
- if not _already_said(notices, notice["kind"]):
543
- notices.append(notice)
544
- yield LoopEvent(notice=notice)
545
- return
546
-
547
- kind = outcome.get("kind")
548
- if kind is None:
549
- return
550
- if _already_said(notices, kind):
551
- return
552
- if kind == "compaction_blocked":
553
- # See the docstring's standing-condition row: one WARNING per run,
554
- # said in the same breath as the one notice, never per evaluation.
555
- logger.warning("Compaction cannot proceed on thread %s: %s",
556
- memory.thread_id, outcome["text"])
557
- notices.append({"kind": kind, "text": outcome["text"]})
558
- yield LoopEvent(notice=notices[-1])
559
-
560
-
561
- def run_to_completion(gen) -> ModelResponse:
562
- """Consumes a _run() generator fully, discarding intermediate events,
563
- and returns the final ModelResponse — this is what keeps
564
- run_agent_conversation() and continue_conversation(), the two public
565
- entry points, synchronous for every caller.
566
-
567
- It reads the response off the TERMINAL EVENT, which is right for a
568
- bare _run(): that generator returns nothing. A generator that returns
569
- its own value -- stream_deep_research_mode(), whose response carries
570
- `thread_id` attached after the loop finished -- needs the other
571
- question asked, and `tests/conftest.drain` is where that lives now
572
- that the pass wrapper is gone."""
573
- final = None
574
- for event in gen:
575
- if event.final_response is not None:
576
- final = event.final_response
577
- if final is None:
578
- raise RuntimeError(
579
- "Generator completed without yielding a final_response"
580
- )
581
- return final
582
-
583
-
584
- # #4 (batch 27). The wrappers' default for max_total_tokens: resolve the
585
- # configured spend cap (settings.json max_token_budget) at CALL time. A
586
- # sentinel rather than a plain None default because explicit-None already
587
- # MEANS something in _run()'s signature -- genuinely uncapped -- and tests
588
- # pass it deliberately. Resolution lives at the call site, not at def time,
589
- # so a settings.json written after import is still honoured.
590
- _SPEND_UNSET = object()
591
-
592
-
593
- def _resolve_spend_cap(max_total_tokens) -> Optional[int]:
594
- """_SPEND_UNSET -> the configured cap; anything else passes through."""
595
- if max_total_tokens is _SPEND_UNSET:
596
- from core.config_loader import spend_cap
597
- return spend_cap()
598
- return max_total_tokens
599
-
600
-
601
- class RunAgentLoop:
602
-
603
- @staticmethod
604
- def _run(
605
- memory: ConversationMemory,
606
- system_prompt: str,
607
- provider_name: str,
608
- model: str,
609
- context: Optional[ToolContext],
610
- max_steps: int,
611
- max_total_tokens: Optional[int] = None,
612
- temperature: Optional[float] = None,
613
- effort: Optional[str] = None,
614
- response_channel=None,
615
- granted_tools: Optional[set] = None,
616
- grant_budget=None,
617
- compaction_mode: Optional[str] = None,
618
- ):
619
- """Generator yielding LoopEvent objects as the loop progresses.
620
-
621
- context: an optional ToolContext (§15) restricting which tools are
622
- available and which need approval this run. None means only global
623
- config policy applies, which is every production caller until §18
624
- introduces agents. This replaced an `allowed_tools` list that did
625
- the same job one layer up: the loop no longer decides tool
626
- membership itself, it just hands the context to the registry and
627
- lets security/permissions.py compose the layers.
628
-
629
- response_channel: an optional core.interaction.ResponseChannel --
630
- the ONE way this loop asks a human anything (§23 AC1). None means
631
- nothing can be asked, and any call that would need approval is
632
- denied. Replaced a queue.Queue plus a separate ApprovalProvider,
633
- which were the same exchange with two sets of plumbing; a shell
634
- that needs the thread handoff (the TUI parks its worker while the
635
- UI thread renders a modal) does that inside its own `ask`.
636
-
637
- granted_tools: names the user has ALREADY approved for this run
638
- (§18 subagent sign-off). A spawned subagent receives the set the
639
- user signed off when approving the spawn, so it does not re-ask
640
- for tools already covered. A grant, not a policy relaxation: it
641
- records an approval given early, which is why it lives on RunInfo
642
- rather than in the ToolContext's approval_overrides (those OR, and
643
- so can only tighten).
644
-
645
- grant_budget: a core.approval.GrantBudget capping how many granted
646
- calls may skip the prompt (§25 R6), or None for no cap. Shared by
647
- reference across every pass of a pipeline run. Exhaustion makes
648
- the grant stop applying, so calls fall back to being asked.
649
-
650
- compaction_mode (§21): None -- the default since batch 16 (#43) --
651
- means DERIVE from what the thread is: research_pass and subagent
652
- threads get M6's backstop wherever they are re-entered, a chat
653
- thread gets the ordinary working-set trigger. An explicit value
654
- wins, and stream_deep_research_mode passes "backstop" explicitly
655
- as the definition site. Before this, every continue_conversation
656
- caller silently ran on the chat trigger: a pass's JSON retry and
657
- the §20 reviewer evaluated a threshold their own threads were
658
- never meant to answer to.
659
- """
660
- # §29/#45 belt. `_run` is a generator, so this fires on the first
661
- # next() -- which for every production caller is immediate, since
662
- # all three wrappers drain eagerly. It exists because the loader
663
- # repairs agent files (#45) while a DIRECT caller bypasses that
664
- # repair entirely: there is no file to lose here, only a bug worth
665
- # naming, so this raises rather than substituting. The old failure
666
- # was an AttributeError on `response.stop_reason` after an empty
667
- # step loop -- a traceback pointing three layers away from the bad
668
- # value that caused it.
669
- if isinstance(max_steps, bool) or not isinstance(max_steps, int) \
670
- or max_steps < 1:
671
- raise ValueError(
672
- f"max_steps must be a positive integer, got {max_steps!r}. "
673
- f"Agent definitions carrying an invalid max_steps are "
674
- f"repaired by the loader to {config.MAX_ITERATIONS} "
675
- f"(config.MAX_ITERATIONS); direct callers must pass a valid "
676
- f"value themselves.")
677
- client = api_initialization(provider_name)
678
- # Validate the level against the model that will RECEIVE it, here
679
- # rather than at each caller. Every path -- CLI chat, each research
680
- # pass, a TUI turn, a spawned subagent -- reaches the wire through
681
- # this function, so this is the one place that cannot be bypassed
682
- # by a new caller forgetting to check. It also makes the agent case
683
- # correct by construction: an agent declaring its own model is
684
- # validated against THAT model, not the one the parent was using.
685
- effort = effort_for(client, provider_name, model, effort)
686
- # Prime the context-window cache on the same one-per-process
687
- # footing, and HERE for effort_for's reason: this is the one place
688
- # every send crosses, so a new caller cannot forget it. On
689
- # ANTHROPIC it is a pure cache hit -- effort_for's own retrieve
690
- # already recorded the window off the ModelInfo it fetched -- so
691
- # the only providers this actually calls for are Google and the
692
- # v1-compatible roster. The result is read back, without a client,
693
- # by compaction.context_limit(); a failure here is warned and
694
- # simply leaves the static table answering.
695
- context_window_for(client, provider_name, model)
696
- # §18 headless callability rule (user-widened from approval-only):
697
- # with no way to ask, a tool that needs approval is uncallable in
698
- # this configuration, so it is not advertised -- but named once in
699
- # a WARNING so the hiding is never silent.
700
- #
701
- # §25: "no way to ask" is not "not a TUI". A research pass with a
702
- # channel CAN ask, so hiding its gated tools would report a
703
- # limitation the run does not have. §23 made this one condition
704
- # again -- it was two while there were two routes.
705
- #
706
- # §25 R15: and "nothing can ask" is not "nothing has answered". A
707
- # GRANT is an answer given before the call existed, so a granted
708
- # tool is callable with no channel at all -- which is the whole
709
- # point of `--grant-tools` on an unattended run. #156: this filter
710
- # could not see the grant, so a grant-only run advertised exactly
711
- # what an unflagged one did while the CLI printed that it had
712
- # authorised something. The enforcement path below honoured the
713
- # grant correctly; the model was simply never told the tool was
714
- # there, so it never emitted the call.
715
- #
716
- # run_info IS BUILT ABOVE THIS, not below it as it was before R15.
717
- # The schemas call now needs the grant, and the grant lives on
718
- # run_info -- building it after would read the name before it is
719
- # bound. Nothing else moved: it depends only on values already in
720
- # scope here.
721
- run_info = RunInfo(
722
- model=model, provider_name=provider_name, effort=effort,
723
- granted_tools=set(granted_tools or ()),
724
- grant_budget=grant_budget)
725
- headless = response_channel is None
726
- # The BUDGET is deliberately not consulted here. Advertisement is
727
- # decided once per step; R6 makes exhaustion fall back to asking
728
- # (or, headless, to denying) at CALL time, which is a per-call
729
- # answer. Hiding a tool mid-run when the ceiling ran out would
730
- # change what the model can see between one step and the next for
731
- # a reason it cannot observe, and the model would read that as the
732
- # tool having ceased to exist.
733
- tool_schemas = registry.schemas(
734
- context, callable_only=headless, granted=run_info.granted_tools)
735
- if headless:
736
- hidden = tuple(sorted(registry.headless_hidden(
737
- context, granted=run_info.granted_tools)))
738
- if hidden and hidden not in _headless_notices_shown:
739
- _headless_notices_shown.add(hidden)
740
- logger.warning(
741
- "Headless run (no permission channel): not advertising "
742
- "%s -- they require approval and nothing here can "
743
- "grant it.",
744
- ", ".join(hidden),
745
- )
746
- total_tokens_used = 0
747
- # #4: the size instrument, kept strictly beside the spend meter.
748
- # turn_baseline_input is the FIRST call's prompt -- the thread as
749
- # this turn inherited it, which this turn did not add. Every later
750
- # step contributes its output plus whatever NEW material entered
751
- # the context (positive input deltas; a mid-turn compaction SHRINKS
752
- # the prompt, and clamping at zero keeps that shrink from being
753
- # read as negative growth). turn_billed/turn_new ride every
754
- # response; memory.record_billed accumulates thread-since-resume.
755
- turn_baseline_input: Optional[int] = None
756
- turn_prev_input = 0
757
- turn_new_tokens = 0
758
- # §25 audit trail. Attached to the response object below rather
759
- # than at each of the three return points, so a stop condition
760
- # added later cannot forget to carry it -- the list is shared by
761
- # reference and keeps filling as tools dispatch.
762
- granted_calls: list = []
763
- # §21, same pattern and the same reason.
764
- notices: list = []
765
-
766
- # Turn boundary (M3). The primary trigger: the thread is settled,
767
- # nothing is mid-flight, and last_input_tokens is the provider's
768
- # own measurement of what the previous call was sent.
769
- yield from _maybe_compact(
770
- memory, model, provider_name, notices,
771
- compaction_mode
772
- if compaction_mode is not None
773
- else _derived_compaction_mode(memory))
774
-
775
- response = None
776
- for _ in range(max_steps):
777
- response = None
778
- for token in call_model_stream(
779
- client, provider_name, model, memory.messages,
780
- system_prompt, tool_schemas, temperature, effort,
781
- ):
782
- if token.text_delta:
783
- yield LoopEvent(token_delta=token.text_delta)
784
- if token.thinking_delta:
785
- yield LoopEvent(thinking_delta=token.thinking_delta)
786
- if token.final_response is not None:
787
- response = token.final_response
788
-
789
- if response is None:
790
- raise RuntimeError(
791
- "call_model_stream completed without yielding a final response"
792
- )
793
-
794
- usage_in = response.usage.get("input_tokens", 0)
795
- usage_out = response.usage.get("output_tokens", 0)
796
- total_tokens_used += usage_in + usage_out
797
- if turn_baseline_input is None:
798
- # First call of the turn: its prompt IS the inherited
799
- # thread, so only this step's OUTPUT counts as new.
800
- turn_baseline_input = usage_in
801
- else:
802
- turn_new_tokens += max(0, usage_in - turn_prev_input)
803
- turn_new_tokens += usage_out
804
- turn_prev_input = usage_in
805
- memory.record_billed(usage_in + usage_out)
806
- response.turn_billed_tokens = total_tokens_used
807
- response.turn_new_tokens = turn_new_tokens
808
-
809
- # D20: persist EVERY assistant turn BEFORE any branching.
810
- memory.add_assistant_message(response)
811
- response.granted_calls = granted_calls
812
- response.notices = notices
813
- # §21: the provider's own count of everything it was just
814
- # sent. Recorded here rather than at the trigger so a thread
815
- # resumed in a later process starts from a real measurement
816
- # instead of a character estimate.
817
- memory.record_input_tokens(response.usage.get("input_tokens", 0))
818
-
819
- if max_total_tokens is not None and total_tokens_used >= max_total_tokens:
820
- response.stop_reason = "token_budget_exceeded"
821
- yield LoopEvent(
822
- final_response=response, stop_reason=response.stop_reason,
823
- )
824
- return
825
-
826
- if not response.tool_calls:
827
- response.stop_reason = "complete"
828
- yield LoopEvent(
829
- final_response=response, stop_reason=response.stop_reason,
830
- )
831
- return
832
-
833
- for call in response.tool_calls:
834
- yield LoopEvent(tool_call_start={
835
- "id": call.id, "name": call.name, "input": call.input,
836
- })
837
-
838
- # Single source of truth with dispatch(): approval_needed()
839
- # ORs the tool's own path/command-dependent approval_check
840
- # with the config/context requirement (§15). Both callers
841
- # go through it so they cannot diverge — a path-dependent
842
- # tool must yield permission_request so the TUI user can
843
- # approve, not be silently denied by one check while the
844
- # other reports no approval needed.
845
- #
846
- # There is no separate allowed_tools membership test here
847
- # any more: dispatch() calls is_tool_allowed(name, context),
848
- # which raises ToolCallDenied with a message distinguishing
849
- # a context restriction from a global policy denial.
850
- needs_approval = registry.approval_needed(
851
- call.name, call.input, context)
852
- # Reachability before approval. Asking first meant a
853
- # context-excluded tool prompted the user, who clicked
854
- # Allow, and was then denied anyway -- and headless, it
855
- # reported "requires approval and was not given", telling
856
- # the model to retry with approval when the actionable
857
- # answer is "not available in this context". dispatch()
858
- # still enforces the denial; this only stops asking a
859
- # question whose answer cannot matter.
860
- if needs_approval and not registry.is_allowed(call.name, context):
861
- needs_approval = False
862
- # §32 A7 (#70): the same rule for a refusal only the
863
- # TOOL can see. spawn_subagent's unknown-agent and
864
- # depth-limit checks lived inside run(), so both
865
- # prompted first and errored second -- and for an
866
- # unknown name the question had an empty notice and an
867
- # empty candidate list, i.e. nothing to tick. The
868
- # call is refused by dispatch with the tool's own
869
- # reason; this only stops ASKING first. It also keeps
870
- # J8's memo from being keyed on
871
- # (spawn_subagent, None) -- a subject of None is the
872
- # absence of the very thing the key exists to
873
- # distinguish.
874
- if needs_approval and registry.refusal_reason(
875
- call.name, call.input, context):
876
- needs_approval = False
877
- # §33 W3 (#37): the same rule for a call whose arguments
878
- # never parsed. Approving it could not make it runnable,
879
- # so the question has no answer that changes anything --
880
- # and headless it would report "requires approval and was
881
- # not given", which is A7's exact wrong-cause failure.
882
- if needs_approval and call.parse_error is not None:
883
- needs_approval = False
884
- # §18 sign-off (S1), narrowed by §25 (R2): a tool whose
885
- # grant_scope is "run" is asked about once per run, not
886
- # once per call. The loop names no tool -- it asks the
887
- # registry, the same way it asks whether approval is
888
- # needed at all.
889
- #
890
- # THREE conditions, and the last two are the §25 change:
891
- #
892
- # registry.grantable -- a tool deciding approval from its
893
- # PARAMS was never consented to by name. Before this,
894
- # a signed-off subagent could run any shell command for
895
- # the rest of the turn on the strength of one prompt
896
- # that said "shell". Re-checked here rather than
897
- # trusted from the set, so a stale or hand-built grant
898
- # cannot widen anything.
899
- #
900
- # grant_budget.take() -- pre-flight authorization trades
901
- # a per-call decision for one up-front decision, and
902
- # gives up the natural bound on how many times the
903
- # tool runs. Exhaustion falls back to ASKING, not to
904
- # failing: supervised runs continue, headless ones deny
905
- # exactly as they would for any gated tool.
906
- #
907
- # §23 AC1b: the memo is keyed by (tool, SUBJECT), not by
908
- # tool name alone. A subagent sign-off answers about one
909
- # agent's candidate list, so reusing it for a different
910
- # agent would grant a set nobody was shown -- which is
911
- # what the name-only version did: approving a spawn of `a`
912
- # silently covered a later spawn of `b` in the same turn.
913
- # Tools with no subject (every other one) memo under None
914
- # and behave exactly as before.
915
- request_payload = registry.request_payload(
916
- call.name, call.input, context)
917
- subject = request_payload.get("subject")
918
- remembered, signoff = run_info.recall_signoff(
919
- call.name, subject)
920
- # grant_policy on a NAME-LEVEL grant only -- R13's literal
921
- # claim is that approving the NAME never carries the
922
- # authority for a GRANT_NEVER tool, on either path, and
923
- # until now no path enforced it. Batch 6 declared the
924
- # policy and taught the two functions that decide what
925
- # may be OFFERED to read it; this is the reader that
926
- # decides whether a grant already in hand APPLIES.
927
- # `spawn_subagent` was covered only because R14's
928
- # subject condition happens to catch it; `remember`
929
- # carries no subject, so a grant naming it dispatched a
930
- # durable cross-session write with no prompt -- M17's
931
- # scenario through the one door R13 left unlocked.
932
- #
933
- # IT MUST NOT REACH THE SIGN-OFF MEMO, and that is not a
934
- # nicety: `spawn_subagent` is GRANT_NEVER, so testing
935
- # the policy unconditionally here stops §23's memo
936
- # applying and the same agent is asked about twice in
937
- # one turn. R13 is about what approving a NAME buys; a
938
- # memo is an answer somebody gave about this subject,
939
- # which J8 and R14 govern instead. recall_signoff
940
- # already separates them by what it returns -- the memo
941
- # always carries a subset, the grant carries None.
942
- answered_by_name = signoff is None
943
- if (needs_approval
944
- and remembered
945
- and registry.grantable(call.name)
946
- and not (answered_by_name
947
- and registry.grant_policy(call.name)
948
- == GRANT_NEVER)
949
- and (run_info.grant_budget is None
950
- or run_info.grant_budget.take())):
951
- needs_approval = False
952
- granted_calls.append(
953
- {"tool": call.name, "params": call.input})
954
- else:
955
- # Not reused: nothing is carried into dispatch unless
956
- # this call's own answer supplies it below.
957
- signoff = None
958
- if needs_approval:
959
- notice = registry.approval_notice(
960
- call.name, call.input, context)
961
- # §42 (RA5). The agent's own stated reason, asked
962
- # of the registry so the loop names no key -- the
963
- # same way it asks what KIND of question a tool
964
- # needs and whether it needs one at all. The VALUE
965
- # travels rather than the key, because the two
966
- # shells that render it are a Textual screen that
967
- # imports nothing from this package and a `print`
968
- # in main.py.
969
- rationale = registry.rationale_for(
970
- call.name, call.input)
971
- # §46 (EP2). The SUBJECT of the question, resolved
972
- # the same way and travelling the same route: for
973
- # `shell` it is the command, and a shell that shows
974
- # it must not have to know that. Every other tool
975
- # answers None and its prompt is unchanged.
976
- headline = registry.headline_for(
977
- call.name, call.input)
978
- yield LoopEvent(permission_request={
979
- "tool_name": call.name, "params": call.input,
980
- "notice": notice,
981
- "rationale": rationale,
982
- "headline": headline,
983
- })
984
- approved, signoff = _obtain_approval(
985
- response_channel, call.name, call.input, notice,
986
- request_payload, rationale=rationale,
987
- headline=headline)
988
- if not approved:
989
- result = {"error": _denial_reason(
990
- call.name, response_channel, context)}
991
- else:
992
- # §25 R11: run-scope is a shortcut for a chat turn,
993
- # where re-asking about the same tool seconds later
994
- # is noise. A provider that declines it is saying
995
- # its whole purpose is per-call supervision --
996
- # attended mode -- and one yes must not silently
997
- # cover later calls there.
998
- if (registry.grant_scope(call.name) == "run"
999
- and (response_channel is None
1000
- or response_channel.honour_run_scope)):
1001
- run_info.remember_signoff(
1002
- call.name, subject, signoff)
1003
- # Approval already obtained — pass a callback that
1004
- # returns True so dispatch()'s internal re-check
1005
- # doesn't deny an already-approved call.
1006
- try:
1007
- result = registry.dispatch(
1008
- call.name, call.input, context=context,
1009
- approval_callback=lambda n, p: True,
1010
- parent_run=run_info,
1011
- response_channel=response_channel,
1012
- signoff=signoff,
1013
- memory=memory,
1014
- )
1015
- except ToolCallDenied as e:
1016
- result = {"error": str(e)}
1017
- elif call.parse_error is not None:
1018
- # §33 W1 (#37). The loop is the only layer that can
1019
- # see this: dispatch() takes (name, params, context)
1020
- # and the failure is a fact about the wire, not about
1021
- # the params -- {} is a perfectly good empty dict by
1022
- # the time it gets there.
1023
- #
1024
- # RETURNED as the tool's result, not raised, for
1025
- # A10's reason: the model is told its arguments did not
1026
- # arrive and can reissue the call, which is the whole
1027
- # difference between losing one step and losing a
1028
- # ten-pass run.
1029
- result = {"error": call.parse_error}
1030
- else:
1031
- try:
1032
- result = registry.dispatch(
1033
- call.name, call.input, context=context,
1034
- parent_run=run_info,
1035
- response_channel=response_channel,
1036
- memory=memory)
1037
- except ToolCallDenied as e:
1038
- result = {"error": str(e)}
1039
-
1040
- # §23 slice 2 (J10). A tool may attach a `notice` to its
1041
- # result for the shell to show; it is forwarded as a
1042
- # LoopEvent and REMOVED from what the model sees.
1043
- #
1044
- # Generic on purpose. The alternative was for the loop to
1045
- # notice that `todo_write` had just run, which puts a tool
1046
- # name in this file -- exactly what ToolSpec.grant_scope and
1047
- # ToolSpec.request_kind exist to avoid. Any later tool that
1048
- # wants to say something mid-turn gets this route free.
1049
- #
1050
- # POPPED, not copied: a notice is plumbing for the shell, and
1051
- # leaving it in the result would send the TUI panel's trigger
1052
- # to the model as if it were part of the tool's answer.
1053
- if isinstance(result, dict) and "notice" in result:
1054
- notice = result.pop("notice")
1055
- if isinstance(notice, dict) and notice.get("kind"):
1056
- yield LoopEvent(notice=notice)
1057
- else:
1058
- # A malformed notice is a bug in the tool, not a
1059
- # reason to fail the call -- the result is still
1060
- # good. Logged rather than raised, and the key is
1061
- # gone either way so it cannot reach the model.
1062
- logger.warning(
1063
- "%s returned a malformed notice; dropping it: %r",
1064
- call.name, notice)
1065
-
1066
- # Persist BEFORE emitting (#42). A generator only advances
1067
- # while someone iterates it, so emitting first made the row
1068
- # depend on a consumer continuing to read -- and an
1069
- # abandoned generator left the `tool_use` D20 already wrote
1070
- # with no matching `tool_result`, which is M4's pairing
1071
- # broken from the other side and an unresumable thread.
1072
- # Same rule §22 gives _Progress.checkpoint() for trace
1073
- # lines. The event carries nothing the persist needs, so
1074
- # the order is free.
1075
- memory.add_tool_result(call.id, result)
1076
- yield LoopEvent(tool_result={"id": call.id, "result": result})
1077
-
1078
- # The mid-turn valve (M3). A single turn can add far more than
1079
- # any buffer covers -- MAX_READ_CHARS alone is ~12.5k tokens
1080
- # per read call, with up to max_steps of them -- so waiting for
1081
- # the next turn boundary would let a tool-heavy turn run the
1082
- # thread into the provider's hard limit.
1083
- #
1084
- # Safe by construction rather than by timing: the current-turn
1085
- # floor makes the current turn unfoldable, so this can only ever
1086
- # summarize exchanges that were already complete when the turn
1087
- # began. That is what made a mid-turn trigger acceptable at
1088
- # all.
1089
- #
1090
- # The floor is `memory.completed_turns()`, asked inside
1091
- # _maybe_compact. This comment used to name a `turn_start`
1092
- # variable recorded at the top of the turn (audit #91); it moved
1093
- # because archive-space turn counting is time-invariant WITHIN a
1094
- # turn -- a completed turn cannot become incomplete -- so asking
1095
- # lazily gives the same answer and keeps a storage read off every
1096
- # turn nowhere near the threshold. See core/compaction.py's note
1097
- # on why it is computed there rather than passed down.
1098
- yield from _maybe_compact(
1099
- memory, model, provider_name, notices,
1100
- compaction_mode
1101
- if compaction_mode is not None
1102
- else _derived_compaction_mode(memory))
1103
-
1104
- # max_steps exhausted without an earlier return
1105
- response.stop_reason = "max_steps_reached"
1106
- yield LoopEvent(
1107
- final_response=response, stop_reason=response.stop_reason,
1108
- )
1109
-
1110
- @staticmethod
1111
- def run_agent_conversation(
1112
- user_goal: str,
1113
- model: str,
1114
- provider_name: str = DEFAULT_PROVIDER,
1115
- max_steps: int = config.MAX_ITERATIONS,
1116
- max_total_tokens: Optional[int] = _SPEND_UNSET,
1117
- thread_id: Optional[UUID] = None,
1118
- temperature: Optional[float] = None,
1119
- effort: Optional[str] = None,
1120
- context: Optional[ToolContext] = None,
1121
- system_prompt: Optional[str] = None,
1122
- response_channel=None,
1123
- granted_tools: Optional[set] = None,
1124
- authorization=None,
1125
- thread_kind: str = THREAD_KIND_CHAT,
1126
- ) -> ModelResponse:
1127
- """Regular conversation — full tool set, default system prompt,
1128
- one continuous thread. Pass thread_id to resume an existing
1129
- thread; omit (or None) to start a fresh one. Pass context (§15) to
1130
- run under an agent's tool restrictions; None means global policy
1131
- only. Pass system_prompt (§18) when running AS an agent -- the
1132
- TUI worker and spawn_subagent both build it via
1133
- agents.manager.AgentManager.system_prompt_for();
1134
- None keeps the default base + skill/agent catalogs.
1135
-
1136
- A thread's persistent goal (§18 goal mode) is appended to WHICHEVER
1137
- prompt is used, so it applies in every shell without the callers
1138
- knowing about it.
1139
-
1140
- response_channel / granted_tools exist for spawn_subagent (§18
1141
- S1). Without the channel a spawned subagent ran HEADLESS while its
1142
- parent could prompt, so it silently lost every approval-gated tool
1143
- -- including all MCP tools -- and the same agent behaved
1144
- differently depending on whether it was activated with /agent or
1145
- spawned.
1146
-
1147
- authorization (§20): a RunAuthorization, for an agent-shaped run
1148
- that is part of a research run -- §20's reviewer is the first.
1149
- §25 added the bundle to the pass entry point and
1150
- continue_conversation and stopped here, because nothing
1151
- agent-shaped needed it yet; a reviewer inheriting its run's grants
1152
- and BUDGET does.
1153
-
1154
- Mutually exclusive with granted_tools, and checked rather than
1155
- silently resolved. The two spell the same underlying argument, so
1156
- one would quietly win -- and if the bundle lost, a reviewer would
1157
- run with no budget and no provider while looking authorised.
1158
-
1159
- thread_kind (§27 T1) is what this entry point creates when it
1160
- creates a thread. It defaults to "chat" because this is ALSO the
1161
- plain-conversation entry point for both shells; the three callers
1162
- that are not a human conversation -- spawn_subagent, §20's reviewer
1163
- and §21a's compactor -- pass THREAD_KIND_SUBAGENT, which is what
1164
- keeps their threads out of the picker. Ignored when thread_id is
1165
- given: resuming does not reclassify."""
1166
- if authorization is not None and granted_tools is not None:
1167
- raise ValueError(
1168
- "run_agent_conversation: pass authorization= or "
1169
- "granted_tools=, not both -- they set the same underlying "
1170
- "argument, so one would silently win."
1171
- )
1172
- memory = ConversationMemory(thread_id=thread_id, kind=thread_kind)
1173
- memory.add_user_message(user_goal)
1174
- # #68, and this branch is the one an unattended CLI run takes.
1175
- _callable_only, _granted = advertisement_facts(
1176
- authorization, response_channel, granted_tools)
1177
- prompt = with_goal(
1178
- system_prompt
1179
- if system_prompt is not None
1180
- else system_prompts.with_catalogs(
1181
- DEFAULT_SYSTEM_PROMPT, callable_only=_callable_only,
1182
- granted=_granted),
1183
- memory,
1184
- )
1185
- # §21b (M13). Only when there is no agent-built prompt: an agent
1186
- # run got its memories inside system_prompt_for(), and appending
1187
- # again here would duplicate them. `agent=None` is what makes a
1188
- # plain chat turn count as opted in -- see memories.manager.
1189
- if system_prompt is None:
1190
- prompt = with_memories(prompt)
1191
- # §21c. UNCONDITIONAL, unlike memories above: a reference belongs to
1192
- # the thread, not to an agent, so an agent-built prompt needs it too --
1193
- # and system_prompt_for() has no memory to read it from.
1194
- prompt = with_refs(prompt, memory)
1195
- # §23 slice 2. Unconditional too, and for with_refs' reason: a
1196
- # checklist is thread state, so an agent-built prompt needs it.
1197
- prompt = with_todos(prompt, memory)
1198
- auth_kwargs = (
1199
- _authorization_kwargs(authorization) if authorization is not None
1200
- else {"granted_tools": granted_tools}
1201
- )
1202
- # §23: the bundle's channel and the explicit parameter are now the
1203
- # SAME kind of thing, so they can collide -- they could not before,
1204
- # when one was a queue named permission_channel and the other an
1205
- # ApprovalProvider named approval_provider, and both were passed.
1206
- #
1207
- # The explicit argument wins, preserving _obtain_approval's old
1208
- # precedence ("the channel wins when both are present"). A caller
1209
- # that passes one is describing THIS run; the bundle may have been
1210
- # assembled for a pipeline several layers up.
1211
- # Popped UNCONDITIONALLY, then merged. `a or kwargs.pop(...)`
1212
- # short-circuits, so the key would survive in auth_kwargs on
1213
- # exactly the path where an explicit channel was passed -- and
1214
- # collide with it at the call below.
1215
- bundled_channel = auth_kwargs.pop("response_channel", None)
1216
- response_channel = response_channel or bundled_channel
1217
- response = run_to_completion(RunAgentLoop._run(
1218
- memory, prompt,
1219
- provider_name, model, context,
1220
- max_steps, _resolve_spend_cap(max_total_tokens),
1221
- temperature=temperature, effort=effort,
1222
- response_channel=response_channel, **auth_kwargs,
1223
- ))
1224
- response.thread_id = memory.thread_id
1225
- return response
1226
-
1227
- @staticmethod
1228
- def stream_deep_research_mode(
1229
- pass_input: str,
1230
- model: str,
1231
- pass_id: str,
1232
- provider_name: str = DEFAULT_PROVIDER,
1233
- max_steps: int = config.MAX_ITERATIONS,
1234
- max_total_tokens: Optional[int] = _SPEND_UNSET,
1235
- temperature: Optional[float] = None,
1236
- effort: Optional[str] = None,
1237
- context: Optional[ToolContext] = None,
1238
- authorization=None,
1239
- ):
1240
- """One research pass, as a generator of LoopEvents, RETURNING the
1241
- ModelResponse (ROADMAP_v2 §26).
1242
-
1243
- Exists because §22's P2 -- "a pass's LoopEvents do not propagate
1244
- up" -- made a pass opaque while it ran. The pipeline could say
1245
- which pass had started and nothing about what it was doing, so a
1246
- pass making fourteen tool calls over several minutes looked
1247
- identical to one that had hung.
1248
-
1249
- P2 is AMENDED here, not repealed. What escapes this generator is
1250
- still consumed inside core/reasoning/orchestrator.py, which
1251
- translates a chosen subset into PipelineEvents at the pass
1252
- boundary -- so a pipeline consumer still only ever sees one event
1253
- type, which is what P1 and P2 were jointly protecting. What
1254
- changed is the premise that a pass's internals are not worth
1255
- seeing.
1256
-
1257
- A GENERATOR SIBLING, not an observer callback on the existing
1258
- function. That shape is the project's own, used three times
1259
- already, and §23 AC1 exists specifically to stop a third bespoke
1260
- request/response channel appearing beside the ones that already
1261
- existed -- which §23 has now merged into core.interaction.
1262
-
1263
- The response is RETURNED rather than read off the terminal event
1264
- because `thread_id` is attached after the loop finishes. A caller
1265
- must therefore drain this with something that keeps a generator's
1266
- return value: `yield from` (what _run_pass does) or
1267
- `tests/conftest.drain`, never run_to_completion().
1268
-
1269
- §27: the fresh thread per pass is a LOCKED INVARIANT, not the
1270
- defect §27's second bug looked like -- passes share distilled JSON
1271
- and never raw history, which is what stops a later pass reading how
1272
- an earlier one argued. What was missing is the thread saying so, so
1273
- that ~15 threads per run stop being offered as conversations.
1274
- """
1275
- memory = ConversationMemory(kind=THREAD_KIND_RESEARCH_PASS)
1276
- memory.add_user_message(pass_input)
1277
- # #68. The catalogs are decided from the SAME bundle the tool
1278
- # schemas are decided from a few frames down, so a pass cannot
1279
- # be told to spawn a subagent it will not be given.
1280
- _callable_only, _granted = advertisement_facts(authorization)
1281
- system_prompt = system_prompts.pass_prompt(
1282
- pass_id, callable_only=_callable_only, granted=_granted)
1283
- response = None
1284
- for event in RunAgentLoop._run(
1285
- memory, system_prompt, provider_name, model, context,
1286
- max_steps, _resolve_spend_cap(max_total_tokens),
1287
- temperature=temperature, effort=effort,
1288
- # §21 M6. A pass is headless and unattended and already returns
1289
- # a distillation, so routine compaction here would spend on a
1290
- # judgment call nobody is watching. The backstop still fires
1291
- # near the model's real context window, because the
1292
- # alternative there is a hard provider error mid-pipeline.
1293
- compaction_mode="backstop",
1294
- **_authorization_kwargs(authorization),
1295
- ):
1296
- if event.final_response is not None:
1297
- response = event.final_response
1298
- yield event
1299
- if response is None:
1300
- # Same contract as run_to_completion's RuntimeError, and for
1301
- # the same reason: a loop that ended without a final response
1302
- # has a bug, and returning None would push the failure into
1303
- # whichever pass dereferences .text first.
1304
- raise RuntimeError(
1305
- "Generator completed without yielding a final_response"
1306
- )
1307
- response.thread_id = memory.thread_id
1308
- return response
1309
-
1310
- @staticmethod
1311
- def continue_conversation(
1312
- thread_id: UUID,
1313
- message: str,
1314
- system_prompt: str,
1315
- model: str,
1316
- provider_name: str = DEFAULT_PROVIDER,
1317
- max_steps: int = config.MAX_ITERATIONS,
1318
- max_total_tokens: Optional[int] = _SPEND_UNSET,
1319
- temperature: Optional[float] = None,
1320
- effort: Optional[str] = None,
1321
- context: Optional[ToolContext] = None,
1322
- authorization=None,
1323
- ) -> ModelResponse:
1324
- """
1325
- Sends a follow-up message into an EXISTING conversation thread and
1326
- runs the loop to completion on it. The resumed thread's full history
1327
- (every prior user/assistant/tool turn, persisted by _run()'s
1328
- always-persist behavior) is loaded by ConversationMemory and
1329
- visible to the next call_model invocation.
1330
-
1331
- ROADMAP §3's JSON-retry path uses this to retry a pass that
1332
- returned malformed JSON: the corrective follow-up is sent into
1333
- the same thread the failed pass used, so the model sees its own
1334
- prior output in context and can correct it.
1335
-
1336
- authorization (§25) is NOT optional in practice for that path. A
1337
- retry is the same pass continuing, so it must run with the same
1338
- authorization; omitting it would silently strip the pass's gated
1339
- tools on precisely the attempt where the model is already
1340
- struggling, and the only symptom would be a second malformed
1341
- answer. The budget carries across, so retries spend from the same
1342
- ceiling rather than resetting it.
1343
- """
1344
- memory = ConversationMemory(thread_id=thread_id)
1345
- memory.add_user_message(message)
1346
- response = run_to_completion(RunAgentLoop._run(
1347
- memory, system_prompt, provider_name, model, context,
1348
- max_steps, _resolve_spend_cap(max_total_tokens),
1349
- temperature=temperature, effort=effort,
1350
- **_authorization_kwargs(authorization),
1351
- ))
1352
- response.thread_id = thread_id
1353
- return response
1
+ """
2
+ core/loop.py
3
+
4
+ RunAgentLoop._run() is the shared ReAct control flow: call the model,
5
+ dispatch any tool calls via the registry, feed results back, repeat until
6
+ the model responds with plain text or max_steps is hit. Both public
7
+ methods below configure and call this same shared method -- neither
8
+ reimplements the loop, so there's exactly one place the mechanics live.
9
+
10
+ §13 streaming refactor: _run() is now a generator yielding LoopEvent
11
+ objects. run_to_completion() drains it for callers that only need the
12
+ final ModelResponse (the three public wrappers). The generator shape
13
+ lets the TUI observe token deltas, tool calls, and permission prompts
14
+ in real time without callbacks.
15
+
16
+ §15 permission refactor: _run()'s old `allowed_tools` list is replaced by
17
+ an optional ToolContext, which carries the same restriction plus approval
18
+ overrides and subagent depth. One representation, threaded to the one
19
+ place that enforces it (registry -> security.permissions), instead of the
20
+ loop keeping a parallel membership check of its own.
21
+
22
+ D20 invariant: memory.add_assistant_message(response) executes
23
+ immediately after every call_model_stream return and BEFORE any
24
+ stop-condition branch. This ordering is load-bearing — persisting only
25
+ on the tool-calling path silently drops plain-text final answers and
26
+ budget-truncated responses.
27
+ """
28
+
29
+ import logging
30
+ from typing import Optional
31
+ from uuid import UUID
32
+
33
+ import config
34
+ import prompts.system_prompts as system_prompts
35
+ from core.client import (
36
+ api_initialization, call_model_stream, context_window_for, effort_for,
37
+ ModelResponse,
38
+ )
39
+ from core.events import LoopEvent
40
+ from core import interaction
41
+ from core.memory import ConversationMemory
42
+ # §27 (T1). The loop is where a thread's kind is DECIDED -- it knows which
43
+ # of its three entry points is running -- and storage.create_thread is
44
+ # where it is stored. Nothing in between learns what a kind means.
45
+ from storage import (
46
+ THREAD_KIND_CHAT, THREAD_KIND_RESEARCH_PASS, THREAD_KIND_SUBAGENT,
47
+ )
48
+ from tools.base import GRANT_NEVER
49
+ from tools.context import ToolContext, RunInfo
50
+ from tools.registry import registry, ToolCallDenied
51
+
52
+ logger = logging.getLogger(__name__)
53
+
54
+ # §32 A8 (#72). "You are a helpful assistant." was the whole of it,
55
+ # and everything else in a chat prompt is generated or thread state --
56
+ # the two catalogs, with_goal, with_memories, with_refs, with_todos,
57
+ # and any active skill bodies. So the one paragraph a chat turn most
58
+ # needs was the one part no tier supplied.
59
+ #
60
+ # Shared with the ten research passes rather than copied: see
61
+ # prompts/untrusted_content and system_prompts.untrusted_content_paragraph.
62
+ DEFAULT_SYSTEM_PROMPT = (
63
+ "You are a helpful assistant.\n\n"
64
+ + system_prompts.untrusted_content_paragraph(
65
+ system_prompts.CHAT_INSTRUCTION_SOURCE))
66
+ DEFAULT_PROVIDER = "ANTHROPIC"
67
+
68
+ # The headless callability notice (§18) names tools hidden by
69
+ # schemas(callable_only=True). Deduplicated on the HIDDEN SET, not once
70
+ # per process: the set is context-dependent, so a once-per-process flag
71
+ # reported only the first run's and left every later one silent. A
72
+ # subagent whose definition tightens approval for another tool hides a
73
+ # strictly larger set, and that difference was exactly what never got
74
+ # logged -- the quiet invisibility this notice exists to prevent.
75
+ _headless_notices_shown: set = set()
76
+
77
+
78
+ def with_goal(system_prompt: str, memory: ConversationMemory) -> str:
79
+ """Append a thread's persistent objective (§18 goal mode) to a system
80
+ prompt. Single source for the '## Persistent objective' marker so the
81
+ CLI wrapper and the TUI worker cannot drift apart. No-op when the
82
+ thread has no goal."""
83
+ goal = memory.extra.get("goal")
84
+ if not goal:
85
+ return system_prompt
86
+ return f"{system_prompt}\n\n## Persistent objective\n{goal}"
87
+
88
+
89
+ def with_todos(system_prompt: str, memory: ConversationMemory) -> str:
90
+ """Append §23's todo list to a prompt. No-op when the thread has none.
91
+
92
+ Beside with_goal and taking the same argument, for the same reason: a
93
+ checklist is a property of the THREAD, not of the agent or the session.
94
+ It survives a resume and applies whichever agent is active.
95
+
96
+ MUST NOT move inside prompts.system_prompts.with_catalogs(). That
97
+ function feeds pass_prompt(), so a checklist written in a chat session
98
+ would land in all ten passes of a research run the user started
99
+ separately -- §19's K6, FOURTH instance, after §21b's memories (M13) and
100
+ §21c's refs (M19). Four copies of one warning is why the convergence of
101
+ these four functions is now a recorded deferral rather than a docstring
102
+ aside (J11).
103
+
104
+ THE MODEL IS TOLD IT OWNS THIS. Unlike a memory or a reference, which
105
+ are context the model reads, the todo list is state the model WRITES --
106
+ so the fragment has to say that the way to change it is to call the
107
+ tool, or the model reports progress in prose while the panel the user is
108
+ watching stays stale.
109
+ """
110
+ todos = memory.extra.get("todos") or []
111
+ if not todos:
112
+ return system_prompt
113
+
114
+ lines = ["## Your checklist for this conversation",
115
+ "",
116
+ "You maintain this with `todo_write`, which replaces the whole "
117
+ "list. Keep it current as you work -- the user can see it.",
118
+ ""]
119
+ for item in todos:
120
+ status = item.get("status", "pending")
121
+ marker = {"completed": "[x]", "in_progress": "[>]"}.get(status, "[ ]")
122
+ lines.append(f"{marker} {item.get('content', '')}")
123
+ return f"{system_prompt}\n\n" + "\n".join(lines)
124
+
125
+
126
+ def with_memories(system_prompt: str, agent=None) -> str:
127
+ """Append §21b's durable-memory tier to a prompt. No-op when this run
128
+ is opted out or nothing is in scope.
129
+
130
+ Beside with_goal for the same reason: both shells assemble a plain-chat
131
+ prompt themselves, so a single named function is what stops the CLI and
132
+ the TUI drifting apart. §23's response channel is where all three of
133
+ these (goal, skills, memories) should eventually converge into one
134
+ assembly point rather than three parallel ones.
135
+
136
+ MUST NOT move inside prompts.system_prompts.with_catalogs() -- that
137
+ feeds pass_prompt(), so memories would land in all ten research passes.
138
+ §19's K6, second instance.
139
+ """
140
+ from memories.manager import manager as memory_manager
141
+
142
+ fragment = memory_manager.prompt_fragment(agent)
143
+ if not fragment:
144
+ return system_prompt
145
+ return f"{system_prompt}\n\n{fragment}"
146
+
147
+
148
+ def with_refs(system_prompt: str, memory: ConversationMemory) -> str:
149
+ """Append §21c's referenced-conversation tier to a prompt. No-op when the
150
+ thread has no references attached.
151
+
152
+ Beside with_goal and taking the same argument, because a reference is a
153
+ property of the THREAD rather than of the agent or the session: it was
154
+ attached to this conversation by a person, it persists across a resume,
155
+ and it applies whichever agent happens to be active. That is also why it
156
+ is not routed through agents.manager.system_prompt_for() the way §21b's
157
+ memories are -- that function has no memory object, and refs are not an
158
+ agent's opt-in.
159
+
160
+ MUST NOT move inside prompts.system_prompts.with_catalogs(). That function
161
+ feeds pass_prompt(), so a reference attached in a chat session would land
162
+ in all ten passes of a research run the user started separately. §19's K6,
163
+ THIRD instance -- after §21b's memories, which is recorded as M13.
164
+
165
+ THE SUMMARIES ARE ANNOUNCED AS SUMMARIES OF SOMETHING ELSE. A referenced
166
+ thread is not this conversation's history, and a model that cannot tell
167
+ the difference will answer as though the user said things they said
168
+ elsewhere. Same reason §21a's compaction summary carries SUMMARY_PREFIX.
169
+ """
170
+ refs = memory.extra.get("refs") or []
171
+ if not refs:
172
+ return system_prompt
173
+
174
+ # CAPPED, AND THE CAP IS STATED IN THE TEXT (M14). The attach path already
175
+ # refuses to exceed it, so this is the second line of defence rather than
176
+ # the first -- but MAX_INJECTED_REFS can be lowered between sessions, and a
177
+ # thread carrying four refs under a cap of three must drop one visibly
178
+ # rather than silently.
179
+ cap = config.MAX_INJECTED_REFS
180
+ shown, total = refs[:cap], len(refs)
181
+ header = ("## Referenced conversations\n\n"
182
+ "Distilled summaries of OTHER conversations, attached by the "
183
+ "user for context. They are NOT part of this conversation's "
184
+ "history -- nothing here was said to you in this thread.")
185
+ if total > cap:
186
+ logger.warning(
187
+ "Injecting %s of %s attached references; the rest are not in this "
188
+ "prompt. Raise config.MAX_INJECTED_REFS or drop one with "
189
+ "/ref --clear.", cap, total)
190
+ header = (f"{header}\n\nShowing {cap} of {total} attached references. "
191
+ f"The others exist and are not included here.")
192
+
193
+ lines = [header]
194
+ for ref in shown:
195
+ label = ref.get("label") or "(no preview)"
196
+ lines.append(f"\n### {label}\n(thread {ref.get('thread_id')})\n"
197
+ f"{ref.get('summary', '')}")
198
+ return f"{system_prompt}\n\n" + "\n".join(lines)
199
+
200
+
201
+ def attach_ref(memory: ConversationMemory, thread_id, summary: str,
202
+ label: str = "") -> tuple:
203
+ """Attach one thread's summary to `memory`'s thread. Returns (ok, message).
204
+
205
+ ONE writer for both shells, like with_refs is one reader: the CLI's --ref
206
+ and the TUI's /ref must agree about the cap, about re-attaching the same
207
+ thread, and about the stored shape, or the two disagree about what a
208
+ reference IS.
209
+
210
+ REFUSES AT THE CAP rather than dropping the oldest. A reference is
211
+ something a person chose by name; making room by discarding one silently
212
+ is what §21b's "removal is by id, never by substring" rule exists to
213
+ prevent. The message names the command that makes room.
214
+
215
+ Re-attaching a thread already attached REPLACES its summary rather than
216
+ adding a second entry -- the newer summary is strictly better information
217
+ about the same source, and two entries for one thread would spend the cap
218
+ on a duplicate.
219
+ """
220
+ from datetime import datetime, timezone
221
+
222
+ refs = [dict(r) for r in (memory.extra.get("refs") or [])]
223
+ key = str(thread_id)
224
+ existing = next((r for r in refs if r.get("thread_id") == key), None)
225
+ if existing is None and len(refs) >= config.MAX_INJECTED_REFS:
226
+ return False, (
227
+ f"Already referencing {len(refs)} conversations, which is the "
228
+ f"limit (config.MAX_INJECTED_REFS). Drop one with /ref --clear "
229
+ f"first -- none is removed automatically.")
230
+
231
+ entry = {
232
+ "thread_id": key,
233
+ "summary": summary,
234
+ "label": label,
235
+ "attached_at": datetime.now(timezone.utc).isoformat(),
236
+ }
237
+ if existing is not None:
238
+ refs[refs.index(existing)] = entry
239
+ verb = "Updated the reference to"
240
+ else:
241
+ refs.append(entry)
242
+ verb = "Now referencing"
243
+ memory.set_extra("refs", refs)
244
+ return True, f"{verb} thread {key}."
245
+
246
+
247
+ def clear_refs(memory: ConversationMemory) -> int:
248
+ """Drop every attached reference. Returns how many were dropped.
249
+
250
+ All or nothing, deliberately: a per-reference removal needs a stable way
251
+ for a person to name one, and the only handle a ref has is a thread uuid
252
+ the user would have to retype. `/ref --list` then `/ref` to re-attach the
253
+ ones still wanted is two steps with nothing to mistype.
254
+ """
255
+ refs = memory.extra.get("refs") or []
256
+ if refs:
257
+ memory.set_extra("refs", None)
258
+ return len(refs)
259
+
260
+
261
+ def _authorization_kwargs(authorization) -> dict:
262
+ """Unpack a RunAuthorization into _run()'s primitives.
263
+
264
+ ONE unpacking point, so the pipeline entry point and its JSON-retry
265
+ twin cannot disagree about what a bundle means. The retry re-enters the
266
+ SAME pass thread through continue_conversation(): passing the bundle
267
+ there and not here (or the reverse) would strip a pass's tools on
268
+ exactly the attempt where the model is already struggling, and the only
269
+ symptom would be a second malformed answer.
270
+ """
271
+ if authorization is None:
272
+ return {}
273
+ return {
274
+ "granted_tools": authorization.granted_tools or None,
275
+ "grant_budget": authorization.budget,
276
+ "response_channel": authorization.provider,
277
+ }
278
+
279
+
280
+ def advertisement_facts(authorization=None, response_channel=None,
281
+ granted_tools=None) -> tuple:
282
+ """(callable_only, granted) for a caller BUILDING A PROMPT (#68).
283
+
284
+ Sibling of _authorization_kwargs above, and there for the same
285
+ reason: one unpacking point, so the places that must agree cannot
286
+ disagree about what a bundle means. The difference is only WHEN --
287
+ _run() computes `headless = response_channel is None` after the
288
+ bundle has been unpacked into its primitives, and a system prompt
289
+ is assembled before _run() is called at all.
290
+
291
+ Accepts either a bundle or the loose primitives, because
292
+ run_agent_conversation takes both and rejects being given both.
293
+ The bundle wins when present, which is the same precedence
294
+ _authorization_kwargs applies.
295
+
296
+ A NULL BUNDLE MEANS HEADLESS, not "unknown": _authorization_kwargs
297
+ returns {} for None, so response_channel stays None and _run reads
298
+ that as headless. Answering anything else here would advertise a
299
+ catalog to exactly the runs that cannot use it.
300
+ """
301
+ if authorization is not None:
302
+ return (authorization.provider is None,
303
+ set(authorization.granted_tools or ()))
304
+ return (response_channel is None, set(granted_tools or ()))
305
+
306
+
307
+ def _denial_reason(tool_name: str, response_channel, context) -> str:
308
+ """Why a gated call was refused, in terms the model can act on (#158).
309
+
310
+ The loop already learned this lesson one branch over -- see the
311
+ reachability comment above the approval block: headless, "requires
312
+ approval and was not given" tells the model to retry WITH approval,
313
+ which is the one thing that cannot happen there. It retries, spends a
314
+ step against max_steps, and gets the same sentence back. That is the
315
+ fetch_url damage class, which §15\\D24 fixed for undeclared permissions
316
+ and left open on this axis.
317
+
318
+ THE DISTINCTION IS WHETHER THE ARGUMENTS CAUSED THE GATE, and it is
319
+ asked the same way schemas(callable_only=…) asks it -- with empty
320
+ params, through the one function both dispatch() and this loop already
321
+ share. A tool gated by NAME is uncallable in this run however it is
322
+ called, so the useful thing to say is "stop". A tool gated by THESE
323
+ PARAMS survived that filter and was advertised precisely because some
324
+ other call shape is callable, so the useful thing to say is "vary the
325
+ arguments" -- which is exactly the case #158 measured: `write` is
326
+ advertised in a headless pass and denied only for paths outside the
327
+ workspace.
328
+
329
+ No tool name and no notion of a path appears here. Which argument to
330
+ vary is the model's problem and the tool's schema already describes it;
331
+ putting "try a path inside the workspace" in this file is the thing
332
+ ToolSpec.grant_scope and request_kind exist to prevent.
333
+ """
334
+ if response_channel is not None:
335
+ # Somebody was asked and declined. Nothing to reframe -- the answer
336
+ # was a decision, not an absence, and telling the model the call
337
+ # was unavailable would misreport a person's "no" as a limitation.
338
+ return f"{tool_name} requires approval and was not given"
339
+ if registry.approval_needed(tool_name, {}, context):
340
+ return (f"{tool_name} requires approval and this run has no way to "
341
+ f"ask, so it is unavailable however it is called. Do not "
342
+ f"retry it.")
343
+ return (f"{tool_name} requires approval for these arguments and this "
344
+ f"run has no way to ask. It may not need approval with "
345
+ f"different arguments.")
346
+
347
+
348
+ def _obtain_approval(response_channel, tool_name: str, params: dict,
349
+ notice, request_payload=None, rationale=None,
350
+ headline=None) -> tuple:
351
+ """Ask the human about one gated call. Returns (approved, grant).
352
+
353
+ ONE route since §23. There were two -- a queue.Queue the TUI's worker
354
+ parked on while its UI thread rendered a modal, and an
355
+ ApprovalProvider callable, which existed only because
356
+ run_to_completion() DISCARDS the LoopEvent carrying the question, so
357
+ the CLI and the pipeline would have blocked on a queue with nothing
358
+ displayed. Both were the same exchange with different plumbing, and
359
+ the shells now supply one ResponseChannel whose `ask` does whichever
360
+ of those two things that shell needs.
361
+
362
+ TWO RETURN VALUES, because §23 AC1b made one question's answer richer
363
+ than a yes/no. A subagent sign-off answers with the SUBSET of tools
364
+ the child may then use unprompted -- None to refuse the spawn, an
365
+ empty set to allow it with nothing granted, a non-empty set to grant
366
+ those. `grant` is None for every other kind, which carries no subset.
367
+
368
+ Which kind to ask comes from the REGISTRY, not from a name test here:
369
+ the loop asks what shape of question a tool needs the same way it asks
370
+ whether it needs one at all. `request_payload` is passed in already
371
+ computed, because the caller needs it before this point to look up the
372
+ sign-off memo -- and building a subagent's candidate list twice per
373
+ call would be wasteful and could disagree with itself.
374
+
375
+ `rationale` is the agent's own account of why it wants this call
376
+ (§42, RA5), and it is passed in already resolved for the same
377
+ reason: the caller needs it for the LoopEvent too, and looking it
378
+ up twice invites the two from disagreeing. DISPLAY ONLY -- it
379
+ reaches the prompt and nothing that decides anything.
380
+
381
+ `headline` is the SUBJECT of the question (§46, EP2) -- for `shell`
382
+ the command itself -- and travels for exactly the same reasons,
383
+ resolved once by the caller and display-only. It is also already
384
+ inside `params`, which every prompt renders whole; what carrying it
385
+ separately buys is a shell's ability to place it where it cannot
386
+ scroll out of view, without learning a parameter name.
387
+ """
388
+ kind = registry.request_kind(tool_name)
389
+ payload = {"tool_name": tool_name, "params": params,
390
+ "rationale": rationale, "headline": headline}
391
+ payload.update(request_payload or {})
392
+ answer = interaction.ask(response_channel, interaction.Request(
393
+ kind=kind, payload=payload, notice=notice))
394
+ if kind == interaction.SUBAGENT_SIGNOFF:
395
+ # None denies the spawn; any set (including empty) allows it.
396
+ return answer is not None, answer
397
+ return bool(answer), None
398
+
399
+
400
+ # ROADMAP_v2 §21. Notice kinds that describe a STANDING CONDITION rather
401
+ # than an event: they stay true on every step until something changes, so
402
+ # emitting them per step fills a transcript with the same line and trains
403
+ # the reader past it -- which is exactly the reader you need when the one
404
+ # that matters arrives. A compaction that actually happened is an event and
405
+ # repeats legitimately, once per compaction.
406
+ _ONCE_PER_RUN_NOTICES = (
407
+ "compaction_warning", # still approaching the threshold
408
+ "compaction_blocked", # the keep floors still protect everything
409
+ "compaction_failed", # provider down, or the compactor returned
410
+ # nothing usable (#136) -- and since batch 20
411
+ # this kind is also the LATCH: seen once, no
412
+ # further compact() calls this turn
413
+ )
414
+
415
+
416
+ def _derived_compaction_mode(memory) -> str:
417
+ """The compaction mode for a caller that did not say (#43, batch 16).
418
+
419
+ The mode is a property of WHAT THE THREAD IS, not of the call site:
420
+ `research_pass` and `subagent` threads are machinery -- unattended,
421
+ already returning a distillation, and (for a retry thread) holding
422
+ exactly the turns §3 exists to keep in front of the model -- so M6's
423
+ backstop applies WHEREVER they are re-entered. A chat thread is a
424
+ conversation someone is present for; the ordinary working-set trigger
425
+ is the feature working as designed there.
426
+
427
+ This closes #43 by construction rather than by forwarding: json_retry
428
+ and the §20 reviewer could not express a mode at all before, and a
429
+ parameter they had to remember would have re-created the D24/R13
430
+ failure shape -- correct today, one forgotten call site from wrong
431
+ again. Unknown kinds fail toward the chat answer: routine compaction
432
+ of an unforeseen thread kind is the conservative default, and a
433
+ future kind states its own answer here when it arrives.
434
+ """
435
+ kind = getattr(memory, "kind", THREAD_KIND_CHAT)
436
+ if kind in (THREAD_KIND_RESEARCH_PASS, THREAD_KIND_SUBAGENT):
437
+ return "backstop"
438
+ return "working_set"
439
+
440
+
441
+ def _already_said(notices, kind: str) -> bool:
442
+ return kind in _ONCE_PER_RUN_NOTICES and any(
443
+ n["kind"] == kind for n in notices)
444
+
445
+
446
+ def _compactor_depth(context) -> int:
447
+ """Where a compaction started from THIS loop sits in the agent stack.
448
+
449
+ One below whoever is running: a compactor spawned from a subagent at
450
+ depth 1 is at 2, exactly as a subagent spawned from there would be.
451
+ Derived from the ToolContext rather than passed down, because
452
+ subagent_depth is already the number C3 bounds and a second count of
453
+ the same thing is the shape §22 spent a section removing.
454
+ """
455
+ return (context.subagent_depth if context is not None else 0) + 1
456
+
457
+
458
+ def _maybe_compact(memory, model, provider_name, notices, mode,
459
+ activity=None, depth: int = 1):
460
+ """Evaluate §21's trigger and act on it. A generator, so the caller
461
+ yields from it and the notice reaches a live UI as it happens.
462
+
463
+ Every notice goes BOTH ways -- appended to `notices`, which _run()
464
+ attaches to the response, and yielded as a LoopEvent. The event alone
465
+ would be invisible to the CLI and the pipeline, which drain the
466
+ generator through run_to_completion() and discard everything that is
467
+ not final. §20 and §25 each shipped that defect once.
468
+
469
+ Outcome routing (batch 16, #44). compact() returns a status dict now,
470
+ and this is where reportability is decided:
471
+
472
+ folded notice every occurrence -- an event, and repeats
473
+ legitimately, once per compaction.
474
+ blocked /
475
+ all-pinned /
476
+ missing-agent STANDING CONDITIONS: notice AND WARNING together,
477
+ ONCE PER RUN under the same dedup list. The two
478
+ WARNINGs used to live inside compact() and fired on
479
+ EVERY evaluation -- up to MAX_ITERATIONS + 1
480
+ identical lines per turn on a thread that stays over
481
+ trigger and fully floored, which is how a reader
482
+ gets trained past the warnings that do not repeat.
483
+ Same defect `context_limit()` had before it learned
484
+ to warn once per model.
485
+ no-progress /
486
+ reentrant no notice at all: ordinary non-events (nothing new
487
+ to fold) or already carried by _summarize's own
488
+ per-call warnings.
489
+
490
+ THE LATCH (#136, batch 20). compaction_failed is both a notice and a
491
+ STOP for the run: once this turn has seen one -- a provider error, or
492
+ an empty compactor response (compact() now reports that as
493
+ status="failed" with the same kind) -- _maybe_compact returns before
494
+ calling compact() again. Without the gate the dedup above only
495
+ silenced the MESSAGE while every step still spent a full compactor
496
+ model call on an attempt that had just failed; measured at seven
497
+ calls in one six-step turn on the thread's first crossing. The latch
498
+ is per-turn by construction -- `notices` dies with _run() -- so the
499
+ next turn retries cleanly and a transient provider hiccup costs one
500
+ wasted call, not one per step. Deliberately NOT latched: no-progress,
501
+ reentrant, blocked and all-pinned exit before any model call, cost
502
+ only storage reads, and are self-correcting if state shifts.
503
+
504
+ Failure here is contained. Compaction is a model call, and a provider
505
+ error during it must not take down a turn that was otherwise fine --
506
+ the same call §20 made for its review stage. The thread simply stays
507
+ uncompacted and the next call fails or succeeds on its own merits,
508
+ which is strictly better than failing this one on its behalf.
509
+
510
+ Takes no authorization, deliberately. The compactor declares
511
+ `allowed_tools: []`, so there is nothing for a grant to apply to --
512
+ a parameter here would read as security-relevant and could not be.
513
+ """
514
+ from core import compaction
515
+
516
+ used = memory.last_input_tokens or compaction.estimated_tokens(memory.messages)
517
+ action = compaction.should_compact(used, model, mode,
518
+ provider_name=provider_name)
519
+ if not action:
520
+ return
521
+ if action == "warn":
522
+ if _already_said(notices, "compaction_warning"):
523
+ return
524
+ notice = {
525
+ "kind": "compaction_warning",
526
+ "text": ("Approaching the compaction threshold — older messages "
527
+ "may be summarized soon."),
528
+ }
529
+ notices.append(notice)
530
+ yield LoopEvent(notice=notice)
531
+ return
532
+ # THE LATCH (#136): a failed attempt already spent its model call this
533
+ # turn. Everything below the warn branch re-enters compact() on every
534
+ # step while the trigger stays open, so without this early return the
535
+ # dedup below only hides the message and the spend repeats. See the
536
+ # docstring's latch paragraph for why only compaction_failed gates.
537
+ if _already_said(notices, "compaction_failed"):
538
+ return
539
+ try:
540
+ # The current-turn floor (M5), computed HERE rather than at the
541
+ # top of the turn. Archive-space makes it time-invariant within
542
+ # a turn -- the archive gains assistant and tool rows as the
543
+ # turn runs but no new USER row, so the answer is the same
544
+ # whenever it is asked. Asking lazily keeps a storage read off
545
+ # every turn that is nowhere near the threshold.
546
+ outcome = compaction.compact(
547
+ memory, model, provider_name,
548
+ current_turn_start=memory.completed_turns(),
549
+ # Batch 59. The compactor is an agent-shaped run nested inside
550
+ # THIS turn, so it is drawn one level below whatever is running
551
+ # -- which is the depth the caller computed from its context,
552
+ # not a constant. `activity` is None everywhere but the TUI.
553
+ activity=activity, depth=depth)
554
+ except Exception as e: # noqa: BLE001 -- contained on purpose, see above
555
+ logger.exception("Compaction failed; continuing uncompacted.")
556
+ notice = {
557
+ "kind": "compaction_failed",
558
+ "text": f"Could not compact this conversation: {e}",
559
+ }
560
+ if not _already_said(notices, notice["kind"]):
561
+ notices.append(notice)
562
+ yield LoopEvent(notice=notice)
563
+ return
564
+
565
+ kind = outcome.get("kind")
566
+ if kind is None:
567
+ return
568
+ if _already_said(notices, kind):
569
+ return
570
+ if kind == "compaction_blocked":
571
+ # See the docstring's standing-condition row: one WARNING per run,
572
+ # said in the same breath as the one notice, never per evaluation.
573
+ logger.warning("Compaction cannot proceed on thread %s: %s",
574
+ memory.thread_id, outcome["text"])
575
+ notices.append({"kind": kind, "text": outcome["text"]})
576
+ yield LoopEvent(notice=notices[-1])
577
+
578
+
579
+ def run_to_completion(gen) -> ModelResponse:
580
+ """Consumes a _run() generator fully, discarding intermediate events,
581
+ and returns the final ModelResponse — this is what keeps
582
+ run_agent_conversation() and continue_conversation(), the two public
583
+ entry points, synchronous for every caller.
584
+
585
+ It reads the response off the TERMINAL EVENT, which is right for a
586
+ bare _run(): that generator returns nothing. A generator that returns
587
+ its own value -- stream_deep_research_mode(), whose response carries
588
+ `thread_id` attached after the loop finished -- needs the other
589
+ question asked, and `tests/conftest.drain` is where that lives now
590
+ that the pass wrapper is gone."""
591
+ final = None
592
+ for event in gen:
593
+ if event.final_response is not None:
594
+ final = event.final_response
595
+ if final is None:
596
+ raise RuntimeError(
597
+ "Generator completed without yielding a final_response"
598
+ )
599
+ return final
600
+
601
+
602
+ # #4 (batch 27). The wrappers' default for max_total_tokens: resolve the
603
+ # configured spend cap (settings.json max_token_budget) at CALL time. A
604
+ # sentinel rather than a plain None default because explicit-None already
605
+ # MEANS something in _run()'s signature -- genuinely uncapped -- and tests
606
+ # pass it deliberately. Resolution lives at the call site, not at def time,
607
+ # so a settings.json written after import is still honoured.
608
+ _SPEND_UNSET = object()
609
+
610
+
611
+ def _resolve_spend_cap(max_total_tokens) -> Optional[int]:
612
+ """_SPEND_UNSET -> the configured cap; anything else passes through."""
613
+ if max_total_tokens is _SPEND_UNSET:
614
+ from core.config_loader import spend_cap
615
+ return spend_cap()
616
+ return max_total_tokens
617
+
618
+
619
+ class RunAgentLoop:
620
+
621
+ @staticmethod
622
+ def _run(
623
+ memory: ConversationMemory,
624
+ system_prompt: str,
625
+ provider_name: str,
626
+ model: str,
627
+ context: Optional[ToolContext],
628
+ max_steps: int,
629
+ max_total_tokens: Optional[int] = None,
630
+ temperature: Optional[float] = None,
631
+ effort: Optional[str] = None,
632
+ response_channel=None,
633
+ granted_tools: Optional[set] = None,
634
+ grant_budget=None,
635
+ compaction_mode: Optional[str] = None,
636
+ activity=None,
637
+ ):
638
+ """Generator yielding LoopEvent objects as the loop progresses.
639
+
640
+ context: an optional ToolContext (§15) restricting which tools are
641
+ available and which need approval this run. None means only global
642
+ config policy applies, which is every production caller until §18
643
+ introduces agents. This replaced an `allowed_tools` list that did
644
+ the same job one layer up: the loop no longer decides tool
645
+ membership itself, it just hands the context to the registry and
646
+ lets security/permissions.py compose the layers.
647
+
648
+ response_channel: an optional core.interaction.ResponseChannel --
649
+ the ONE way this loop asks a human anything (§23 AC1). None means
650
+ nothing can be asked, and any call that would need approval is
651
+ denied. Replaced a queue.Queue plus a separate ApprovalProvider,
652
+ which were the same exchange with two sets of plumbing; a shell
653
+ that needs the thread handoff (the TUI parks its worker while the
654
+ UI thread renders a modal) does that inside its own `ask`.
655
+
656
+ granted_tools: names the user has ALREADY approved for this run
657
+ (§18 subagent sign-off). A spawned subagent receives the set the
658
+ user signed off when approving the spawn, so it does not re-ask
659
+ for tools already covered. A grant, not a policy relaxation: it
660
+ records an approval given early, which is why it lives on RunInfo
661
+ rather than in the ToolContext's approval_overrides (those OR, and
662
+ so can only tighten).
663
+
664
+ grant_budget: a core.approval.GrantBudget capping how many granted
665
+ calls may skip the prompt (§25 R6), or None for no cap. Shared by
666
+ reference across every pass of a pipeline run. Exhaustion makes
667
+ the grant stop applying, so calls fall back to being asked.
668
+
669
+ compaction_mode (§21): None -- the default since batch 16 (#43) --
670
+ means DERIVE from what the thread is: research_pass and subagent
671
+ threads get M6's backstop wherever they are re-entered, a chat
672
+ thread gets the ordinary working-set trigger. An explicit value
673
+ wins, and stream_deep_research_mode passes "backstop" explicitly
674
+ as the definition site. Before this, every continue_conversation
675
+ caller silently ran on the chat trigger: a pass's JSON retry and
676
+ the §20 reviewer evaluated a threshold their own threads were
677
+ never meant to answer to.
678
+ """
679
+ # §29/#45 belt. `_run` is a generator, so this fires on the first
680
+ # next() -- which for every production caller is immediate, since
681
+ # all three wrappers drain eagerly. It exists because the loader
682
+ # repairs agent files (#45) while a DIRECT caller bypasses that
683
+ # repair entirely: there is no file to lose here, only a bug worth
684
+ # naming, so this raises rather than substituting. The old failure
685
+ # was an AttributeError on `response.stop_reason` after an empty
686
+ # step loop -- a traceback pointing three layers away from the bad
687
+ # value that caused it.
688
+ if isinstance(max_steps, bool) or not isinstance(max_steps, int) \
689
+ or max_steps < 1:
690
+ raise ValueError(
691
+ f"max_steps must be a positive integer, got {max_steps!r}. "
692
+ f"Agent definitions carrying an invalid max_steps are "
693
+ f"repaired by the loader to {config.MAX_ITERATIONS} "
694
+ f"(config.MAX_ITERATIONS); direct callers must pass a valid "
695
+ f"value themselves.")
696
+ client = api_initialization(provider_name)
697
+ # Validate the level against the model that will RECEIVE it, here
698
+ # rather than at each caller. Every path -- CLI chat, each research
699
+ # pass, a TUI turn, a spawned subagent -- reaches the wire through
700
+ # this function, so this is the one place that cannot be bypassed
701
+ # by a new caller forgetting to check. It also makes the agent case
702
+ # correct by construction: an agent declaring its own model is
703
+ # validated against THAT model, not the one the parent was using.
704
+ effort = effort_for(client, provider_name, model, effort)
705
+ # Prime the context-window cache on the same one-per-process
706
+ # footing, and HERE for effort_for's reason: this is the one place
707
+ # every send crosses, so a new caller cannot forget it. On
708
+ # ANTHROPIC it is a pure cache hit -- effort_for's own retrieve
709
+ # already recorded the window off the ModelInfo it fetched -- so
710
+ # the only providers this actually calls for are Google and the
711
+ # v1-compatible roster. The result is read back, without a client,
712
+ # by compaction.context_limit(); a failure here is warned and
713
+ # simply leaves the static table answering.
714
+ context_window_for(client, provider_name, model)
715
+ # §18 headless callability rule (user-widened from approval-only):
716
+ # with no way to ask, a tool that needs approval is uncallable in
717
+ # this configuration, so it is not advertised -- but named once in
718
+ # a WARNING so the hiding is never silent.
719
+ #
720
+ # §25: "no way to ask" is not "not a TUI". A research pass with a
721
+ # channel CAN ask, so hiding its gated tools would report a
722
+ # limitation the run does not have. §23 made this one condition
723
+ # again -- it was two while there were two routes.
724
+ #
725
+ # §25 R15: and "nothing can ask" is not "nothing has answered". A
726
+ # GRANT is an answer given before the call existed, so a granted
727
+ # tool is callable with no channel at all -- which is the whole
728
+ # point of `--grant-tools` on an unattended run. #156: this filter
729
+ # could not see the grant, so a grant-only run advertised exactly
730
+ # what an unflagged one did while the CLI printed that it had
731
+ # authorised something. The enforcement path below honoured the
732
+ # grant correctly; the model was simply never told the tool was
733
+ # there, so it never emitted the call.
734
+ #
735
+ # run_info IS BUILT ABOVE THIS, not below it as it was before R15.
736
+ # The schemas call now needs the grant, and the grant lives on
737
+ # run_info -- building it after would read the name before it is
738
+ # bound. Nothing else moved: it depends only on values already in
739
+ # scope here.
740
+ run_info = RunInfo(
741
+ model=model, provider_name=provider_name, effort=effort,
742
+ granted_tools=set(granted_tools or ()),
743
+ grant_budget=grant_budget)
744
+ headless = response_channel is None
745
+ # The BUDGET is deliberately not consulted here. Advertisement is
746
+ # decided once per step; R6 makes exhaustion fall back to asking
747
+ # (or, headless, to denying) at CALL time, which is a per-call
748
+ # answer. Hiding a tool mid-run when the ceiling ran out would
749
+ # change what the model can see between one step and the next for
750
+ # a reason it cannot observe, and the model would read that as the
751
+ # tool having ceased to exist.
752
+ tool_schemas = registry.schemas(
753
+ context, callable_only=headless, granted=run_info.granted_tools)
754
+ if headless:
755
+ hidden = tuple(sorted(registry.headless_hidden(
756
+ context, granted=run_info.granted_tools)))
757
+ if hidden and hidden not in _headless_notices_shown:
758
+ _headless_notices_shown.add(hidden)
759
+ logger.warning(
760
+ "Headless run (no permission channel): not advertising "
761
+ "%s -- they require approval and nothing here can "
762
+ "grant it.",
763
+ ", ".join(hidden),
764
+ )
765
+ total_tokens_used = 0
766
+ # #4: the size instrument, kept strictly beside the spend meter.
767
+ # turn_baseline_input is the FIRST call's prompt -- the thread as
768
+ # this turn inherited it, which this turn did not add. Every later
769
+ # step contributes its output plus whatever NEW material entered
770
+ # the context (positive input deltas; a mid-turn compaction SHRINKS
771
+ # the prompt, and clamping at zero keeps that shrink from being
772
+ # read as negative growth). turn_billed/turn_new ride every
773
+ # response; memory.record_billed accumulates thread-since-resume.
774
+ turn_baseline_input: Optional[int] = None
775
+ turn_prev_input = 0
776
+ turn_new_tokens = 0
777
+ # Batch 61, the third instrument. Output ALONE -- see the field's
778
+ # comment on ModelResponse for why neither of the two above can
779
+ # stand in for it once a duration is divided into the result.
780
+ turn_output_tokens = 0
781
+ # §25 audit trail. Attached to the response object below rather
782
+ # than at each of the three return points, so a stop condition
783
+ # added later cannot forget to carry it -- the list is shared by
784
+ # reference and keeps filling as tools dispatch.
785
+ granted_calls: list = []
786
+ # §21, same pattern and the same reason.
787
+ notices: list = []
788
+
789
+ # Turn boundary (M3). The primary trigger: the thread is settled,
790
+ # nothing is mid-flight, and last_input_tokens is the provider's
791
+ # own measurement of what the previous call was sent.
792
+ yield from _maybe_compact(
793
+ memory, model, provider_name, notices,
794
+ compaction_mode
795
+ if compaction_mode is not None
796
+ else _derived_compaction_mode(memory),
797
+ activity=activity, depth=_compactor_depth(context))
798
+
799
+ response = None
800
+ for _ in range(max_steps):
801
+ response = None
802
+ for token in call_model_stream(
803
+ client, provider_name, model, memory.messages,
804
+ system_prompt, tool_schemas, temperature, effort,
805
+ ):
806
+ if token.text_delta:
807
+ yield LoopEvent(token_delta=token.text_delta)
808
+ if token.thinking_delta:
809
+ yield LoopEvent(thinking_delta=token.thinking_delta)
810
+ if token.final_response is not None:
811
+ response = token.final_response
812
+
813
+ if response is None:
814
+ raise RuntimeError(
815
+ "call_model_stream completed without yielding a final response"
816
+ )
817
+
818
+ usage_in = response.usage.get("input_tokens", 0)
819
+ usage_out = response.usage.get("output_tokens", 0)
820
+ total_tokens_used += usage_in + usage_out
821
+ if turn_baseline_input is None:
822
+ # First call of the turn: its prompt IS the inherited
823
+ # thread, so only this step's OUTPUT counts as new.
824
+ turn_baseline_input = usage_in
825
+ else:
826
+ turn_new_tokens += max(0, usage_in - turn_prev_input)
827
+ turn_new_tokens += usage_out
828
+ turn_output_tokens += usage_out
829
+ turn_prev_input = usage_in
830
+ memory.record_billed(usage_in + usage_out)
831
+ response.turn_billed_tokens = total_tokens_used
832
+ response.turn_new_tokens = turn_new_tokens
833
+ response.turn_output_tokens = turn_output_tokens
834
+
835
+ # D20: persist EVERY assistant turn BEFORE any branching.
836
+ memory.add_assistant_message(response)
837
+ response.granted_calls = granted_calls
838
+ response.notices = notices
839
+ # §21: the provider's own count of everything it was just
840
+ # sent. Recorded here rather than at the trigger so a thread
841
+ # resumed in a later process starts from a real measurement
842
+ # instead of a character estimate.
843
+ memory.record_input_tokens(response.usage.get("input_tokens", 0))
844
+
845
+ if max_total_tokens is not None and total_tokens_used >= max_total_tokens:
846
+ response.stop_reason = "token_budget_exceeded"
847
+ yield LoopEvent(
848
+ final_response=response, stop_reason=response.stop_reason,
849
+ )
850
+ return
851
+
852
+ if not response.tool_calls:
853
+ response.stop_reason = "complete"
854
+ yield LoopEvent(
855
+ final_response=response, stop_reason=response.stop_reason,
856
+ )
857
+ return
858
+
859
+ for call in response.tool_calls:
860
+ yield LoopEvent(tool_call_start={
861
+ "id": call.id, "name": call.name, "input": call.input,
862
+ })
863
+
864
+ # Single source of truth with dispatch(): approval_needed()
865
+ # ORs the tool's own path/command-dependent approval_check
866
+ # with the config/context requirement (§15). Both callers
867
+ # go through it so they cannot diverge — a path-dependent
868
+ # tool must yield permission_request so the TUI user can
869
+ # approve, not be silently denied by one check while the
870
+ # other reports no approval needed.
871
+ #
872
+ # There is no separate allowed_tools membership test here
873
+ # any more: dispatch() calls is_tool_allowed(name, context),
874
+ # which raises ToolCallDenied with a message distinguishing
875
+ # a context restriction from a global policy denial.
876
+ needs_approval = registry.approval_needed(
877
+ call.name, call.input, context)
878
+ # Reachability before approval. Asking first meant a
879
+ # context-excluded tool prompted the user, who clicked
880
+ # Allow, and was then denied anyway -- and headless, it
881
+ # reported "requires approval and was not given", telling
882
+ # the model to retry with approval when the actionable
883
+ # answer is "not available in this context". dispatch()
884
+ # still enforces the denial; this only stops asking a
885
+ # question whose answer cannot matter.
886
+ if needs_approval and not registry.is_allowed(call.name, context):
887
+ needs_approval = False
888
+ # §32 A7 (#70): the same rule for a refusal only the
889
+ # TOOL can see. spawn_subagent's unknown-agent and
890
+ # depth-limit checks lived inside run(), so both
891
+ # prompted first and errored second -- and for an
892
+ # unknown name the question had an empty notice and an
893
+ # empty candidate list, i.e. nothing to tick. The
894
+ # call is refused by dispatch with the tool's own
895
+ # reason; this only stops ASKING first. It also keeps
896
+ # J8's memo from being keyed on
897
+ # (spawn_subagent, None) -- a subject of None is the
898
+ # absence of the very thing the key exists to
899
+ # distinguish.
900
+ if needs_approval and registry.refusal_reason(
901
+ call.name, call.input, context):
902
+ needs_approval = False
903
+ # §33 W3 (#37): the same rule for a call whose arguments
904
+ # never parsed. Approving it could not make it runnable,
905
+ # so the question has no answer that changes anything --
906
+ # and headless it would report "requires approval and was
907
+ # not given", which is A7's exact wrong-cause failure.
908
+ if needs_approval and call.parse_error is not None:
909
+ needs_approval = False
910
+ # §18 sign-off (S1), narrowed by §25 (R2): a tool whose
911
+ # grant_scope is "run" is asked about once per run, not
912
+ # once per call. The loop names no tool -- it asks the
913
+ # registry, the same way it asks whether approval is
914
+ # needed at all.
915
+ #
916
+ # THREE conditions, and the last two are the §25 change:
917
+ #
918
+ # registry.grantable -- a tool deciding approval from its
919
+ # PARAMS was never consented to by name. Before this,
920
+ # a signed-off subagent could run any shell command for
921
+ # the rest of the turn on the strength of one prompt
922
+ # that said "shell". Re-checked here rather than
923
+ # trusted from the set, so a stale or hand-built grant
924
+ # cannot widen anything.
925
+ #
926
+ # grant_budget.take() -- pre-flight authorization trades
927
+ # a per-call decision for one up-front decision, and
928
+ # gives up the natural bound on how many times the
929
+ # tool runs. Exhaustion falls back to ASKING, not to
930
+ # failing: supervised runs continue, headless ones deny
931
+ # exactly as they would for any gated tool.
932
+ #
933
+ # §23 AC1b: the memo is keyed by (tool, SUBJECT), not by
934
+ # tool name alone. A subagent sign-off answers about one
935
+ # agent's candidate list, so reusing it for a different
936
+ # agent would grant a set nobody was shown -- which is
937
+ # what the name-only version did: approving a spawn of `a`
938
+ # silently covered a later spawn of `b` in the same turn.
939
+ # Tools with no subject (every other one) memo under None
940
+ # and behave exactly as before.
941
+ # Batch 60. THIS CALL'S authorization decision, and the
942
+ # error that stands in for it.
943
+ #
944
+ # THE ID, NOT A BOOLEAN, and that is the whole point of the
945
+ # shape. `authorized` began as a bool whose correctness
946
+ # rested on WHERE it was declared -- one indent level in and
947
+ # it covers one call, one level out and it covers every
948
+ # later call in the turn, which is the exact widening §25
949
+ # R11 refuses to make even for the one tool whose grant
950
+ # scope IS the run. That version was measured: hoisting it
951
+ # out of `for call in response.tool_calls:` survived the
952
+ # entire suite, because no path currently reaches dispatch
953
+ # with a stale flag that changes anything. Correct, and
954
+ # undefended.
955
+ #
956
+ # A call id cannot be hoisted into a lie. Moved out of the
957
+ # loop it holds the PREVIOUS call's id, the comparison at
958
+ # the dispatch site fails, and the call is DENIED -- the
959
+ # mistake fails closed instead of failing open. It rests on
960
+ # ids being distinct within a response, which is not a new
961
+ # assumption: add_tool_result pairs every tool_result to its
962
+ # tool_use by that same id (M4/D20), and a thread whose ids
963
+ # collide is already unresumable.
964
+ #
965
+ # Set in exactly two places -- the memo/grant hit below and
966
+ # the fresh approval further down -- and read in exactly
967
+ # one, the single dispatch() call at the end of this block.
968
+ authorized_call = None
969
+ denial = None
970
+ request_payload = registry.request_payload(
971
+ call.name, call.input, context)
972
+ subject = request_payload.get("subject")
973
+ remembered, signoff = run_info.recall_signoff(
974
+ call.name, subject)
975
+ # grant_policy on a NAME-LEVEL grant only -- R13's literal
976
+ # claim is that approving the NAME never carries the
977
+ # authority for a GRANT_NEVER tool, on either path, and
978
+ # until now no path enforced it. Batch 6 declared the
979
+ # policy and taught the two functions that decide what
980
+ # may be OFFERED to read it; this is the reader that
981
+ # decides whether a grant already in hand APPLIES.
982
+ # `spawn_subagent` was covered only because R14's
983
+ # subject condition happens to catch it; `remember`
984
+ # carries no subject, so a grant naming it dispatched a
985
+ # durable cross-session write with no prompt -- M17's
986
+ # scenario through the one door R13 left unlocked.
987
+ #
988
+ # IT MUST NOT REACH THE SIGN-OFF MEMO, and that is not a
989
+ # nicety: `spawn_subagent` is GRANT_NEVER, so testing
990
+ # the policy unconditionally here stops §23's memo
991
+ # applying and the same agent is asked about twice in
992
+ # one turn. R13 is about what approving a NAME buys; a
993
+ # memo is an answer somebody gave about this subject,
994
+ # which J8 and R14 govern instead. recall_signoff
995
+ # already separates them by what it returns -- the memo
996
+ # always carries a subset, the grant carries None.
997
+ answered_by_name = signoff is None
998
+ if (needs_approval
999
+ and remembered
1000
+ and registry.grantable(call.name)
1001
+ and not (answered_by_name
1002
+ and registry.grant_policy(call.name)
1003
+ == GRANT_NEVER)
1004
+ and (run_info.grant_budget is None
1005
+ or run_info.grant_budget.take())):
1006
+ needs_approval = False
1007
+ # Batch 60. The half that was missing: the loop
1008
+ # decided this call needs no fresh answer, and until now
1009
+ # it did not tell dispatch(), which re-checks approval on
1010
+ # its own and denied every call this branch let through.
1011
+ # A second spawn of one agent, a subagent calling a tool
1012
+ # it was signed off for, and a --grant-tools name in an
1013
+ # unattended run were all refused with "requires approval
1014
+ # and was not given" and no prompt.
1015
+ authorized_call = call.id
1016
+ granted_calls.append(
1017
+ {"tool": call.name, "params": call.input})
1018
+ # §21, and the reason this batch adds a notice at all:
1019
+ # a gated call proceeding with NO modal is the intended
1020
+ # behaviour (§25 R11 -- re-asking about the same tool
1021
+ # seconds later is noise), and it is also exactly what
1022
+ # the bug looked like from the outside. Silence was
1023
+ # indistinguishable from the bug.
1024
+ #
1025
+ # DELIBERATELY NOT in _ONCE_PER_RUN_NOTICES. That list
1026
+ # is for a STANDING CONDITION re-emitted on every step;
1027
+ # this fires only when a call actually reuses an answer,
1028
+ # which is an event -- the same reading that lets a
1029
+ # compaction that HAPPENED repeat once per compaction.
1030
+ # Once per run would go quiet again from the third spawn
1031
+ # onward, which is the silence that started this.
1032
+ #
1033
+ # Two texts, discriminated by `answered_by_name`, which
1034
+ # is already computed above: one sentence cannot honestly
1035
+ # cover both. A memo is something the user answered
1036
+ # minutes ago in this turn; a grant is something they
1037
+ # decided before the run existed, possibly on a command
1038
+ # line, and telling them they "answered earlier" would
1039
+ # describe a moment that never happened.
1040
+ notices.append({
1041
+ "kind": "approval_reused",
1042
+ "text": (
1043
+ "%s is running on the authorization granted "
1044
+ "before this run started." % call.name
1045
+ if answered_by_name else
1046
+ "Reusing your approval for %s, given earlier in "
1047
+ "this turn." % (subject or call.name)),
1048
+ })
1049
+ yield LoopEvent(notice=notices[-1])
1050
+ else:
1051
+ # Not reused: nothing is carried into dispatch unless
1052
+ # this call's own answer supplies it below.
1053
+ signoff = None
1054
+ if needs_approval:
1055
+ notice = registry.approval_notice(
1056
+ call.name, call.input, context)
1057
+ # §42 (RA5). The agent's own stated reason, asked
1058
+ # of the registry so the loop names no key -- the
1059
+ # same way it asks what KIND of question a tool
1060
+ # needs and whether it needs one at all. The VALUE
1061
+ # travels rather than the key, because the two
1062
+ # shells that render it are a Textual screen that
1063
+ # imports nothing from this package and a `print`
1064
+ # in main.py.
1065
+ rationale = registry.rationale_for(
1066
+ call.name, call.input)
1067
+ # §46 (EP2). The SUBJECT of the question, resolved
1068
+ # the same way and travelling the same route: for
1069
+ # `shell` it is the command, and a shell that shows
1070
+ # it must not have to know that. Every other tool
1071
+ # answers None and its prompt is unchanged.
1072
+ headline = registry.headline_for(
1073
+ call.name, call.input)
1074
+ yield LoopEvent(permission_request={
1075
+ "tool_name": call.name, "params": call.input,
1076
+ "notice": notice,
1077
+ "rationale": rationale,
1078
+ "headline": headline,
1079
+ })
1080
+ approved, signoff = _obtain_approval(
1081
+ response_channel, call.name, call.input, notice,
1082
+ request_payload, rationale=rationale,
1083
+ headline=headline)
1084
+ if approved:
1085
+ authorized_call = call.id
1086
+ # §25 R11: run-scope is a shortcut for a chat turn,
1087
+ # where re-asking about the same tool seconds later
1088
+ # is noise. A provider that declines it is saying
1089
+ # its whole purpose is per-call supervision --
1090
+ # attended mode -- and one yes must not silently
1091
+ # cover later calls there.
1092
+ #
1093
+ # This is ALSO the only thing that carries an answer
1094
+ # between calls in a turn, and it is asked of the
1095
+ # registry rather than assumed: `spawn_subagent` is
1096
+ # the sole tool declaring grant_scope "run", so
1097
+ # `shell`, `write` and `edit` are re-asked per call
1098
+ # however many times one turn calls them.
1099
+ if (registry.grant_scope(call.name) == "run"
1100
+ and (response_channel is None
1101
+ or response_channel.honour_run_scope)):
1102
+ run_info.remember_signoff(
1103
+ call.name, subject, signoff)
1104
+ else:
1105
+ denial = _denial_reason(
1106
+ call.name, response_channel, context)
1107
+ elif call.parse_error is not None:
1108
+ # §33 W1 (#37). The loop is the only layer that can
1109
+ # see this: dispatch() takes (name, params, context)
1110
+ # and the failure is a fact about the wire, not about
1111
+ # the params -- {} is a perfectly good empty dict by
1112
+ # the time it gets there.
1113
+ #
1114
+ # RETURNED as the tool's result, not raised, for
1115
+ # A10's reason: the model is told its arguments did not
1116
+ # arrive and can reissue the call, which is the whole
1117
+ # difference between losing one step and losing a
1118
+ # ten-pass run.
1119
+ denial = call.parse_error
1120
+
1121
+ # ONE dispatch call site (batch 60). There were two,
1122
+ # identical but for the two arguments that carry
1123
+ # authorization, and the one reached by an already-answered
1124
+ # call was the one that omitted them -- so every call this
1125
+ # loop decided was authorized got denied by dispatch's own
1126
+ # re-check. Two sites that must agree is the defect; one
1127
+ # site is the fix, and no future edit can teach them to
1128
+ # disagree again.
1129
+ #
1130
+ # dispatch() STILL RE-CHECKS, and must: it is a public
1131
+ # entry point and the fail-closed backstop. What changed is
1132
+ # that the loop now answers it.
1133
+ #
1134
+ # The callback is CONDITIONAL, and the condition is an
1135
+ # identity check rather than a truth test: this call, not
1136
+ # some call. See the declaration above for why the id and
1137
+ # not a flag.
1138
+ #
1139
+ # DELETING the condition -- `lambda n, p: True` outright --
1140
+ # is the one mutation here that no behaviour can catch, and
1141
+ # it is caught in tests/test_grants.py by reading this
1142
+ # source instead. Nothing can reach this line gated and
1143
+ # un-authorized today, because the three other reasons
1144
+ # `needs_approval` is turned off above are each
1145
+ # short-circuited INSIDE dispatch before its gate
1146
+ # (is_allowed raises, refusal_check returns, and a
1147
+ # parse_error never gets here at all). A fourth reason added
1148
+ # later would otherwise silently become "approve
1149
+ # everything", with every test still green.
1150
+ #
1151
+ # `signoff` needs no condition -- the memo branch above nulls
1152
+ # it on every path that is not a hit, so it is already None
1153
+ # everywhere it should be.
1154
+ if denial is not None:
1155
+ result = {"error": denial}
1156
+ else:
1157
+ try:
1158
+ result = registry.dispatch(
1159
+ call.name, call.input, context=context,
1160
+ approval_callback=(
1161
+ (lambda n, p: True)
1162
+ if authorized_call == call.id else None),
1163
+ parent_run=run_info,
1164
+ response_channel=response_channel,
1165
+ signoff=signoff,
1166
+ memory=memory,
1167
+ activity=activity,
1168
+ )
1169
+ except ToolCallDenied as e:
1170
+ result = {"error": str(e)}
1171
+ # §23 slice 2 (J10). A tool may attach a `notice` to its
1172
+ # result for the shell to show; it is forwarded as a
1173
+ # LoopEvent and REMOVED from what the model sees.
1174
+ #
1175
+ # Generic on purpose. The alternative was for the loop to
1176
+ # notice that `todo_write` had just run, which puts a tool
1177
+ # name in this file -- exactly what ToolSpec.grant_scope and
1178
+ # ToolSpec.request_kind exist to avoid. Any later tool that
1179
+ # wants to say something mid-turn gets this route free.
1180
+ #
1181
+ # POPPED, not copied: a notice is plumbing for the shell, and
1182
+ # leaving it in the result would send the TUI panel's trigger
1183
+ # to the model as if it were part of the tool's answer.
1184
+ if isinstance(result, dict) and "notice" in result:
1185
+ notice = result.pop("notice")
1186
+ if isinstance(notice, dict) and notice.get("kind"):
1187
+ yield LoopEvent(notice=notice)
1188
+ else:
1189
+ # A malformed notice is a bug in the tool, not a
1190
+ # reason to fail the call -- the result is still
1191
+ # good. Logged rather than raised, and the key is
1192
+ # gone either way so it cannot reach the model.
1193
+ logger.warning(
1194
+ "%s returned a malformed notice; dropping it: %r",
1195
+ call.name, notice)
1196
+
1197
+ # Persist BEFORE emitting (#42). A generator only advances
1198
+ # while someone iterates it, so emitting first made the row
1199
+ # depend on a consumer continuing to read -- and an
1200
+ # abandoned generator left the `tool_use` D20 already wrote
1201
+ # with no matching `tool_result`, which is M4's pairing
1202
+ # broken from the other side and an unresumable thread.
1203
+ # Same rule §22 gives _Progress.checkpoint() for trace
1204
+ # lines. The event carries nothing the persist needs, so
1205
+ # the order is free.
1206
+ memory.add_tool_result(call.id, result)
1207
+ yield LoopEvent(tool_result={"id": call.id, "result": result})
1208
+
1209
+ # The mid-turn valve (M3). A single turn can add far more than
1210
+ # any buffer covers -- MAX_READ_CHARS alone is ~12.5k tokens
1211
+ # per read call, with up to max_steps of them -- so waiting for
1212
+ # the next turn boundary would let a tool-heavy turn run the
1213
+ # thread into the provider's hard limit.
1214
+ #
1215
+ # Safe by construction rather than by timing: the current-turn
1216
+ # floor makes the current turn unfoldable, so this can only ever
1217
+ # summarize exchanges that were already complete when the turn
1218
+ # began. That is what made a mid-turn trigger acceptable at
1219
+ # all.
1220
+ #
1221
+ # The floor is `memory.completed_turns()`, asked inside
1222
+ # _maybe_compact. This comment used to name a `turn_start`
1223
+ # variable recorded at the top of the turn (audit #91); it moved
1224
+ # because archive-space turn counting is time-invariant WITHIN a
1225
+ # turn -- a completed turn cannot become incomplete -- so asking
1226
+ # lazily gives the same answer and keeps a storage read off every
1227
+ # turn nowhere near the threshold. See core/compaction.py's note
1228
+ # on why it is computed there rather than passed down.
1229
+ yield from _maybe_compact(
1230
+ memory, model, provider_name, notices,
1231
+ compaction_mode
1232
+ if compaction_mode is not None
1233
+ else _derived_compaction_mode(memory),
1234
+ activity=activity, depth=_compactor_depth(context))
1235
+
1236
+ # max_steps exhausted without an earlier return
1237
+ response.stop_reason = "max_steps_reached"
1238
+ yield LoopEvent(
1239
+ final_response=response, stop_reason=response.stop_reason,
1240
+ )
1241
+
1242
+ @staticmethod
1243
+ def run_agent_conversation(
1244
+ user_goal: str,
1245
+ model: str,
1246
+ provider_name: str = DEFAULT_PROVIDER,
1247
+ max_steps: int = config.MAX_ITERATIONS,
1248
+ max_total_tokens: Optional[int] = _SPEND_UNSET,
1249
+ thread_id: Optional[UUID] = None,
1250
+ temperature: Optional[float] = None,
1251
+ effort: Optional[str] = None,
1252
+ context: Optional[ToolContext] = None,
1253
+ system_prompt: Optional[str] = None,
1254
+ response_channel=None,
1255
+ granted_tools: Optional[set] = None,
1256
+ authorization=None,
1257
+ thread_kind: str = THREAD_KIND_CHAT,
1258
+ activity=None,
1259
+ ) -> ModelResponse:
1260
+ """Regular conversation — full tool set, default system prompt,
1261
+ one continuous thread. Pass thread_id to resume an existing
1262
+ thread; omit (or None) to start a fresh one. Pass context (§15) to
1263
+ run under an agent's tool restrictions; None means global policy
1264
+ only. Pass system_prompt (§18) when running AS an agent -- the
1265
+ TUI worker and spawn_subagent both build it via
1266
+ agents.manager.AgentManager.system_prompt_for();
1267
+ None keeps the default base + skill/agent catalogs.
1268
+
1269
+ A thread's persistent goal (§18 goal mode) is appended to WHICHEVER
1270
+ prompt is used, so it applies in every shell without the callers
1271
+ knowing about it.
1272
+
1273
+ response_channel / granted_tools exist for spawn_subagent (§18
1274
+ S1). Without the channel a spawned subagent ran HEADLESS while its
1275
+ parent could prompt, so it silently lost every approval-gated tool
1276
+ -- including all MCP tools -- and the same agent behaved
1277
+ differently depending on whether it was activated with /agent or
1278
+ spawned.
1279
+
1280
+ authorization (§20): a RunAuthorization, for an agent-shaped run
1281
+ that is part of a research run -- §20's reviewer is the first.
1282
+ §25 added the bundle to the pass entry point and
1283
+ continue_conversation and stopped here, because nothing
1284
+ agent-shaped needed it yet; a reviewer inheriting its run's grants
1285
+ and BUDGET does.
1286
+
1287
+ Mutually exclusive with granted_tools, and checked rather than
1288
+ silently resolved. The two spell the same underlying argument, so
1289
+ one would quietly win -- and if the bundle lost, a reviewer would
1290
+ run with no budget and no provider while looking authorised.
1291
+
1292
+ thread_kind (§27 T1) is what this entry point creates when it
1293
+ creates a thread. It defaults to "chat" because this is ALSO the
1294
+ plain-conversation entry point for both shells; the three callers
1295
+ that are not a human conversation -- spawn_subagent, §20's reviewer
1296
+ and §21a's compactor -- pass THREAD_KIND_SUBAGENT, which is what
1297
+ keeps their threads out of the picker. Ignored when thread_id is
1298
+ given: resuming does not reclassify."""
1299
+ if authorization is not None and granted_tools is not None:
1300
+ raise ValueError(
1301
+ "run_agent_conversation: pass authorization= or "
1302
+ "granted_tools=, not both -- they set the same underlying "
1303
+ "argument, so one would silently win."
1304
+ )
1305
+ memory = ConversationMemory(thread_id=thread_id, kind=thread_kind)
1306
+ memory.add_user_message(user_goal)
1307
+ # #68, and this branch is the one an unattended CLI run takes.
1308
+ _callable_only, _granted = advertisement_facts(
1309
+ authorization, response_channel, granted_tools)
1310
+ prompt = with_goal(
1311
+ system_prompt
1312
+ if system_prompt is not None
1313
+ else system_prompts.with_catalogs(
1314
+ DEFAULT_SYSTEM_PROMPT, callable_only=_callable_only,
1315
+ granted=_granted),
1316
+ memory,
1317
+ )
1318
+ # §21b (M13). Only when there is no agent-built prompt: an agent
1319
+ # run got its memories inside system_prompt_for(), and appending
1320
+ # again here would duplicate them. `agent=None` is what makes a
1321
+ # plain chat turn count as opted in -- see memories.manager.
1322
+ if system_prompt is None:
1323
+ prompt = with_memories(prompt)
1324
+ # §21c. UNCONDITIONAL, unlike memories above: a reference belongs to
1325
+ # the thread, not to an agent, so an agent-built prompt needs it too --
1326
+ # and system_prompt_for() has no memory to read it from.
1327
+ prompt = with_refs(prompt, memory)
1328
+ # §23 slice 2. Unconditional too, and for with_refs' reason: a
1329
+ # checklist is thread state, so an agent-built prompt needs it.
1330
+ prompt = with_todos(prompt, memory)
1331
+ auth_kwargs = (
1332
+ _authorization_kwargs(authorization) if authorization is not None
1333
+ else {"granted_tools": granted_tools}
1334
+ )
1335
+ # §23: the bundle's channel and the explicit parameter are now the
1336
+ # SAME kind of thing, so they can collide -- they could not before,
1337
+ # when one was a queue named permission_channel and the other an
1338
+ # ApprovalProvider named approval_provider, and both were passed.
1339
+ #
1340
+ # The explicit argument wins, preserving _obtain_approval's old
1341
+ # precedence ("the channel wins when both are present"). A caller
1342
+ # that passes one is describing THIS run; the bundle may have been
1343
+ # assembled for a pipeline several layers up.
1344
+ # Popped UNCONDITIONALLY, then merged. `a or kwargs.pop(...)`
1345
+ # short-circuits, so the key would survive in auth_kwargs on
1346
+ # exactly the path where an explicit channel was passed -- and
1347
+ # collide with it at the call below.
1348
+ bundled_channel = auth_kwargs.pop("response_channel", None)
1349
+ response_channel = response_channel or bundled_channel
1350
+ response = run_to_completion(RunAgentLoop._run(
1351
+ memory, prompt,
1352
+ provider_name, model, context,
1353
+ max_steps, _resolve_spend_cap(max_total_tokens),
1354
+ temperature=temperature, effort=effort,
1355
+ response_channel=response_channel, activity=activity,
1356
+ **auth_kwargs,
1357
+ ))
1358
+ response.thread_id = memory.thread_id
1359
+ return response
1360
+
1361
+ @staticmethod
1362
+ def stream_deep_research_mode(
1363
+ pass_input: str,
1364
+ model: str,
1365
+ pass_id: str,
1366
+ provider_name: str = DEFAULT_PROVIDER,
1367
+ max_steps: int = config.MAX_ITERATIONS,
1368
+ max_total_tokens: Optional[int] = _SPEND_UNSET,
1369
+ temperature: Optional[float] = None,
1370
+ effort: Optional[str] = None,
1371
+ context: Optional[ToolContext] = None,
1372
+ authorization=None,
1373
+ activity=None,
1374
+ ):
1375
+ """One research pass, as a generator of LoopEvents, RETURNING the
1376
+ ModelResponse (ROADMAP_v2 §26).
1377
+
1378
+ Exists because §22's P2 -- "a pass's LoopEvents do not propagate
1379
+ up" -- made a pass opaque while it ran. The pipeline could say
1380
+ which pass had started and nothing about what it was doing, so a
1381
+ pass making fourteen tool calls over several minutes looked
1382
+ identical to one that had hung.
1383
+
1384
+ P2 is AMENDED here, not repealed. What escapes this generator is
1385
+ still consumed inside core/reasoning/orchestrator.py, which
1386
+ translates a chosen subset into PipelineEvents at the pass
1387
+ boundary -- so a pipeline consumer still only ever sees one event
1388
+ type, which is what P1 and P2 were jointly protecting. What
1389
+ changed is the premise that a pass's internals are not worth
1390
+ seeing.
1391
+
1392
+ A GENERATOR SIBLING, not an observer callback on the existing
1393
+ function. That shape is the project's own, used three times
1394
+ already, and §23 AC1 exists specifically to stop a third bespoke
1395
+ request/response channel appearing beside the ones that already
1396
+ existed -- which §23 has now merged into core.interaction.
1397
+
1398
+ The response is RETURNED rather than read off the terminal event
1399
+ because `thread_id` is attached after the loop finishes. A caller
1400
+ must therefore drain this with something that keeps a generator's
1401
+ return value: `yield from` (what _run_pass does) or
1402
+ `tests/conftest.drain`, never run_to_completion().
1403
+
1404
+ §27: the fresh thread per pass is a LOCKED INVARIANT, not the
1405
+ defect §27's second bug looked like -- passes share distilled JSON
1406
+ and never raw history, which is what stops a later pass reading how
1407
+ an earlier one argued. What was missing is the thread saying so, so
1408
+ that ~15 threads per run stop being offered as conversations.
1409
+ """
1410
+ memory = ConversationMemory(kind=THREAD_KIND_RESEARCH_PASS)
1411
+ memory.add_user_message(pass_input)
1412
+ # #68. The catalogs are decided from the SAME bundle the tool
1413
+ # schemas are decided from a few frames down, so a pass cannot
1414
+ # be told to spawn a subagent it will not be given.
1415
+ _callable_only, _granted = advertisement_facts(authorization)
1416
+ system_prompt = system_prompts.pass_prompt(
1417
+ pass_id, callable_only=_callable_only, granted=_granted)
1418
+ response = None
1419
+ for event in RunAgentLoop._run(
1420
+ memory, system_prompt, provider_name, model, context,
1421
+ max_steps, _resolve_spend_cap(max_total_tokens),
1422
+ temperature=temperature, effort=effort,
1423
+ # §21 M6. A pass is headless and unattended and already returns
1424
+ # a distillation, so routine compaction here would spend on a
1425
+ # judgment call nobody is watching. The backstop still fires
1426
+ # near the model's real context window, because the
1427
+ # alternative there is a hard provider error mid-pipeline.
1428
+ compaction_mode="backstop",
1429
+ activity=activity,
1430
+ **_authorization_kwargs(authorization),
1431
+ ):
1432
+ if event.final_response is not None:
1433
+ response = event.final_response
1434
+ yield event
1435
+ if response is None:
1436
+ # Same contract as run_to_completion's RuntimeError, and for
1437
+ # the same reason: a loop that ended without a final response
1438
+ # has a bug, and returning None would push the failure into
1439
+ # whichever pass dereferences .text first.
1440
+ raise RuntimeError(
1441
+ "Generator completed without yielding a final_response"
1442
+ )
1443
+ response.thread_id = memory.thread_id
1444
+ return response
1445
+
1446
+ @staticmethod
1447
+ def continue_conversation(
1448
+ thread_id: UUID,
1449
+ message: str,
1450
+ system_prompt: str,
1451
+ model: str,
1452
+ provider_name: str = DEFAULT_PROVIDER,
1453
+ max_steps: int = config.MAX_ITERATIONS,
1454
+ max_total_tokens: Optional[int] = _SPEND_UNSET,
1455
+ temperature: Optional[float] = None,
1456
+ effort: Optional[str] = None,
1457
+ context: Optional[ToolContext] = None,
1458
+ authorization=None,
1459
+ activity=None,
1460
+ ) -> ModelResponse:
1461
+ """
1462
+ Sends a follow-up message into an EXISTING conversation thread and
1463
+ runs the loop to completion on it. The resumed thread's full history
1464
+ (every prior user/assistant/tool turn, persisted by _run()'s
1465
+ always-persist behavior) is loaded by ConversationMemory and
1466
+ visible to the next call_model invocation.
1467
+
1468
+ ROADMAP §3's JSON-retry path uses this to retry a pass that
1469
+ returned malformed JSON: the corrective follow-up is sent into
1470
+ the same thread the failed pass used, so the model sees its own
1471
+ prior output in context and can correct it.
1472
+
1473
+ authorization (§25) is NOT optional in practice for that path. A
1474
+ retry is the same pass continuing, so it must run with the same
1475
+ authorization; omitting it would silently strip the pass's gated
1476
+ tools on precisely the attempt where the model is already
1477
+ struggling, and the only symptom would be a second malformed
1478
+ answer. The budget carries across, so retries spend from the same
1479
+ ceiling rather than resetting it.
1480
+ """
1481
+ memory = ConversationMemory(thread_id=thread_id)
1482
+ memory.add_user_message(message)
1483
+ response = run_to_completion(RunAgentLoop._run(
1484
+ memory, system_prompt, provider_name, model, context,
1485
+ max_steps, _resolve_spend_cap(max_total_tokens),
1486
+ temperature=temperature, effort=effort, activity=activity,
1487
+ **_authorization_kwargs(authorization),
1488
+ ))
1489
+ response.thread_id = thread_id
1490
+ return response