@yahyahammad/venastine 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/.env.example +14 -0
  2. package/LICENSE +201 -0
  3. package/NOTICE +14 -0
  4. package/PRIVACY.md +66 -0
  5. package/README.md +878 -0
  6. package/SECURITY.md +88 -0
  7. package/THIRD_PARTY_NOTICES.md +100 -0
  8. package/agents/builtin/compactor.md +75 -0
  9. package/agents/builtin/explore.md +77 -0
  10. package/agents/builtin/grill-me.md +27 -0
  11. package/agents/builtin/initializer.md +94 -0
  12. package/agents/builtin/pipeline-reviewer.md +100 -0
  13. package/agents/builtin/plan.md +88 -0
  14. package/agents/builtin/review.md +74 -0
  15. package/agents/manager.py +195 -0
  16. package/agents/subagent_tool.py +215 -0
  17. package/agents/tui_commands.py +136 -0
  18. package/bin/venastine.mjs +382 -0
  19. package/config.py +1198 -0
  20. package/core/__init__.py +0 -0
  21. package/core/approval.py +115 -0
  22. package/core/client.py +1548 -0
  23. package/core/compaction.py +959 -0
  24. package/core/config_loader.py +1563 -0
  25. package/core/events.py +61 -0
  26. package/core/interaction.py +328 -0
  27. package/core/loop.py +1353 -0
  28. package/core/memory.py +370 -0
  29. package/core/model_windows.py +175 -0
  30. package/core/pipeline_models.py +176 -0
  31. package/core/reasoning/__init__.py +0 -0
  32. package/core/reasoning/authorization.py +130 -0
  33. package/core/reasoning/base.py +219 -0
  34. package/core/reasoning/confidence_scoring.py +468 -0
  35. package/core/reasoning/events.py +149 -0
  36. package/core/reasoning/json_retry.py +219 -0
  37. package/core/reasoning/orchestrator.py +1382 -0
  38. package/core/reasoning/output_writer.py +286 -0
  39. package/core/reasoning/payload_validation.py +398 -0
  40. package/core/reasoning/pipeline_storage.py +361 -0
  41. package/core/reasoning/review.py +615 -0
  42. package/core/reasoning/scholar.py +434 -0
  43. package/core/reasoning/source_corpus.py +327 -0
  44. package/core/reasoning/source_scoring.py +855 -0
  45. package/core/replay.py +167 -0
  46. package/core/session.py +153 -0
  47. package/core/workspace_trust.py +207 -0
  48. package/credentials.py +180 -0
  49. package/database.py +252 -0
  50. package/env_secrets.py +77 -0
  51. package/json_store.py +123 -0
  52. package/logging_setup.py +185 -0
  53. package/main.py +1959 -0
  54. package/mcp_client/__init__.py +15 -0
  55. package/mcp_client/client.py +499 -0
  56. package/mcp_client/config.py +379 -0
  57. package/mcp_client/registration.py +182 -0
  58. package/memories/manager.py +132 -0
  59. package/memories/tui_commands.py +232 -0
  60. package/package.json +73 -0
  61. package/project_init/config_files.py +278 -0
  62. package/project_init/doc_sets.py +330 -0
  63. package/project_init/generator.py +564 -0
  64. package/project_init/manifest.py +406 -0
  65. package/project_init/tui_commands.py +144 -0
  66. package/prompts/__init__.py +0 -0
  67. package/prompts/annotate.md +14 -0
  68. package/prompts/assumption_audit.md +27 -0
  69. package/prompts/claim_extraction.md +34 -0
  70. package/prompts/completeness.md +18 -0
  71. package/prompts/confidence_tiers.md +23 -0
  72. package/prompts/critic_pass.md +24 -0
  73. package/prompts/final_synthesis.md +13 -0
  74. package/prompts/initial_generation.md +14 -0
  75. package/prompts/preliminary_plan.md +23 -0
  76. package/prompts/revalidate.md +25 -0
  77. package/prompts/revise.md +15 -0
  78. package/prompts/source_grounding.md +66 -0
  79. package/prompts/system_prompts.py +262 -0
  80. package/prompts/universal_system_prompt +7 -0
  81. package/prompts/untrusted_content +1 -0
  82. package/providers.json.example +92 -0
  83. package/pyproject.toml +100 -0
  84. package/requirements.txt +72 -0
  85. package/safety/__init__.py +0 -0
  86. package/safety/policy_enforcement.py +682 -0
  87. package/security/__init__.py +0 -0
  88. package/security/capability.py +177 -0
  89. package/security/permissions.py +190 -0
  90. package/security/posture.py +302 -0
  91. package/security/protected_paths.py +309 -0
  92. package/security/sandbox.py +985 -0
  93. package/skills/builtin/crypto/cryptography-verification.md +66 -0
  94. package/skills/builtin/math/formal-specification.md +67 -0
  95. package/skills/builtin/math/numerical-methods.md +66 -0
  96. package/skills/builtin/math/proof-writing.md +61 -0
  97. package/skills/builtin/research/experiment-design.md +62 -0
  98. package/skills/builtin/research/literature-review.md +65 -0
  99. package/skills/builtin/research/reproducibility.md +55 -0
  100. package/skills/builtin/research/source-evaluation.md +70 -0
  101. package/skills/builtin/research/statistical-inference.md +63 -0
  102. package/skills/builtin/research/technical-writing.md +59 -0
  103. package/skills/builtin/security/cybersecurity-research.md +63 -0
  104. package/skills/builtin/software/code-review.md +61 -0
  105. package/skills/builtin/software/debugging.md +61 -0
  106. package/skills/builtin/software/test-design.md +67 -0
  107. package/skills/manager.py +138 -0
  108. package/skills/tui_commands.py +111 -0
  109. package/storage.py +1005 -0
  110. package/tools/base.py +305 -0
  111. package/tools/builtin/__init__.py +0 -0
  112. package/tools/builtin/_math_common.py +377 -0
  113. package/tools/builtin/_net_common.py +61 -0
  114. package/tools/builtin/arxiv.py +223 -0
  115. package/tools/builtin/ask_user.py +213 -0
  116. package/tools/builtin/discrete_math.py +134 -0
  117. package/tools/builtin/fetch_url.py +153 -0
  118. package/tools/builtin/file_ops.py +412 -0
  119. package/tools/builtin/geometry.py +148 -0
  120. package/tools/builtin/get_time.py +20 -0
  121. package/tools/builtin/linear_algebra.py +155 -0
  122. package/tools/builtin/load_skill.py +66 -0
  123. package/tools/builtin/logic.py +96 -0
  124. package/tools/builtin/pin.py +193 -0
  125. package/tools/builtin/probability_stats.py +228 -0
  126. package/tools/builtin/project_docs.py +405 -0
  127. package/tools/builtin/remember.py +158 -0
  128. package/tools/builtin/shell.py +461 -0
  129. package/tools/builtin/symbolic_math.py +119 -0
  130. package/tools/builtin/todo.py +147 -0
  131. package/tools/builtin/web_search.py +151 -0
  132. package/tools/context.py +136 -0
  133. package/tools/isolation.py +314 -0
  134. package/tools/registry.py +897 -0
  135. package/tui/app.py +3423 -0
  136. package/tui/app.tcss +458 -0
  137. package/tui/commands.py +61 -0
  138. package/tui/diffs.py +265 -0
  139. package/tui/preferences.py +182 -0
  140. package/tui/ravens.py +146 -0
  141. package/tui/screens.py +943 -0
  142. package/tui/themes.py +366 -0
  143. package/tui/widgets.py +1239 -0
@@ -0,0 +1,959 @@
1
+ """
2
+ core/compaction.py
3
+
4
+ ROADMAP_v2 §21. Decides WHEN a thread should be compacted, WHAT may be
5
+ folded, and runs the compactor agent to do it.
6
+
7
+ The archive is never touched. Compaction adds a CompactionCheckpoint row
8
+ and nothing else; storage.archive_history() keeps returning every message
9
+ ever written, however many times this has run (§21 AC1). That is the property
10
+ that makes an early, frequent trigger safe -- nothing is destroyed, only
11
+ hidden from the next call.
12
+
13
+ WHAT THIS MODULE OWNS
14
+ should_compact() the trigger, in both its modes
15
+ compactable_span() what may be folded, and the three floors that bound it
16
+ compact() run the compactor, record the checkpoint, rebuild the view
17
+
18
+ WHAT IT DOES NOT OWN
19
+ Where the trigger is evaluated -- core/loop.py calls in, at a turn
20
+ boundary and between steps. Assembling the derived view -- core/memory.py
21
+ owns that, and compact() asks it to rebuild rather than splicing a
22
+ summary in from here. Resolving settings -- core/config_loader.py's
23
+ effective_compaction(), because it owns the merge and this module reaches
24
+ agents.manager, which imports it.
25
+
26
+ MEASUREMENT IS BY CHARACTERS, AND SAYS SO (M10). §21 asks for the summary's
27
+ tokens to be counted; no tokenizer is available offline, and
28
+ usage["input_tokens"] measures a whole prompt rather than a segment of one.
29
+ Chars/4 applied to both sides of a ratio is self-consistent, which is all a
30
+ ratio check needs -- it is never compared against a provider's count.
31
+ """
32
+
33
+ import logging
34
+ import re
35
+ from typing import Optional
36
+
37
+ import config
38
+ from core import config_loader, model_windows, session
39
+ from storage import THREAD_KIND_SUBAGENT
40
+
41
+ logger = logging.getLogger(__name__)
42
+
43
+ # M10. Rough and deliberately uncalibrated: it appears on both sides of
44
+ # every comparison this module makes, so its accuracy cancels out. Never
45
+ # compare a value derived from it against a provider's token count.
46
+ CHARS_PER_TOKEN = 4
47
+
48
+ # The re-entrancy guard (M3). The compactor is an agent, so compacting runs
49
+ # the loop, which evaluates the trigger, which could compact. A thread-local
50
+ # would be more precise, but the loop is synchronous and a compaction runs
51
+ # to completion before its caller resumes -- and a module flag has the
52
+ # property that matters more: it is impossible to be half-set.
53
+ _compacting = False
54
+
55
+
56
+ def _chars(messages) -> int:
57
+ """Size of a neutral-shape message list, in characters."""
58
+ total = 0
59
+ for message in messages:
60
+ total += len(str(message.get("content", "")))
61
+ total += len(str(message.get("text", "")))
62
+ for call in message.get("tool_calls", ()) or ():
63
+ total += len(str(call))
64
+ return total
65
+
66
+
67
+ def estimated_tokens(messages) -> int:
68
+ """A character-based estimate of what a message list will cost to send.
69
+
70
+ Only used when nothing has been MEASURED yet -- a thread written before
71
+ §21 existed, resumed for the first time. Every other evaluation reads
72
+ memory.last_input_tokens, which is the provider's own count of exactly
73
+ what it was sent and needs no estimate at all.
74
+ """
75
+ return _chars(messages) // CHARS_PER_TOKEN
76
+
77
+
78
+ # Models already warned about, so the fallback notice is said once per
79
+ # model rather than once per evaluation. Deduplicated on the MODEL, not
80
+ # with a single boolean, for the same reason core/loop.py's
81
+ # _headless_notices_shown keys on the hidden SET: a run that switches
82
+ # model (or a pipeline whose critic model differs, §11) must still hear
83
+ # about the second one.
84
+ _context_window_warned: set = set()
85
+
86
+
87
+ # A trailing release date, as Anthropic and Google spell it.
88
+ _DATE_SUFFIX = re.compile(r"-\d{8}$")
89
+
90
+ # Region prefixes an inference gateway prepends, as an ALLOWLIST rather
91
+ # than a dot-split. Two entries in the table -- `gpt-5.1` and
92
+ # `qwen3.8-max-preview` -- contain dots inside the model name itself, so a
93
+ # general "strip up to the first dot" rule would corrupt exactly the names
94
+ # it was meant to preserve. Matching a known set cannot.
95
+ _REGION_PREFIXES = ("us.", "eu.", "apac.", "us-gov.")
96
+
97
+
98
+ def _normalized(model: str) -> str:
99
+ """A model id reduced to the form MODEL_CONTEXT_WINDOWS is keyed in.
100
+
101
+ LOOKUP ONLY. This never touches what is sent on the wire -- the model
102
+ string the provider receives is always the one the caller supplied.
103
+
104
+ Exact matching alone is why `claude-sonnet-5-20260724` used to resolve
105
+ to the 200k default while `claude-sonnet-5` sat in the table at a
106
+ million: a 5x undershoot that is silent after the first warning, and
107
+ which reaches summarize_thread()'s truncation gate, not just the
108
+ backstop. The table itself demonstrated the problem -- it keyed one
109
+ Claude entry with a date suffix and five without.
110
+
111
+ Deliberately narrow, and the rules only ever REMOVE a decoration that
112
+ a gateway added:
113
+
114
+ anthropic/claude-opus-5 -> claude-opus-5 (OpenRouter style)
115
+ us.anthropic.claude-opus-5 -> claude-opus-5 (Bedrock style)
116
+ claude-sonnet-5-20260724 -> claude-sonnet-5
117
+ gpt-5.1 -> gpt-5.1 (untouched)
118
+ qwen3.8-max-preview -> qwen3.8-max-preview
119
+
120
+ An id that fits no rule is returned unchanged, so an unrecognised name
121
+ still MISSES and still warns. That direction is deliberate: a loud
122
+ miss costs an early compaction, a wrong prefix match costs a silently
123
+ wrong window, and this table feeds a lossy truncation.
124
+ """
125
+ name = model.strip()
126
+ if "/" in name:
127
+ name = name.rsplit("/", 1)[-1]
128
+ for prefix in _REGION_PREFIXES:
129
+ if name.startswith(prefix):
130
+ name = name[len(prefix):]
131
+ break
132
+ # Bedrock spells the vendor as a dotted segment after the region.
133
+ if name.startswith("anthropic."):
134
+ name = name[len("anthropic."):]
135
+ return _DATE_SUFFIX.sub("", name)
136
+
137
+
138
+ def context_limit(model: str, provider_name: Optional[str] = None) -> int:
139
+ """The model's context window, for the pipeline backstop (M6).
140
+
141
+ FOUR SOURCES, most local first:
142
+
143
+ 0. A REMEMBERED override for this exact (provider, model) --
144
+ core/model_windows.py, written by /window and kept across
145
+ launches (batch 44). It wins because it is the one source a
146
+ human set on purpose, and because the query it overrides can be
147
+ confidently wrong: a gateway may report its own default rather
148
+ than the deployment's, and a self-hosted endpoint's window is
149
+ a fact only its operator has. Everywhere else in this project a
150
+ measured value beats a configured one; here the direction is
151
+ deliberately inverted.
152
+ 1. What the provider SAID, via core.client.known_context_window --
153
+ a cache-only read, primed by core/loop.py before every send. §21
154
+ rejected querying on the belief that no provider in the roster
155
+ could answer; re-measured, Anthropic reports max_input_tokens on
156
+ the very ModelInfo the effort query already fetches, Google
157
+ reports input_token_limit, and Groq/Mistral/Together/OpenRouter
158
+ report a window under one of four field names. A queried answer
159
+ also beats the table on a point the table cannot reach: it is
160
+ what THIS account actually has, not what the model can have.
161
+ 2. config.MODEL_CONTEXT_WINDOWS, normalized (see _normalized), which
162
+ still answers for OpenAI, DeepSeek and Perplexity -- they report
163
+ nothing, which is why §21's conclusion outlived its premise.
164
+ 3. DEFAULT_CONTEXT_WINDOW, warned.
165
+
166
+ Warned on fallback, per §21: a wrong-by-default window means the
167
+ backstop fires at the wrong time in whichever direction the guess is
168
+ wrong, and silent guessing turns a tuning problem into a mystery. Much
169
+ less load-bearing than §21 assumed, though -- M1 moved the ordinary
170
+ trigger off this table entirely, so a missing entry costs a mistimed
171
+ backstop and a mistimed summarizer input budget rather than mistimed
172
+ compaction.
173
+
174
+ ONCE PER MODEL. This is called from thresholds(), which
175
+ should_compact() calls on EVERY evaluation -- and _maybe_compact runs
176
+ at the top of every step of every turn, so in a research run that is
177
+ twice a step across ten passes. Unbounded repetition of a line the
178
+ user can do nothing more about after reading it once is how a warning
179
+ that matters gets trained past, and on the TUI it was also painting
180
+ over the screen.
181
+
182
+ `provider_name` is optional so that every existing caller keeps
183
+ working; without it only sources 2 and 3 are available, which is
184
+ exactly the behaviour that predated the query.
185
+
186
+ THIS IS NOW ON EVERY THREAD'S PATH, not just a research pass's. Batch
187
+ 44 made the working-set trigger a fraction of what this returns, so a
188
+ wrong answer here no longer costs a mistimed backstop -- it decides
189
+ when every chat compacts. That is why source 0 exists at the user tier
190
+ and why the fallback below still warns.
191
+ """
192
+ override = model_windows.window_for(provider_name, model)
193
+ if override:
194
+ return override
195
+
196
+ if provider_name:
197
+ # Imported HERE, not at module scope. core/client.py defers its SDK
198
+ # imports to keep `--help` fast (see its header), and this module is
199
+ # reached from agents.manager, so a top-level import would both undo
200
+ # that and add an import cycle. Guarded because a failure to read a
201
+ # cache must never be the reason a thread cannot compact.
202
+ try:
203
+ from core import client as _client
204
+
205
+ queried = _client.known_context_window(provider_name, model)
206
+ except Exception: # pragma: no cover - defensive
207
+ queried = None
208
+ if queried:
209
+ return queried
210
+
211
+ window = config.MODEL_CONTEXT_WINDOWS.get(_normalized(model))
212
+ if window is None:
213
+ if model not in _context_window_warned:
214
+ _context_window_warned.add(model)
215
+ logger.warning(
216
+ "No context window known for model %r%s; assuming %s. Set it "
217
+ "with /window (remembered for this model) or add it to "
218
+ "config.MODEL_CONTEXT_WINDOWS -- the compaction trigger, the "
219
+ "research backstop and the summarizer's input budget all "
220
+ "fire against this number.", model,
221
+ f" on {provider_name}" if provider_name else "",
222
+ config.DEFAULT_CONTEXT_WINDOW)
223
+ return config.DEFAULT_CONTEXT_WINDOW
224
+ return window
225
+
226
+
227
+ def derived_trigger(model: str, provider_name: Optional[str] = None) -> int:
228
+ """The working-set trigger for this model: a share of its window.
229
+
230
+ config.COMPACTION_TRIGGER_FRACTION x context_limit(), which is 850k on
231
+ a 1M model and 170k on a 200k one. Batch 44's reversal of M1; the
232
+ argument is on the constant.
233
+
234
+ A FRACTION AND NOT `window - margin`, which was the shape considered
235
+ first and is what the research backstop below actually uses. A fixed
236
+ margin cannot be right at both ends of the roster: 40k below a 1M
237
+ window is a rounding error, and 40k below a 64k window leaves 24k,
238
+ which is not a working thread. A share scales by construction.
239
+
240
+ Floored at 1 rather than validated: context_limit() can only return a
241
+ positive number, and effective_compaction() raises on a trigger below
242
+ 1, so this exists to keep an absurd window from turning into a
243
+ confusing error two layers away from the value that caused it.
244
+ """
245
+ return max(1, int(config.COMPACTION_TRIGGER_FRACTION
246
+ * context_limit(model, provider_name)))
247
+
248
+
249
+ def _trigger_is_configured() -> bool:
250
+ """Whether settings.json names compaction.trigger_tokens.
251
+
252
+ The derived trigger is a DEFAULT, so it must not outrank a number a
253
+ human wrote in a config file -- and effective_compaction() cannot tell
254
+ a configured value from the config.py fallback, because it merges them
255
+ into one dict by design (TECHNICAL_DEBT item 12: the merge records no
256
+ provenance). Asking the settings directly is the narrow question this
257
+ one decision needs, rather than a provenance mechanism the rest of
258
+ that dict has done without.
259
+ """
260
+ return "trigger_tokens" in (config_loader.get_settings().get(
261
+ "compaction") or {})
262
+
263
+
264
+ def thresholds(model: str, mode: str = "working_set",
265
+ overrides: Optional[dict] = None,
266
+ provider_name: Optional[str] = None) -> tuple:
267
+ """(warn_at, compact_at) in tokens, for this mode.
268
+
269
+ TWO MODES, because a research pass and a chat turn want different
270
+ answers (M6). "working_set" is the ordinary one: compaction keeps the
271
+ thread inside a share of the model's window, so turns stay affordable
272
+ and keep their tool steps. "backstop" is what a pipeline pass gets --
273
+ it fires only just under the window, because a pass is headless and
274
+ unattended and each one already returns a distillation, so routine
275
+ compaction there would spend on a judgment call nobody is watching.
276
+ Since batch 44 both are derived from context_limit(); what still
277
+ separates them is how much room each leaves.
278
+
279
+ FOUR SOURCES FOR THE WORKING-SET TRIGGER, nearest first: an explicit
280
+ `overrides` (a `/compact --strength N`), a session `/trigger` bound to
281
+ this pair, settings.json's compaction.trigger_tokens, and the derived
282
+ share. The first three are absolute numbers a human named and all
283
+ outrank the derivation, which is what makes batch 44 a change of
284
+ DEFAULT rather than a removal of the control.
285
+ """
286
+ # Resolved HERE because this is the only place the working-set trigger
287
+ # is read and the only place that knows the model. effective_compaction
288
+ # stays model-agnostic, and D27's four-tier chain is unchanged -- both
289
+ # the session value and the derived default arrive through the existing
290
+ # per-invocation slot rather than adding a fifth tier. An explicit
291
+ # `overrides` still wins, since nearest is meant to win and the caller
292
+ # is nearer.
293
+ trigger = session.trigger_for(provider_name, model)
294
+ if trigger is None and mode != "backstop" and not _trigger_is_configured():
295
+ trigger = derived_trigger(model, provider_name)
296
+ if trigger is not None:
297
+ overrides = {"trigger_tokens": trigger, **(overrides or {})}
298
+ settings = config_loader.effective_compaction(overrides)
299
+ if mode == "backstop":
300
+ compact_at = max(
301
+ 1, context_limit(model, provider_name)
302
+ - config.COMPACTION_PIPELINE_BACKSTOP_TOKENS)
303
+ return compact_at, compact_at
304
+ compact_at = settings["trigger_tokens"]
305
+ return compact_at - settings["warning_margin_tokens"], compact_at
306
+
307
+
308
+ def should_compact(used: int, model: str, mode: str = "working_set",
309
+ overrides: Optional[dict] = None,
310
+ provider_name: Optional[str] = None) -> str:
311
+ """"" | "warn" | "compact", from a measured context size.
312
+
313
+ `used` is ModelResponse.usage["input_tokens"] from the most recent
314
+ call -- the provider's own measurement of everything it was sent, which
315
+ is exact but retrospective. That is why the check runs AFTER a response
316
+ and acts before the next call.
317
+
318
+ D21 is what makes this work rather than pedantry: a streaming path that
319
+ silently reported zero usage would leave `used` at 0 forever, so
320
+ compaction would never fire and the harness would run into a hard
321
+ context-limit error from the provider instead. The budget stop
322
+ condition and this trigger fail together, from the same cause.
323
+ """
324
+ if _compacting:
325
+ return ""
326
+ warn_at, compact_at = thresholds(model, mode, overrides, provider_name)
327
+ if used >= compact_at:
328
+ return "compact"
329
+ if used >= warn_at:
330
+ return "warn"
331
+ return ""
332
+
333
+
334
+ def compactable_span(memory, current_turn_start: Optional[int] = None,
335
+ overrides: Optional[dict] = None):
336
+ """The id of the last message that may be folded, or None if nothing
337
+ may be.
338
+
339
+ THREE FLOORS, AND THE EARLIEST WINS (M5):
340
+
341
+ * the current turn -- structural, never foldable at all. A summary
342
+ that swallowed the messages the model is reasoning about right now
343
+ would rewrite the question mid-answer.
344
+ * the last N COMPLETED turns, verbatim. A single tool-heavy turn can
345
+ consume the whole keep-token budget by itself, leaving the
346
+ immediately preceding exchange summarized, and a follow-up like
347
+ "no, the other one" then has no referent.
348
+ * keep_recent_tokens, the token floor §21 specifies.
349
+
350
+ THE BOUNDARY IS ALWAYS A TURN START (M4). Every floor resolves to one,
351
+ so a fold can never separate an assistant message carrying tool_calls
352
+ from the tool results that answer them. That is not a quality concern:
353
+ a tool_result with no preceding tool_use is an HTTP 400 on Anthropic --
354
+ a hard wire error on the next call, on a thread that was working a
355
+ moment ago.
356
+
357
+ Returns None when the floors leave nothing, which is a real state and
358
+ not a failure: a thread of three enormous turns is over the trigger and
359
+ entirely protected. The caller reports it rather than silently doing
360
+ nothing.
361
+
362
+ EVERY FLOOR IS MEASURED IN ARCHIVE SPACE (M11), from one load of the
363
+ thread's rows. The keep-tokens floor used to slice `memory.messages`
364
+ and map positions back onto the archive's turn list; M8 puts a
365
+ `role: "user"` summary at the head of a compacted view, so the two
366
+ lists disagree by one turn and `current_turn_start` arrived in the
367
+ wrong space entirely. That was §21a's shipped defect -- the watermark
368
+ went backwards and the second compaction of a thread un-compacted it.
369
+ Reading everything from `rows` makes the mismatch unrepresentable
370
+ rather than merely fixed.
371
+ """
372
+ from storage import _ordered_rows
373
+
374
+ settings = config_loader.effective_compaction(overrides)
375
+ rows = _ordered_rows(memory.thread_id)
376
+ starts = [i for i, row in enumerate(rows) if row["role"] == "user"]
377
+ if not starts:
378
+ return None
379
+
380
+ # Every candidate boundary is "fold everything strictly before turn i".
381
+ # The floors each veto some suffix of the turn list; the surviving
382
+ # boundary is the earliest turn any of them still allows.
383
+ cutoff = len(starts) - settings["keep_recent_turns"]
384
+ if current_turn_start is not None:
385
+ cutoff = min(cutoff, current_turn_start)
386
+
387
+ # The token floor: keep whole turns, walking back from the end, until
388
+ # keep_recent_tokens is covered. Sized from the persisted content,
389
+ # which is the same character proxy M10 uses everywhere else.
390
+ bounds = starts + [len(rows)]
391
+ kept = 0
392
+ token_cutoff = len(starts)
393
+ for index in range(len(starts) - 1, -1, -1):
394
+ kept += sum(len(rows[i]["content"])
395
+ for i in range(bounds[index], bounds[index + 1]))
396
+ token_cutoff = index
397
+ if kept // CHARS_PER_TOKEN >= settings["keep_recent_tokens"]:
398
+ break
399
+ cutoff = min(cutoff, token_cutoff)
400
+
401
+ if cutoff < 1:
402
+ return None
403
+ # Fold everything strictly before turn `cutoff`, so the watermark is
404
+ # the row just before that turn's first row. cutoff >= 1 guarantees
405
+ # there is one.
406
+ return rows[starts[cutoff] - 1]["id"]
407
+
408
+
409
+ def _target_chars(original_chars: int, strength: int) -> int:
410
+ return int(original_chars * config.COMPACTION_TARGET_RATIOS[strength])
411
+
412
+
413
+ def _instruction(segment_text: str, target: int, original: int) -> str:
414
+ return (
415
+ f"Summarize the conversation excerpt below.\n\n"
416
+ f"The original is {original} characters. Your summary must be at "
417
+ f"most {target} characters -- shorter is fine, longer will be "
418
+ f"rejected.\n\n"
419
+ f"--- BEGIN EXCERPT ---\n{segment_text}\n--- END EXCERPT ---"
420
+ )
421
+
422
+
423
+ def _as_text(messages) -> str:
424
+ lines = []
425
+ for message in messages:
426
+ role = message.get("role", "?")
427
+ if role == "assistant":
428
+ body = message.get("text", "")
429
+ calls = message.get("tool_calls") or []
430
+ if calls:
431
+ body = f"{body}\n[called: {', '.join(c.get('name', '?') for c in calls)}]"
432
+ else:
433
+ body = str(message.get("content", ""))
434
+ lines.append(f"{role}: {body}")
435
+ return "\n\n".join(lines)
436
+
437
+
438
+ def pin_measurements(thread_id, last_n: int) -> dict:
439
+ """What #89's cap is measured against, in M10's character proxy.
440
+
441
+ Returns three numbers:
442
+ requested_tokens what the last `last_n` turns' ARCHIVE rows cost --
443
+ the span one more `pin(last_n)` would protect
444
+ pinned_tokens everything ALREADY pinned on this thread
445
+ max_tokens PIN_MAX_TRIGGER_FRACTION x the resolved trigger
446
+
447
+ Reads the ARCHIVE (the same rows compactable_span walks), so a pinned
448
+ prefix is counted at full weight exactly as the derived view will
449
+ carry it. Never compared against a provider count (M10).
450
+ """
451
+ from storage import _ordered_rows
452
+ from core import config_loader
453
+
454
+ settings = config_loader.effective_compaction()
455
+ # THE MINIMUM, and the min() is load-bearing rather than defensive.
456
+ # This function has no (provider, model) to match a session override
457
+ # against, so it reads the binding-free one -- and takes whichever
458
+ # trigger is SMALLER, which is safe under every binding. A raised
459
+ # session trigger leaves the cap stricter than it needs to be; a
460
+ # lowered one pulls the cap down with it. Reading the configured
461
+ # trigger alone would let `/trigger 12k` leave this cap at 20k, so a
462
+ # pin could protect more than the whole trigger and floor the thread
463
+ # permanently above it -- which is the exact defect #89 exists to
464
+ # prevent, reintroduced through a number typed in a different command.
465
+ trigger = min(settings["trigger_tokens"],
466
+ session.trigger_any() or settings["trigger_tokens"])
467
+ max_tokens = int(config.PIN_MAX_TRIGGER_FRACTION * trigger)
468
+
469
+ rows = _ordered_rows(thread_id)
470
+ starts = [i for i, row in enumerate(rows) if row["role"] == "user"]
471
+ bounds = starts + [len(rows)]
472
+
473
+ def _tokens(lo: int) -> int:
474
+ return sum(len(rows[i]["content"])
475
+ for i in range(lo, len(rows))) // CHARS_PER_TOKEN
476
+
477
+ requested = _tokens(bounds[max(0, len(starts) - last_n)]) if starts else 0
478
+ pinned = sum(len(row["content"]) for row in rows
479
+ if row["pinned"]) // CHARS_PER_TOKEN
480
+ return {
481
+ "requested_tokens": requested,
482
+ "pinned_tokens": pinned,
483
+ "max_tokens": max_tokens,
484
+ }
485
+
486
+
487
+ def _truncate_oldest(text: str, budget: int) -> tuple:
488
+ """`(text, notice)` -- drop the earliest characters over `budget`.
489
+
490
+ Returns the text unchanged and `None` when it already fits. The OLDEST
491
+ material goes, because recent turns are the referent-heavy ones, and
492
+ the cut is stated in the returned notice so both callers can put it in
493
+ the instruction AND in the stored row.
494
+
495
+ ONE COPY, since batch 45. This was eight identical lines in `compact()`
496
+ and in `summarize_thread()`, notice string included, marked as
497
+ deliberate nowhere -- the shape
498
+ `safety/policy_enforcement.param_digest` calls this project's canonical
499
+ bug when it explains why the digest is not written twice. The two were
500
+ equivalent only by accident: the second guarded on a separate
501
+ `original` variable that happened to equal `len(thread_text)` at that
502
+ point, so the condition and the arithmetic under it could have drifted
503
+ apart without either copy looking wrong.
504
+ """
505
+ if len(text) // CHARS_PER_TOKEN <= budget:
506
+ return text, None
507
+ keep_chars = max(0, budget * CHARS_PER_TOKEN)
508
+ dropped = len(text) - keep_chars
509
+ notice = (f"[Source truncated: the earliest {dropped} characters were "
510
+ f"not summarized.]")
511
+ return f"{notice}\n{text[-keep_chars:]}", notice
512
+
513
+
514
+ def _input_budget(model: str, provider_name: Optional[str] = None) -> int:
515
+ """One-call INPUT ceiling for a summarizer prompt, in proxy tokens.
516
+
517
+ context_limit minus the pipeline backstop margin -- generous by
518
+ construction, and honest about its weakness (M10): this compares the
519
+ chars-per-token PROXY against a real window number, which every other
520
+ comparison in this module refuses to do. The comparison is inherently
521
+ approximate; what makes it safe is the failure direction. Undershooting
522
+ the budget costs truncation, which is stated; overshooting it costs an
523
+ oversized send, which is a provider error mid-compaction on exactly
524
+ the thread that can least afford one.
525
+ """
526
+ return max(1_000, context_limit(model, provider_name)
527
+ - config.COMPACTION_PIPELINE_BACKSTOP_TOKENS)
528
+
529
+
530
+ def compact(memory, model: str, provider_name: str,
531
+ current_turn_start: Optional[int] = None,
532
+ overrides: Optional[dict] = None,
533
+ authorization=None) -> dict:
534
+ """Compact `memory`'s thread. Returns an OUTCOME dict -- never None.
535
+
536
+ Batch 16 (#44) changed this contract. It used to return a notice dict
537
+ on success and None on every other exit -- five flavors of "nothing
538
+ happened" that were indistinguishable at the call site and logged by
539
+ three different conventions (WARNING per evaluation, debug-only,
540
+ nothing at all). Now every exit is data:
541
+
542
+ {"status": <one of the seven below>,
543
+ "kind": <notice kind for the loop's per-run dedup, or None>,
544
+ "text": <a human line both shells render>}
545
+
546
+ The seven statuses, which are this function's whole vocabulary:
547
+
548
+ folded a checkpoint was written (kind="compaction")
549
+ blocked the keep floors protect everything
550
+ all-pinned everything newly foldable is pinned (#89's silent
551
+ path, named now)
552
+ missing-agent the compactor definition is absent
553
+ no-progress the fold boundary has not moved past the last
554
+ checkpoint
555
+ reentrant a compaction is already running in this process
556
+ failed the compactor produced nothing usable
557
+
558
+ Three of them are STANDING CONDITIONS (blocked, all-pinned,
559
+ missing-agent) and share one notice kind, so the loop's existing
560
+ once-per-run dedup governs their WARNINGs too instead of each firing
561
+ its own per-evaluation line. They are enumerated HERE, at the function
562
+ that produces them, rather than in a module-level tuple: batch 16 kept
563
+ a `COMPACTION_OUTCOMES` sequence beside this docstring and nothing ever
564
+ read it, which is `CODE_STAGES`' shape exactly (see the note in
565
+ core/reasoning/events.py) -- a list whose only future was drifting away
566
+ from the call sites while looking like a contract.
567
+
568
+ REPORTABILITY LIVES WITH THE CALLER. How often this function runs --
569
+ once at the turn boundary plus once after every step -- is run-scope
570
+ knowledge the loop has and this module lacks, which is why the
571
+ standing-condition WARNINGs moved out of it into _maybe_compact's
572
+ once-per-run guard (#44). What remains here is mechanics: DEBUG for a
573
+ fold boundary that did not move, and _summarize's own retry warnings,
574
+ which are events about one model call rather than states of the run.
575
+
576
+ Runs the compactor as an ORDINARY AGENT through the same
577
+ RunAgentLoop everything else uses. §21 is explicit that summarizing is
578
+ the shape of an agent call -- a system prompt, a task, a judgment-based
579
+ output -- and that routing it through the shared machinery is also what
580
+ gives it real autonomy over WHAT to preserve. Mechanical truncation
581
+ cannot exercise judgment at all.
582
+
583
+ The compactor's own run gets its own fresh thread, no tools
584
+ (`allowed_tools: []` in its definition), and the re-entrancy guard, so
585
+ it can neither call anything nor trigger a compaction of its own.
586
+ """
587
+ global _compacting
588
+
589
+ from agents.manager import manager
590
+ from core.loop import RunAgentLoop, DEFAULT_SYSTEM_PROMPT
591
+ from storage import (
592
+ advances, history_through, latest_checkpoint, save_checkpoint,
593
+ )
594
+ storage_advances = advances
595
+
596
+ if _compacting:
597
+ return {"status": "reentrant", "kind": None,
598
+ "text": "A compaction is already running."}
599
+
600
+ settings = config_loader.effective_compaction(overrides)
601
+ watermark = compactable_span(memory, current_turn_start, overrides)
602
+ if watermark is None:
603
+ logger.debug(
604
+ "Compaction wanted on thread %s but every message is protected "
605
+ "by the keep floors; nothing folded.", memory.thread_id)
606
+ return {
607
+ "status": "blocked",
608
+ "kind": "compaction_blocked",
609
+ "text": ("Context is full but the most recent turns fill it "
610
+ "entirely -- nothing could be compacted."),
611
+ }
612
+
613
+ agent = manager.get(config.COMPACTOR_AGENT)
614
+ if agent is None:
615
+ logger.debug(
616
+ "Compaction wanted but the %r agent is not available.",
617
+ config.COMPACTOR_AGENT)
618
+ return {
619
+ "status": "missing-agent",
620
+ "kind": "compaction_blocked",
621
+ "text": (f"The {config.COMPACTOR_AGENT!r} agent is not "
622
+ f"available, so this conversation cannot be "
623
+ f"condensed."),
624
+ }
625
+
626
+ previous = latest_checkpoint(memory.thread_id)
627
+ # NO PROGRESS, NO CALL. A watermark at or before the existing one means
628
+ # the floors have not moved past what is already summarized -- there is
629
+ # nothing new to fold. Checked HERE rather than only in save_checkpoint
630
+ # because the model call happens in between: without it, rederive would
631
+ # re-summarize a span it has already summarized, and chain would be
632
+ # handed an empty segment and summarize its own previous summary,
633
+ # degrading it a little on every attempt. Both spend a real call to
634
+ # achieve nothing.
635
+ if previous is not None and not storage_advances(
636
+ memory.thread_id, previous["covers_up_to_message_id"], watermark):
637
+ logger.debug(
638
+ "Compaction wanted on thread %s but the fold boundary has not "
639
+ "moved past the last checkpoint; nothing to do.", memory.thread_id)
640
+ return {"status": "no-progress", "kind": None,
641
+ "text": "Nothing new to fold since the last summary."}
642
+
643
+ # M2 as amended by batch 16 (#90). Re-derive from the archive by
644
+ # default WHEN CONFIGURED; chain summarizes the previous summary plus
645
+ # the tail. Whichever is configured, a span that outgrows one call
646
+ # falls back to chain (verbose -- see the WARNING below), and one that
647
+ # outgrows it even chained truncates its oldest material with the
648
+ # truncation stated twice: to the model in the instruction, and in the
649
+ # stored summary every later turn reads.
650
+ chaining = settings["strategy"] == "chain" and previous is not None
651
+
652
+ def _build(chain_mode: bool):
653
+ lower = previous["covers_up_to_message_id"] if chain_mode else None
654
+ seg = history_through(memory.thread_id, watermark,
655
+ after_message_id=lower)
656
+ text = _as_text(seg)
657
+ if chain_mode:
658
+ text = ("Summary of everything before this point:\n"
659
+ f"{previous['summary_text']}\n\n{text}")
660
+ return seg, text
661
+
662
+ segment, segment_text = _build(chaining)
663
+ if not segment and not chaining:
664
+ # Everything newly foldable is pinned (#89). This used to be the
665
+ # one silent no-progress path -- indistinguishable, at every
666
+ # level, from "nothing needed doing" -- while the state it reports
667
+ # is exactly what compaction_blocked's text describes, reached by
668
+ # another door. Named now, and worded so the reader knows the way
669
+ # out.
670
+ return {
671
+ "status": "all-pinned",
672
+ "kind": "compaction_blocked",
673
+ "text": ("Context is full, but everything foldable is pinned. "
674
+ "Unpin something with /unpin, or let more unpinned "
675
+ "turns accumulate."),
676
+ }
677
+
678
+ original = len(segment_text)
679
+ target = _target_chars(original, settings["strength"])
680
+
681
+ # M2's automatic fallback, built at last after three documents
682
+ # promised it for two sections. Forced only when there IS a previous
683
+ # summary to chain onto; VERBOSE because a configured strategy
684
+ # quietly doing the other thing is exactly what "(traced)" in the
685
+ # spec was about.
686
+ budget = _input_budget(model, provider_name)
687
+ forced_chain_note = ""
688
+ if (not chaining and previous is not None
689
+ and len(segment_text) // CHARS_PER_TOKEN > budget):
690
+ logger.warning(
691
+ "Compaction on thread %s: the rederive span is ~%s tokens "
692
+ "against a %s-token one-call budget; forcing chain onto the "
693
+ "previous summary.",
694
+ memory.thread_id, len(segment_text) // CHARS_PER_TOKEN, budget)
695
+ chaining = True
696
+ segment, segment_text = _build(True)
697
+ original = len(segment_text)
698
+ target = _target_chars(original, settings["strength"])
699
+ forced_chain_note = (" Chained onto the previous summary because "
700
+ "the span outgrew one call.")
701
+
702
+ # Still over -- or no previous summary existed to chain onto. Truncate
703
+ # the OLDEST material (recent turns are the referent-heavy ones) and
704
+ # state the cut in both places the summary travels.
705
+ segment_text, truncation_notice = _truncate_oldest(segment_text, budget)
706
+ if truncation_notice:
707
+ original = len(segment_text)
708
+
709
+ _compacting = True
710
+ try:
711
+ summary = _summarize(
712
+ RunAgentLoop, manager, agent, DEFAULT_SYSTEM_PROMPT,
713
+ segment_text, target, original, model, provider_name,
714
+ settings["max_retries"], authorization)
715
+ finally:
716
+ _compacting = False
717
+
718
+ if summary is None:
719
+ # #136, batch 20: this used to be kind=None -- invisible to both
720
+ # shells -- while its own WARNING fired inside _summarize on every
721
+ # evaluation. A compactor that returned nothing IS a failed
722
+ # compaction (AGENTS.md's containment rule says so), so it now
723
+ # rides compaction_failed: visible once per run through the loop's
724
+ # dedup, and the latch in _maybe_compact stops the per-step model
725
+ # calls the silence was hiding.
726
+ return {"status": "failed", "kind": "compaction_failed",
727
+ "text": "The compactor returned nothing usable."}
728
+
729
+ if truncation_notice is not None:
730
+ # Deterministic rather than trusted from the model's compliance:
731
+ # the marker rides on the STORED summary, so the derived view --
732
+ # and everyone reading it later -- sees the cut even if the model
733
+ # never mentioned it.
734
+ summary = f"{summary}\n\n{truncation_notice}"
735
+
736
+ strategy = "chain" if chaining else "rederive"
737
+ save_checkpoint(memory.thread_id, summary, watermark, strategy)
738
+ memory.apply_checkpoint()
739
+ folded = len(segment)
740
+ logger.info("Compacted thread %s: %s messages -> %s chars (%s).",
741
+ memory.thread_id, folded, len(summary), strategy)
742
+ return {
743
+ "status": "folded",
744
+ "kind": "compaction",
745
+ "text": f"{folded} earlier messages compacted into a summary"
746
+ + forced_chain_note,
747
+ }
748
+
749
+
750
+ def summarize_thread(thread_id, model: str, provider_name: str,
751
+ overrides: Optional[dict] = None,
752
+ authorization=None) -> Optional[dict]:
753
+ """Distil a WHOLE thread into a stored summary (ROADMAP_v2 §21c).
754
+
755
+ Returns a notice dict carrying the summary, or None if the thread has
756
+ nothing in it. The shape matches compact()'s notices, so both shells
757
+ render it with code they already have.
758
+
759
+ THIS IS NOT A COMPACTION and must never become one. compact() folds a
760
+ span of the thread it is given, changing what the model sees on the next
761
+ turn; this reads a thread and writes a description of it, leaving the
762
+ thread untouched -- which is what lets `/ref` inject one thread's summary
763
+ into a different thread, and what lets `/summary` show one without
764
+ quietly shortening the conversation the user is having (M20).
765
+
766
+ THE ARCHIVE, NEVER THE DERIVED VIEW. `archive_history` returns every
767
+ message ever written, so a summary of a compacted thread describes the
768
+ conversation rather than describing the summary of it. A view-based
769
+ version would degrade a little on every compaction, exactly the chaining
770
+ loss M2 arranges the fold to avoid.
771
+
772
+ CACHED, AND STALENESS IS DETECTED RATHER THAN ASSUMED. The stored row
773
+ carries the last archive position it accounts for, so `advances()` --
774
+ which already answers "has this thread moved past that point?" for
775
+ compaction -- says whether the summary still holds. An unchanged thread
776
+ therefore costs no model call however many times it is referenced, and a
777
+ grown one is re-summarized rather than silently served stale.
778
+
779
+ A THREAD ALREADY SHORTER THAN THE TARGET IS NOT SENT TO A MODEL. Its
780
+ rendered text IS its own best summary, and asking a model to compress
781
+ three messages into more characters than they occupy spends a call to
782
+ make the answer worse. Same instinct as compact()'s no-progress-no-call
783
+ guard.
784
+
785
+ The target is an ABSOLUTE character budget (`config.SUMMARY_TARGET_CHARS`),
786
+ not `_target_chars`' strength ratio. A ratio is right for a fold, where
787
+ the summary replaces the span it came from and scales with it. It is wrong
788
+ here: §21c's consumer is a prompt tier present on EVERY turn of the
789
+ referencing thread, so a proportional target would let one big thread
790
+ inject tens of kilobytes into every call indefinitely.
791
+ """
792
+ global _compacting
793
+
794
+ from agents.manager import manager
795
+ from core.loop import RunAgentLoop, DEFAULT_SYSTEM_PROMPT
796
+ from storage import (
797
+ advances, archive_history, last_message_id, latest_thread_summary,
798
+ save_thread_summary,
799
+ )
800
+
801
+ if _compacting:
802
+ return None
803
+
804
+ watermark = last_message_id(thread_id)
805
+ if watermark is None:
806
+ return None
807
+
808
+ existing = latest_thread_summary(thread_id)
809
+ if existing is not None and not advances(
810
+ thread_id, existing["covers_up_to_message_id"], watermark):
811
+ return {
812
+ "kind": "thread_summary",
813
+ "text": existing["summary_text"],
814
+ "thread_id": str(thread_id),
815
+ "fresh": False,
816
+ }
817
+
818
+ thread_text = _as_text(archive_history(thread_id))
819
+ original = len(thread_text)
820
+ target = config.SUMMARY_TARGET_CHARS
821
+
822
+ if original <= target:
823
+ save_thread_summary(thread_id, thread_text, watermark)
824
+ return {
825
+ "kind": "thread_summary",
826
+ "text": thread_text,
827
+ "thread_id": str(thread_id),
828
+ "fresh": True,
829
+ "verbatim": True,
830
+ }
831
+
832
+ # #90's gate on the second consumer. A thread summary's input is
833
+ # bounded by NOTHING else -- compact()'s span at least waits for a
834
+ # threshold -- and T4 documents what tool rows weigh. Reserve room for
835
+ # the summary this call is meant to PRODUCE, then truncate the oldest
836
+ # material with the cut stated in the instruction AND in the stored
837
+ # row, so every referencing thread sees it.
838
+ budget = max(1_000, _input_budget(model, provider_name)
839
+ - config.SUMMARY_TARGET_CHARS // CHARS_PER_TOKEN)
840
+ thread_text, truncation_notice = _truncate_oldest(thread_text, budget)
841
+ if truncation_notice:
842
+ original = len(thread_text)
843
+
844
+ agent = manager.get(config.COMPACTOR_AGENT)
845
+ if agent is None:
846
+ logger.warning(
847
+ "A thread summary was wanted but the %r agent is not available; "
848
+ "skipped.", config.COMPACTOR_AGENT)
849
+ return None
850
+
851
+ settings = config_loader.effective_compaction(overrides)
852
+ # The guard for the same reason compact() sets it (M3): the compactor is
853
+ # an agent, so summarizing runs the loop, which evaluates the compaction
854
+ # trigger. Its own thread is one turn long and would not trip it today,
855
+ # which is exactly the kind of "cannot happen yet" that stops being true
856
+ # quietly.
857
+ _compacting = True
858
+ try:
859
+ summary = _summarize(
860
+ RunAgentLoop, manager, agent, DEFAULT_SYSTEM_PROMPT,
861
+ thread_text, target, original, model, provider_name,
862
+ settings["max_retries"], authorization)
863
+ finally:
864
+ _compacting = False
865
+
866
+ if summary is None:
867
+ return None
868
+
869
+ if truncation_notice is not None:
870
+ # Same determinism rule as compact(): the marker rides on the
871
+ # stored row regardless of whether the model mentioned the cut.
872
+ summary = f"{summary}\n\n{truncation_notice}"
873
+
874
+ save_thread_summary(thread_id, summary, watermark)
875
+ logger.info("Summarized thread %s: %s chars -> %s.",
876
+ thread_id, original, len(summary))
877
+ return {
878
+ "kind": "thread_summary",
879
+ "text": summary,
880
+ "thread_id": str(thread_id),
881
+ "fresh": True,
882
+ }
883
+
884
+
885
+ def _summarize(loop_cls, manager, agent, base_prompt, segment_text, target,
886
+ original, model, provider_name, max_retries, authorization):
887
+ """Run the compactor, retrying while the summary overshoots its target.
888
+
889
+ STRUCTURALLY §3's JSON-retry loop with a length constraint in place of
890
+ a parse -- a corrective follow-up into the SAME thread, so the model
891
+ sees its own output and can tighten it, bounded by a retry cap.
892
+ Deliberately not routed through core/reasoning/json_retry.py: that
893
+ module is pipeline-shaped (it takes a run trace and names passes), and
894
+ bending it to serve memory would invert the dependency for three lines
895
+ of shared structure.
896
+
897
+ UNDERSHOOTING IS NOT AN ERROR. A summary tighter than asked for has
898
+ already done the job, and rejecting it would spend a second model call
899
+ to make the context bigger.
900
+
901
+ A summary still too long after the last retry is USED ANYWAY, with a
902
+ warning. It is smaller than what it replaces, which is the whole point;
903
+ discarding it would spend the calls and keep the full history, leaving
904
+ the thread exactly as stuck as before with less budget to fix it.
905
+ """
906
+ context = manager.active_context(agent)
907
+ system_prompt = manager.system_prompt_for(agent, base_prompt, context=context)
908
+ tolerance = int(target * (1 + config.COMPACTION_RATIO_TOLERANCE))
909
+
910
+ response = loop_cls.run_agent_conversation(
911
+ user_goal=_instruction(segment_text, target, original),
912
+ model=agent.model or model,
913
+ provider_name=agent.provider or provider_name,
914
+ max_steps=1,
915
+ context=context,
916
+ system_prompt=system_prompt,
917
+ authorization=authorization,
918
+ # §27, and NOT in that section's spec: the compactor is a fifth
919
+ # thread source nobody had counted. Every automatic compaction of a
920
+ # long chat creates one of these, so on exactly the threads §21a
921
+ # exists to serve, the picker filled up with the machinery that
922
+ # served them. The compactor is an agent, so it is labelled like the
923
+ # other agent-shaped runs.
924
+ thread_kind=THREAD_KIND_SUBAGENT,
925
+ )
926
+ summary = (response.text or "").strip()
927
+
928
+ for _ in range(max_retries):
929
+ if not summary:
930
+ logger.warning("Compactor returned nothing; compaction skipped.")
931
+ return None
932
+ if len(summary) <= tolerance:
933
+ return summary
934
+ response = loop_cls.continue_conversation(
935
+ thread_id=response.thread_id,
936
+ message=(
937
+ f"That summary is {len(summary)} characters; the target is "
938
+ f"{target}. Tighten it -- drop the least important material "
939
+ f"rather than making the remaining facts vaguer. Output only "
940
+ f"the summary."),
941
+ system_prompt=system_prompt,
942
+ model=agent.model or model,
943
+ provider_name=agent.provider or provider_name,
944
+ max_steps=1,
945
+ context=context,
946
+ authorization=authorization,
947
+ )
948
+ summary = (response.text or "").strip()
949
+
950
+ if not summary:
951
+ logger.warning("Compactor returned nothing; compaction skipped.")
952
+ return None
953
+ if len(summary) > tolerance:
954
+ logger.warning(
955
+ "Compactor missed its target after %s retries (%s chars against a "
956
+ "target of %s); using it anyway -- it is still smaller than the "
957
+ "%s characters it replaces.", max_retries, len(summary), target,
958
+ original)
959
+ return summary