@yahyahammad/venastine 0.2.0 → 0.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/.env.example +19 -14
  2. package/CONFIG_ARCHITECTURE.md +2002 -0
  3. package/NOTICE +1 -1
  4. package/README.md +233 -94
  5. package/agents/builtin/build.md +73 -0
  6. package/agents/builtin/general.md +80 -0
  7. package/agents/builtin/plan.md +23 -5
  8. package/agents/builtin/test.md +65 -0
  9. package/agents/builtin/writer.md +67 -0
  10. package/agents/manager.py +261 -195
  11. package/agents/subagent_tool.py +277 -43
  12. package/agents/tui_commands.py +7 -12
  13. package/bin/venastine.mjs +528 -382
  14. package/bin/venastine.py +71 -0
  15. package/cli_commands.py +835 -0
  16. package/config.py +95 -1198
  17. package/config.yaml +1090 -0
  18. package/config_edit.py +1301 -0
  19. package/config_schema.py +966 -0
  20. package/config_update.py +749 -0
  21. package/core/agent_activity.py +158 -20
  22. package/core/ansi.py +452 -0
  23. package/core/approval.py +27 -5
  24. package/core/call_view.py +364 -0
  25. package/core/cancel.py +121 -0
  26. package/core/chatgpt_auth.py +747 -0
  27. package/core/claims_view.py +107 -0
  28. package/core/client.py +1238 -82
  29. package/core/commands.py +454 -0
  30. package/core/compaction.py +830 -66
  31. package/core/config_loader.py +516 -97
  32. package/core/connect.py +686 -0
  33. package/core/events.py +68 -6
  34. package/core/file_mentions.py +280 -0
  35. package/core/interaction.py +87 -4
  36. package/core/line_pattern.py +128 -0
  37. package/core/loop.py +1516 -440
  38. package/core/memory.py +120 -8
  39. package/core/pipeline_models.py +23 -9
  40. package/core/provider_errors.py +234 -0
  41. package/core/reasoning/confidence_scoring.py +70 -54
  42. package/core/reasoning/json_retry.py +19 -1
  43. package/core/reasoning/orchestrator.py +232 -53
  44. package/core/reasoning/output_writer.py +90 -31
  45. package/core/reasoning/payload_validation.py +133 -7
  46. package/core/reasoning/pipeline_storage.py +427 -361
  47. package/core/reasoning/review.py +659 -615
  48. package/core/reasoning/scholar.py +38 -14
  49. package/core/reasoning/source_corpus.py +110 -4
  50. package/core/replay.py +158 -26
  51. package/core/session.py +36 -12
  52. package/core/session_view.py +259 -0
  53. package/core/session_wake.py +259 -0
  54. package/core/shell_sessions.py +1553 -0
  55. package/core/workspace_trust.py +1 -1
  56. package/credentials.py +308 -180
  57. package/database.py +82 -1
  58. package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
  59. package/docs/SECURITY.md +108 -0
  60. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  61. package/env_secrets.py +118 -77
  62. package/json_store.py +236 -16
  63. package/logging_setup.py +121 -11
  64. package/main.py +1671 -199
  65. package/mcp_client/client.py +257 -23
  66. package/mcp_client/config.py +484 -379
  67. package/mcp_client/registration.py +163 -2
  68. package/memories/manager.py +15 -0
  69. package/memories/tui_commands.py +7 -12
  70. package/package.json +234 -4
  71. package/project_init/config_files.py +4 -0
  72. package/project_init/generator.py +201 -37
  73. package/project_init/manifest.py +418 -406
  74. package/project_init/tui_commands.py +7 -4
  75. package/prompts/revalidate.md +1 -1
  76. package/prompts/revise.md +17 -15
  77. package/prompts/system_prompts.py +64 -10
  78. package/providers.json.example +94 -92
  79. package/pyproject.toml +51 -2
  80. package/requirements.txt +252 -29
  81. package/safety/policy_enforcement.py +353 -17
  82. package/security/askpass.py +419 -0
  83. package/security/capability.py +37 -7
  84. package/security/confine.py +484 -0
  85. package/security/hosted.py +182 -0
  86. package/security/permissions.py +54 -7
  87. package/security/posture.py +124 -12
  88. package/security/protected_paths.py +198 -6
  89. package/security/sandbox.py +3262 -139
  90. package/security/secrets.py +855 -0
  91. package/skills/tui_commands.py +3 -4
  92. package/storage.py +368 -35
  93. package/tools/base.py +465 -305
  94. package/tools/builtin/_math_common.py +306 -16
  95. package/tools/builtin/_net_common.py +267 -61
  96. package/tools/builtin/arxiv.py +681 -223
  97. package/tools/builtin/ask_user.py +231 -213
  98. package/tools/builtin/discrete_math.py +35 -4
  99. package/tools/builtin/fetch_url.py +239 -21
  100. package/tools/builtin/file_ops.py +341 -33
  101. package/tools/builtin/geometry.py +184 -148
  102. package/tools/builtin/get_time.py +2 -2
  103. package/tools/builtin/linear_algebra.py +66 -23
  104. package/tools/builtin/load_skill.py +10 -0
  105. package/tools/builtin/logic.py +171 -12
  106. package/tools/builtin/pin.py +46 -12
  107. package/tools/builtin/probability_stats.py +224 -228
  108. package/tools/builtin/project_docs.py +80 -21
  109. package/tools/builtin/remember.py +21 -4
  110. package/tools/builtin/shell.py +586 -67
  111. package/tools/builtin/shell_sessions.py +613 -0
  112. package/tools/builtin/symbolic_math.py +80 -10
  113. package/tools/builtin/todo.py +5 -3
  114. package/tools/builtin/web_search.py +329 -151
  115. package/tools/context.py +23 -0
  116. package/tools/isolation.py +387 -314
  117. package/tools/registry.py +347 -30
  118. package/tui/app.py +4645 -438
  119. package/tui/app.tcss +185 -2
  120. package/tui/commands.py +10 -154
  121. package/tui/diffs.py +286 -265
  122. package/tui/hosted_driver.py +102 -0
  123. package/tui/markdown.py +60 -10
  124. package/tui/meters.py +103 -7
  125. package/tui/screens.py +346 -92
  126. package/tui/themes.py +59 -19
  127. package/tui/widgets.py +1575 -111
  128. package/SECURITY.md +0 -88
  129. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -30,8 +30,10 @@ Chars/4 applied to both sides of a ratio is self-consistent, which is all a
30
30
  ratio check needs -- it is never compared against a provider's count.
31
31
  """
32
32
 
33
+ import json
33
34
  import logging
34
35
  import re
36
+ import threading
35
37
  from typing import Optional
36
38
 
37
39
  import config
@@ -45,12 +47,83 @@ logger = logging.getLogger(__name__)
45
47
  # compare a value derived from it against a provider's token count.
46
48
  CHARS_PER_TOKEN = 4
47
49
 
50
+ # ROADMAP_v3 §81 (IM8). What one image weighs in that proxy, in tokens.
51
+ #
52
+ # A FIXED number, because the alternative is decoding every image to learn
53
+ # its dimensions, which would take an imaging dependency this harness does
54
+ # not carry. 1,600 is the top of Anthropic's own estimate (width x height /
55
+ # 750, on an image it has already scaled to about 1.15 megapixels), so a
56
+ # large image is counted at roughly its real cost and a small one is
57
+ # overcounted. Overcounting folds a little early; undercounting would let a
58
+ # thread of screenshots run past the window. Counting an image by its base64
59
+ # LENGTH would be the third option, and the worst: a 1 MB file is about 1.3
60
+ # million characters, which reads as 330,000 tokens and folds at once.
61
+ IMAGE_TOKEN_ESTIMATE = 1600
62
+ _IMAGE_CHARS = IMAGE_TOKEN_ESTIMATE * CHARS_PER_TOKEN
63
+
64
+
65
+ def _image_count(images) -> int:
66
+ """How many images one row or message carries (IM8): the decoded list on
67
+ a neutral message, the JSON text `MessageLog.images` stores on an archive
68
+ row, or nothing."""
69
+ if not images:
70
+ return 0
71
+ if isinstance(images, str):
72
+ try:
73
+ images = json.loads(images)
74
+ except ValueError:
75
+ return 0
76
+ return len(images) if isinstance(images, list) else 0
77
+
78
+
79
+ def _row_chars(row) -> int:
80
+ """One archive row's size in the proxy: its persisted content, plus IM8's
81
+ fixed weight per image. EVERY row-size sum below goes through here, so a
82
+ floor and the pin cap cannot disagree about what an image costs."""
83
+ return len(row["content"]) + _image_count(row.get("images")) * _IMAGE_CHARS
84
+
48
85
  # The re-entrancy guard (M3). The compactor is an agent, so compacting runs
49
- # the loop, which evaluates the trigger, which could compact. A thread-local
50
- # would be more precise, but the loop is synchronous and a compaction runs
51
- # to completion before its caller resumes -- and a module flag has the
52
- # property that matters more: it is impossible to be half-set.
53
- _compacting = False
86
+ # the loop, which evaluates the trigger, which could compact.
87
+ #
88
+ # PER THREAD SINCE BATCH 75, AND IT WAS A MODULE FLAG UNTIL THEN. That flag
89
+ # said: "a thread-local would be more precise, but the loop is synchronous
90
+ # and a compaction runs to completion before its caller resumes -- and a
91
+ # module flag has the property that matters more: it is impossible to be
92
+ # half-set." True when written, and §47's NA9 removed the premise. Two
93
+ # children of one response each reach `_maybe_compact` on their own thread,
94
+ # and a shared flag then does two wrong things:
95
+ #
96
+ # * the second child's `should_compact` returns "" and its compaction is
97
+ # silently skipped for that step, because a sibling is mid-fold;
98
+ # * whichever finishes first clears the flag while the other's compactor
99
+ # is still running, reopening exactly the window this guard closes.
100
+ #
101
+ # Neither is loud: a skipped compaction self-corrects on the next step, so
102
+ # the visible symptom would be a thread running into a provider's context
103
+ # limit with a guard that looked like it was working. It was the seventh
104
+ # member of slice 8's "six things that were only safe alone", found by
105
+ # reading this comment against what NA9 had done to it.
106
+ #
107
+ # The half-set property survives, which is why this is still a flag and not
108
+ # a lock: one bool per thread, set and cleared in a `finally` by the frame
109
+ # that owns it. Nothing here serialises anything -- two conversations
110
+ # folding at once is correct, since each needs its own fold.
111
+ #
112
+ # A FUNCTION RATHER THAN A BARE ATTRIBUTE, and the module name is GONE
113
+ # rather than kept as an alias: `if _compacting:` against a callable is
114
+ # always true, so a reader left holding the old name has to fail at import
115
+ # instead of quietly disabling every compaction in the process.
116
+ _reentrancy = threading.local()
117
+
118
+
119
+ def _is_compacting() -> bool:
120
+ """Whether THIS thread is already inside a compaction."""
121
+ return getattr(_reentrancy, "active", False)
122
+
123
+
124
+ def _set_compacting(active: bool) -> None:
125
+ """Mark this thread as compacting, or done."""
126
+ _reentrancy.active = bool(active)
54
127
 
55
128
 
56
129
  def _chars(messages) -> int:
@@ -59,6 +132,7 @@ def _chars(messages) -> int:
59
132
  for message in messages:
60
133
  total += len(str(message.get("content", "")))
61
134
  total += len(str(message.get("text", "")))
135
+ total += _image_count(message.get("images")) * _IMAGE_CHARS
62
136
  for call in message.get("tool_calls", ()) or ():
63
137
  total += len(str(call))
64
138
  return total
@@ -83,6 +157,27 @@ def estimated_tokens(messages) -> int:
83
157
  # about the second one.
84
158
  _context_window_warned: set = set()
85
159
 
160
+ # The same once-per-model discipline for a DIFFERENT warning: a window so
161
+ # small that the dependent floor leaves the trigger no usable headroom
162
+ # (ROADMAP_v3 §65 A24). A separate set rather than a shared one, keyed on
163
+ # the PAIR, because the two warnings answer different questions: the one
164
+ # above is about a model missing from a table, which has no provider
165
+ # dimension, while this one is about the number a provider REPORTED -- and
166
+ # the same model name behind a gateway and behind its own vendor can report
167
+ # different windows, so the pair is the thing that was warned about. Sharing
168
+ # one set would also mean whichever fired first silenced the other.
169
+ _floor_warned: set = set()
170
+
171
+ # And a third, for the case A24 left out on purpose: the headroom CAP, not
172
+ # the floor, leaves less room than one bounded read (TECHNICAL_DEBT 96,
173
+ # batch 220). Its own set, keyed the same way and for the same reasons.
174
+ _read_warned: set = set()
175
+
176
+ # Threads whose turn count has already been reported as yielding (#217,
177
+ # batch 221), so the INFO line is said once per thread. Keyed on the
178
+ # THREAD, not the model: the condition is a property of the conversation.
179
+ _yield_logged: set = set()
180
+
86
181
 
87
182
  # A trailing release date, as Anthropic and Google spell it.
88
183
  _DATE_SUFFIX = re.compile(r"-\d{8}$")
@@ -224,26 +319,282 @@ def context_limit(model: str, provider_name: Optional[str] = None) -> int:
224
319
  return window
225
320
 
226
321
 
322
+ def headroom_for(window: int) -> int:
323
+ """Tokens held between the trigger and `window` -- a CLAMPED share.
324
+
325
+ THE HEADROOM IS THE POINT, and batch 119 is where that got said out
326
+ loud. The trigger's job is not to cap the thread at a tidy percentage;
327
+ it is to leave room for the step that runs AFTER it fires, because
328
+ nothing in this harness recovers from a prompt that overflows the
329
+ window. There is no context_length_exceeded handler anywhere in
330
+ core/ -- grep for one. The provider raises, core/loop.py retries the
331
+ identical oversized prompt three times, and the turn dies. This number
332
+ is the only thing standing between a thread and that.
333
+
334
+ SO A BARE SHARE IS WRONG AT BOTH ENDS, which is the same shape of
335
+ error batch 44 corrected in the other direction. Batch 44 replaced a
336
+ flat 40k margin with a share because 40k is a rounding error under 1M
337
+ and strangling under 64k; a bare share has the mirror-image defect,
338
+ because what it holds CONSTANT is the percentage while the thing the
339
+ headroom must contain -- one step's growth -- does not scale with the
340
+ window at all. A single `max_read_chars` result is 12,500 proxy tokens
341
+ whether the model holds 64k or a million. 0.10 of 1M is 100k, which is
342
+ more than any step needs; 0.10 of 64k is 6,553, which one ordinary file
343
+ read overflows.
344
+
345
+ Hence a floor AND a cap, and each has a different job:
346
+
347
+ * COMPACTION_MIN_HEADROOM_TOKENS keeps a small window usable. It
348
+ binds under roughly 160k and nowhere above it, so a 262k or 1M
349
+ thread gets the fraction exactly as configured.
350
+ * COMPACTION_MAX_HEADROOM_FRACTION keeps the floor from eating a
351
+ small window whole -- 16k of a 32k window is half of it -- and it
352
+ is what keeps the resulting trigger positive at the bottom of the
353
+ range.
354
+
355
+ AND THAT IS NOT SUFFICIENT, which is TECHNICAL_DEBT 42 and the reason
356
+ derived_trigger() now floors below. This bullet used to end "and above
357
+ keep_recent_tokens at the bottom of the range, where
358
+ effective_compaction() would otherwise raise" -- true, and it names
359
+ ONE of the two dependent floors that function validates. The cap holds
360
+ the trigger at 0.75*window down here, so it clears keep_recent_tokens
361
+ (4,000) from a window of 5,334 up and clears warning_margin_tokens
362
+ (8,000) only from 10,667 up. Measured: keep_recent_tokens never bites
363
+ first anywhere in the range, which is exactly what made checking only
364
+ it read as covered.
365
+
366
+ MEASURED, not reasoned: the thread that produced this batch grew by
367
+ 31,733 proxy tokens in ONE step (a single 117k-character arxiv_search
368
+ result). At 0.85 its 262k window left 39,322 and that step fit; a bare
369
+ 0.90 leaves 26,215 and it does not.
370
+ """
371
+ # `window - int(fraction * window)` rather than
372
+ # `(1 - fraction) * window`, so the unclamped case reproduces the
373
+ # fraction EXACTLY. The two differ by the truncation: 0.10 * 262144 is
374
+ # 26214.4, and truncating the headroom down put the trigger one token
375
+ # ABOVE int(0.90 * 262144). One token is harmless and an arithmetic
376
+ # identity that holds only approximately is not -- every documented
377
+ # table would be off by one with nothing to point at.
378
+ share = window - int(config.COMPACTION_TRIGGER_FRACTION * window)
379
+ floor = config.COMPACTION_MIN_HEADROOM_TOKENS
380
+ cap = config.COMPACTION_MAX_HEADROOM_FRACTION * window
381
+ return int(min(max(share, floor), cap))
382
+
383
+
384
+ def _dependent_floor() -> int:
385
+ """The smallest trigger effective_compaction() will accept.
386
+
387
+ TECHNICAL_DEBT 42. That function validates the trigger against two
388
+ settings of identical form -- `warning_margin_tokens` and
389
+ `keep_recent_tokens`, each of which must be STRICTLY below it -- so the
390
+ smallest acceptable trigger is one above the larger of the two.
391
+
392
+ READ, NEVER HARDCODED, and that is the whole point of the helper. The
393
+ numbers are 8,000 and 4,000 today; a literal 8,001 here would be
394
+ correct until someone edited config.yaml:830, and then it would be a
395
+ floor that no longer matches the validator it exists to satisfy. This
396
+ reads the SAME resolution the validator reads, so the two cannot drift
397
+ apart -- which is item 41's lesson applied to the fix and not only to
398
+ the register entry.
399
+
400
+ Bare, with no overrides, deliberately: a per-invocation
401
+ `trigger_tokens` is the value being floored and must not decide its own
402
+ floor, while the two margins resolve identically either way.
403
+
404
+ COSTS ONE MERGE on should_compact()'s path. Measured: 3.267us against
405
+ thresholds()'s 4.918us, so 66% of it in relative terms and 0.065ms per
406
+ step of a research run in absolute ones -- against 4.696ms for one
407
+ ordinary tool result's redaction pass. The relative figure is the one
408
+ that looks alarming and the absolute figure is the one that is true.
409
+
410
+ DEFENSIVE ABOUT ITS OWN READ. effective_compaction() cannot raise under
411
+ config.yaml's 40,000 default, but a user settings.json naming a small
412
+ `trigger_tokens` makes it raise on the way to telling us the margins --
413
+ and derived_trigger() has callers outside thresholds(). A floor that
414
+ cannot be computed must never become the reason a thread cannot
415
+ compact, so this falls back to the config.py constants, which are the
416
+ same numbers one tier down.
417
+ """
418
+ try:
419
+ settings = config_loader.effective_compaction()
420
+ margin = settings["warning_margin_tokens"]
421
+ keep_recent = settings["keep_recent_tokens"]
422
+ except Exception: # noqa: BLE001 -- see above
423
+ margin = config.COMPACTION_WARNING_MARGIN_TOKENS
424
+ keep_recent = config.COMPACTION_KEEP_RECENT_TOKENS
425
+ return max(margin, keep_recent) + 1
426
+
427
+
428
+ def _warn_if_the_floor_left_no_room(model: str, provider_name: Optional[str],
429
+ window: int, share: int, floor: int,
430
+ trigger: int) -> None:
431
+ """Say ONCE, per (provider, model), that the floor is now the ceiling.
432
+
433
+ TECHNICAL_DEBT 42 raised a floor under the derived trigger so that a
434
+ small window could not make effective_compaction() raise on every
435
+ turn. The floor is read from the same settings the validator reads, and
436
+ it is applied WITH NO REFERENCE TO THE WINDOW -- so under the shipped
437
+ 8,000/4,000 margins it is 8,001 whatever the model holds, and at or
438
+ below 8,000 the trigger it produces is ABOVE the window. `used` is the
439
+ provider's own count of what it was sent, so it can never reach a
440
+ trigger above the window: compaction never fires, silently, and the
441
+ only line the user sees is the warn branch's "approaching the
442
+ compaction threshold", which collapses to warn_at == 1 down here and is
443
+ therefore both constant and false.
444
+
445
+ That is precisely the outcome item 42's own fix section rejected --
446
+ "compaction would then silently never run on that model, which is the
447
+ quiet-failure class this project keeps closing" -- arrived at from the
448
+ other side. The floor stays, because the raise it removed was worse;
449
+ what was missing is this line.
450
+
451
+ FIRES ONLY WHERE THE FLOOR IS THE OPERATIVE VALUE, which is A24's
452
+ ruling and is measured rather than guessed: the floor binds at or below
453
+ a 10,666-token window and nowhere above it, because headroom's cap
454
+ holds the share-derived trigger at 0.75*window down there. A window of
455
+ 16,384 or 32,768 also leaves less headroom than one bounded read
456
+ (12,500 proxy tokens), but the CAP is what bound it, the cap is
457
+ deliberate (see headroom_for), and the only remedy there is a bigger
458
+ model -- so a warning naming two settings would name the wrong thing.
459
+ That case is TECHNICAL_DEBT 96, and since batch 220 it has a line of its
460
+ own, `_warn_if_one_read_cannot_fit`, naming the read instead.
461
+
462
+ Two wordings for one condition, because the remedy differs: at or above
463
+ the window the trigger cannot be reached at all, and just below it the
464
+ thread folds but the next step has almost nothing to grow into.
465
+ """
466
+ if floor < share:
467
+ return
468
+ one_read = int(config.MAX_READ_CHARS) // CHARS_PER_TOKEN
469
+ if window - trigger >= one_read:
470
+ return
471
+ key = (provider_name or "", model)
472
+ if key in _floor_warned:
473
+ return
474
+ _floor_warned.add(key)
475
+ where = f" on {provider_name}" if provider_name else ""
476
+ settings = (f"compaction.warning_margin_tokens "
477
+ f"({config.COMPACTION_WARNING_MARGIN_TOKENS}) and "
478
+ f"compaction.keep_recent_tokens "
479
+ f"({config.COMPACTION_KEEP_RECENT_TOKENS})")
480
+ if trigger >= window:
481
+ logger.warning(
482
+ "Compaction can NEVER fire for %r%s: its context window is %s "
483
+ "tokens and the smallest trigger the settings allow is %s, which "
484
+ "is above it. The thread will grow until the provider refuses "
485
+ "the request. The floor is one token above the larger of %s -- "
486
+ "lowering them lowers it.", model, where, window, trigger,
487
+ settings)
488
+ return
489
+ logger.warning(
490
+ "Compaction for %r%s can only fire with %s tokens of headroom left "
491
+ "(window %s, trigger %s, held up by the floor rather than derived "
492
+ "from the window): one bounded tool read is about %s proxy tokens, "
493
+ "so the step after a fold can still overflow. The floor is one token "
494
+ "above the larger of %s -- lowering them lowers it.",
495
+ model, where, window - trigger, window, trigger, one_read, settings)
496
+
497
+
498
+ def _warn_if_one_read_cannot_fit(model: str, provider_name: Optional[str],
499
+ window: int, share: int, floor: int,
500
+ trigger: int) -> None:
501
+ """Say ONCE, per (provider, model), that one read can overflow the window.
502
+
503
+ TECHNICAL_DEBT 96 (#246, batch 220): the case the warning above leaves
504
+ out on purpose. On a 16,384- or 32,768-token window the floor does not
505
+ bind; headroom_for's CAP does, and it leaves 4,096 or 8,192 tokens
506
+ between the trigger and the window -- less than one bounded read's
507
+ 12,500. A thread just under the trigger takes one large `read`, the
508
+ prompt overflows, and nothing in core/ recovers from that.
509
+
510
+ A24 was right that naming the two margins would name the wrong remedy.
511
+ The right one is the READ: `max_read_chars` is a setting, and a value
512
+ that fits the headroom keeps one read inside the window. The owner chose
513
+ to say so over shrinking reads per model, which would need the window
514
+ plumbed into every tool that reads the global. It applies to every
515
+ model, and the line says that too.
516
+ """
517
+ if floor >= share:
518
+ return # the floor binds; the warning above owns that case
519
+ one_read = int(config.MAX_READ_CHARS) // CHARS_PER_TOKEN
520
+ headroom = window - trigger
521
+ if headroom >= one_read:
522
+ return
523
+ key = (provider_name or "", model)
524
+ if key in _read_warned:
525
+ return
526
+ _read_warned.add(key)
527
+ where = f" on {provider_name}" if provider_name else ""
528
+ logger.warning(
529
+ "Compaction for %r%s fires at %s tokens of a %s-token window, leaving "
530
+ "%s tokens of headroom: less than one bounded tool read "
531
+ "(max_read_chars is %s characters, about %s tokens), so one large "
532
+ "read just below the trigger can overflow the window and end the "
533
+ "turn. A max_read_chars of %s or less in config.yaml keeps a read "
534
+ "inside it; it applies to every model.",
535
+ model, where, trigger, window, headroom, config.MAX_READ_CHARS,
536
+ one_read, headroom * CHARS_PER_TOKEN)
537
+
538
+
227
539
  def derived_trigger(model: str, provider_name: Optional[str] = None) -> int:
228
- """The working-set trigger for this model: a share of its window.
540
+ """The working-set trigger for this model: its window less headroom.
229
541
 
230
- config.COMPACTION_TRIGGER_FRACTION x context_limit(), which is 850k on
231
- a 1M model and 170k on a 200k one. Batch 44's reversal of M1; the
232
- argument is on the constant.
542
+ Batch 44's reversal of M1, as corrected by batch 119. The argument for
543
+ deriving this from the window at all is on COMPACTION_TRIGGER_FRACTION;
544
+ the argument for the clamps is on headroom_for() above.
233
545
 
234
- A FRACTION AND NOT `window - margin`, which was the shape considered
235
- first and is what the research backstop below actually uses. A fixed
236
- margin cannot be right at both ends of the roster: 40k below a 1M
237
- window is a rounding error, and 40k below a 64k window leaves 24k,
238
- which is not a working thread. A share scales by construction.
546
+ STILL NOT `window - margin`, which is the BACKSTOP's shape and is what
547
+ batch 44 rejected: that is a FIXED margin, right at neither end of the
548
+ roster. This is a share that may not shrink below an absolute nor grow
549
+ past a share -- which is the same number at every window above ~160k
550
+ and a floor below it.
239
551
 
240
552
  Floored at 1 rather than validated: context_limit() can only return a
241
553
  positive number, and effective_compaction() raises on a trigger below
242
554
  1, so this exists to keep an absurd window from turning into a
243
555
  confusing error two layers away from the value that caused it.
556
+
557
+ AND FLOORED AGAIN, above the dependent floors (TECHNICAL_DEBT 42).
558
+ Headroom's cap holds the trigger at 0.75*window down here, so a window
559
+ of 10,666 or less derives a trigger at or below warning_margin_tokens
560
+ and effective_compaction() raises -- on should_compact()'s path, which
561
+ runs at the top of every step, with both _maybe_compact call sites bare
562
+ `yield from`. Every turn on such a model died before the first model
563
+ call.
564
+
565
+ REACHED BY A PROVIDER, NOT BY A COMMAND, which is what the register
566
+ entry got wrong. /window refuses anything at or below the pipeline
567
+ backstop margin (20,000), which is stricter than 10,666 and covers that
568
+ route by coincidence; MODEL_CONTEXT_WINDOWS's smallest entry is 200,000
569
+ and DEFAULT_CONTEXT_WINDOW is 256,000. Source 1 -- what the provider
570
+ SAID -- has no guard at all, and 8,192 is the commonest real window a
571
+ self-hosted or gateway endpoint reports.
572
+
573
+ BELOW ~10,667 THIS STOPS BEING A DERIVATION, and that is said out loud
574
+ rather than left to be discovered. The returned value is the floor, not
575
+ 0.75*window, so the fraction no longer describes it. A derivation that
576
+ silently stops deriving is the drift class this record is full of; the
577
+ alternative was distrusting the provider's own number, which is a
578
+ larger claim for a smaller gain.
579
+
580
+ AND AT OR BELOW 8,000 IT STOPS BEING A TRIGGER, which is the step the
581
+ paragraph above did not take (batch 148). The floor is applied
582
+ with no reference to the window, so below itself it is simply larger
583
+ than the window and nothing can reach it. Measured: a 4,096-token
584
+ window derives 8,001 and should_compact() answers "warn" at every value
585
+ from 1 to 4,097 and "compact" at none of them. The value is unchanged --
586
+ _warn_if_the_floor_left_no_room above is the whole fix, because what was
587
+ missing was never a number.
244
588
  """
245
- return max(1, int(config.COMPACTION_TRIGGER_FRACTION
246
- * context_limit(model, provider_name)))
589
+ window = context_limit(model, provider_name)
590
+ share = window - headroom_for(window)
591
+ floor = _dependent_floor()
592
+ trigger = max(1, share, floor)
593
+ _warn_if_the_floor_left_no_room(model, provider_name, window, share,
594
+ floor, trigger)
595
+ _warn_if_one_read_cannot_fit(model, provider_name, window, share, floor,
596
+ trigger)
597
+ return trigger
247
598
 
248
599
 
249
600
  def _trigger_is_configured() -> bool:
@@ -274,7 +625,10 @@ def thresholds(model: str, mode: str = "working_set",
274
625
  unattended and each one already returns a distillation, so routine
275
626
  compaction there would spend on a judgment call nobody is watching.
276
627
  Since batch 44 both are derived from context_limit(); what still
277
- separates them is how much room each leaves.
628
+ separates them is how much room each leaves -- AND SINCE BATCH 148 the
629
+ backstop is ordered against the trigger rather than merely given a
630
+ bigger number, because "how much room each leaves" was answering the
631
+ question backwards below 200,000 (batch 148).
278
632
 
279
633
  FOUR SOURCES FOR THE WORKING-SET TRIGGER, nearest first: an explicit
280
634
  `overrides` (a `/compact --strength N`), a session `/trigger` bound to
@@ -297,8 +651,32 @@ def thresholds(model: str, mode: str = "working_set",
297
651
  overrides = {"trigger_tokens": trigger, **(overrides or {})}
298
652
  settings = config_loader.effective_compaction(overrides)
299
653
  if mode == "backstop":
654
+ # ABOVE THE WORKING-SET TRIGGER, STRUCTURALLY (batch 148).
655
+ # "A last resort" is a statement about ORDER, and this was a flat
656
+ # 20,000-token margin while batch 44 made the ordinary trigger a
657
+ # clamped share -- so the backstop left MORE room than the trigger,
658
+ # that is fired EARLIER, at every window below 200,000. Measured:
659
+ # 4,000 tokens earlier at 64,000 and at 131,072, 11,808 earlier at
660
+ # 32,768, and at or below 20,001 the old max(1, ...) clamped it to 1,
661
+ # so every evaluation of every step of every pass asked compact().
662
+ # M6's argument -- a pass is headless and unattended, so routine
663
+ # compaction there spends on a judgment call nobody is watching --
664
+ # was inverted for most of the roster.
665
+ #
666
+ # THE SHIPPED TABLE IS WHY NOTHING CAUGHT IT: every entry is at or
667
+ # above 200,000, where the two numbers were equal or already in the
668
+ # right order. The inversion is invisible on the table and universal
669
+ # off it, and source 1 -- what a provider SAID -- is the way in.
670
+ #
671
+ # derived_trigger rather than the fully resolved working-set value:
672
+ # it cannot raise (see its floor), and M6 speaks about the derived
673
+ # default rather than about a number a human typed. A configured
674
+ # trigger high enough to still invert this is one that is itself
675
+ # near or above the window, and a reachable backstop under an
676
+ # unreachable trigger is the right way round.
300
677
  compact_at = max(
301
- 1, context_limit(model, provider_name)
678
+ derived_trigger(model, provider_name) + 1,
679
+ context_limit(model, provider_name)
302
680
  - config.COMPACTION_PIPELINE_BACKSTOP_TOKENS)
303
681
  return compact_at, compact_at
304
682
  compact_at = settings["trigger_tokens"]
@@ -321,7 +699,7 @@ def should_compact(used: int, model: str, mode: str = "working_set",
321
699
  context-limit error from the provider instead. The budget stop
322
700
  condition and this trigger fail together, from the same cause.
323
701
  """
324
- if _compacting:
702
+ if _is_compacting():
325
703
  return ""
326
704
  warn_at, compact_at = thresholds(model, mode, overrides, provider_name)
327
705
  if used >= compact_at:
@@ -331,21 +709,107 @@ def should_compact(used: int, model: str, mode: str = "working_set",
331
709
  return ""
332
710
 
333
711
 
712
+ def _sub_turn_span(rows, starts, settings, current_turn_start):
713
+ """A watermark INSIDE one turn, or None. Batch 120, register item 40.
714
+
715
+ THE LAST RESORT, reached only when no turn-aligned fold exists at all.
716
+ Batch 119 made the turn COUNT yield, which unsticks any thread with two
717
+ completed turns; what it could not reach is a thread whose only turn is
718
+ the one still running. The reported case was 142 rows and 168,311 proxy
719
+ tokens inside ONE turn: over the trigger, entirely unfoldable, and
720
+ answering `no-progress` on every evaluation forever.
721
+
722
+ WHAT THIS AMENDS, PRECISELY. M5 says the current turn is never foldable
723
+ because "a summary that swallowed the messages the model is reasoning
724
+ about right now would rewrite the question mid-answer." The reason is
725
+ right and it does not cover the whole turn: after forty-six steps the
726
+ model is not reasoning about step three's tool output, and what must
727
+ survive verbatim is the turn's own REQUEST plus the recent steps. Both
728
+ do. The opening user row is re-added by `pinned_through` (see its
729
+ docstring) and `keep_recent_tokens` of the tail is held back here, so
730
+ the question is never rewritten -- it is carried unchanged, with the
731
+ middle of the work folded between it and the recent steps.
732
+
733
+ M4 IS SATISFIED BY CONSTRUCTION, not by checking. Its rule is that a
734
+ fold must never separate an assistant message carrying `tool_calls`
735
+ from the `tool_result` rows answering them, because a `tool_result`
736
+ with no preceding `tool_use` is an HTTP 400 on Anthropic. Inside a turn
737
+ the archive is `user, (assistant, tool+)*` -- D20 persists the
738
+ assistant row before dispatch and the answering rows after it returns,
739
+ including every row of a parallel dispatch -- so **cutting immediately
740
+ before an assistant row leaves every earlier call answered.** Measured
741
+ on the reported thread: 47 assistant rows inside the turn, 46 candidate
742
+ boundaries, and 20 of its steps carry between 2 and 5 tool rows, which
743
+ is exactly why the boundary is the assistant row and not an arbitrary
744
+ position.
745
+
746
+ Returns None rather than a boundary when the turn holds fewer than two
747
+ steps. One step cannot be folded without folding all of it, and a turn
748
+ with no steps is a bare request.
749
+ """
750
+ index = (current_turn_start if current_turn_start is not None
751
+ else len(starts) - 1)
752
+ if not 0 <= index < len(starts):
753
+ return None
754
+ opener = starts[index]
755
+
756
+ steps = [i for i in range(opener + 1, len(rows))
757
+ if rows[i]["role"] == "assistant"]
758
+ if len(steps) < 2:
759
+ return None
760
+
761
+ # keep_recent_tokens of the tail, WHOLE STEPS at a time -- the same
762
+ # floor and the same character proxy the turn-aligned path uses, just
763
+ # counted over steps instead of turns, so the two cannot disagree about
764
+ # how much verbatim tail a thread is promised.
765
+ bounds = steps + [len(rows)]
766
+ kept = 0
767
+ cut = steps[-1]
768
+ for position in range(len(steps) - 1, 0, -1):
769
+ kept += sum(_row_chars(rows[i])
770
+ for i in range(bounds[position], bounds[position + 1]))
771
+ cut = steps[position]
772
+ if kept // CHARS_PER_TOKEN >= settings["keep_recent_tokens"]:
773
+ break
774
+
775
+ # A BACKSTOP, and it is unreachable: the loop stops at `steps[1]` at the
776
+ # earliest, so `cut` always exceeds `steps[0]` once there are two steps.
777
+ # Kept rather than argued away, for `pinned_through`'s reason one file
778
+ # over -- the failure it guards is a watermark that folds the request
779
+ # and nothing else, which is the one outcome here that would be wrong
780
+ # rather than merely unhelpful.
781
+ if cut <= steps[0]:
782
+ return None
783
+ return rows[cut - 1]["id"]
784
+
785
+
334
786
  def compactable_span(memory, current_turn_start: Optional[int] = None,
335
787
  overrides: Optional[dict] = None):
336
788
  """The id of the last message that may be folded, or None if nothing
337
789
  may be.
338
790
 
339
- THREE FLOORS, AND THE EARLIEST WINS (M5):
791
+ THREE FLOORS, AND THE EARLIEST WINS (M5) -- WITH ONE AMENDMENT, batch
792
+ 119, argued in full at the `preferred` line below:
340
793
 
341
794
  * the current turn -- structural, never foldable at all. A summary
342
795
  that swallowed the messages the model is reasoning about right now
343
- would rewrite the question mid-answer.
796
+ would rewrite the question mid-answer. ABSOLUTE.
797
+ * keep_recent_tokens, the token floor §21 specifies. ABSOLUTE, and
798
+ it can be absolute safely because it is a SIZE: it holds exactly
799
+ what it says and then stops.
344
800
  * the last N COMPLETED turns, verbatim. A single tool-heavy turn can
345
801
  consume the whole keep-token budget by itself, leaving the
346
802
  immediately preceding exchange summarized, and a follow-up like
347
- "no, the other one" then has no referent.
348
- * keep_recent_tokens, the token floor §21 specifies.
803
+ "no, the other one" then has no referent. A PREFERENCE SINCE BATCH
804
+ 119: it is a COUNT, so it cannot tell three small turns from three
805
+ enormous ones, and it applies only while it still leaves something
806
+ foldable. Measured at 230k tokens held by `3 - 3 = 0` on a thread
807
+ 92.5% of which could have been folded.
808
+
809
+ So the guarantee this function makes is size-based rather than
810
+ count-based: the current turn verbatim, plus at least
811
+ keep_recent_tokens of completed turns verbatim, plus the last N
812
+ completed turns whenever that is affordable.
349
813
 
350
814
  THE BOUNDARY IS ALWAYS A TURN START (M4). Every floor resolves to one,
351
815
  so a fold can never separate an assistant message carrying tool_calls
@@ -380,10 +844,7 @@ def compactable_span(memory, current_turn_start: Optional[int] = None,
380
844
  # Every candidate boundary is "fold everything strictly before turn i".
381
845
  # The floors each veto some suffix of the turn list; the surviving
382
846
  # boundary is the earliest turn any of them still allows.
383
- cutoff = len(starts) - settings["keep_recent_turns"]
384
- if current_turn_start is not None:
385
- cutoff = min(cutoff, current_turn_start)
386
-
847
+ #
387
848
  # The token floor: keep whole turns, walking back from the end, until
388
849
  # keep_recent_tokens is covered. Sized from the persisted content,
389
850
  # which is the same character proxy M10 uses everywhere else.
@@ -391,15 +852,159 @@ def compactable_span(memory, current_turn_start: Optional[int] = None,
391
852
  kept = 0
392
853
  token_cutoff = len(starts)
393
854
  for index in range(len(starts) - 1, -1, -1):
394
- kept += sum(len(rows[i]["content"])
855
+ kept += sum(_row_chars(rows[i])
395
856
  for i in range(bounds[index], bounds[index + 1]))
396
857
  token_cutoff = index
397
858
  if kept // CHARS_PER_TOKEN >= settings["keep_recent_tokens"]:
398
859
  break
399
- cutoff = min(cutoff, token_cutoff)
400
860
 
401
- if cutoff < 1:
402
- return None
861
+ # THE TWO FLOORS THAT CAN MEASURE WHAT THEY PROTECT, intersected first.
862
+ cutoff = token_cutoff
863
+ if current_turn_start is not None:
864
+ cutoff = min(cutoff, current_turn_start)
865
+
866
+ # ...and then keep_recent_turns, WHICH YIELDS RATHER THAN VETOES.
867
+ # Batch 119, and this is an amendment to M5 rather than a fix to a
868
+ # mistake in it.
869
+ #
870
+ # M5 names three floors and says the earliest wins. That is right for
871
+ # two of them, because only two can measure what they are holding:
872
+ #
873
+ # * the current turn is STRUCTURAL -- folding it would rewrite the
874
+ # question being answered -- and is never negotiable;
875
+ # * keep_recent_tokens is a SIZE, so it protects exactly as much as
876
+ # it says and then stops;
877
+ # * keep_recent_turns is a COUNT, and a count is blind. Three turns
878
+ # can be 2k tokens or 230k.
879
+ #
880
+ # Blind is harmless until the thread is over the trigger. Then it is
881
+ # the one floor that can hold EVERYTHING while claiming to hold "the
882
+ # last three exchanges" -- and at that point the alternative to folding
883
+ # is not "keep the recent turns", it is the provider refusing the next
884
+ # prompt. There is no context-overflow handler anywhere in core/, so
885
+ # nothing downstream recovers.
886
+ #
887
+ # MEASURED, on the owner-reported thread that produced this batch: 182
888
+ # rows, 729k chars, 230k tokens by the provider's own count, and
889
+ # exactly THREE user turns. `3 - 3 = 0`, cutoff < 1, blocked -- while
890
+ # the token floor on its own would have folded everything before turn 2:
891
+ # 168,564 proxy tokens, 92.5% of the thread, with the last completed
892
+ # turn and the current one untouched. A good fold existed and a turn
893
+ # count vetoed it: on every evaluation, through a switch to a 1M-window
894
+ # model (this function takes no model and never did), and through a
895
+ # restart (it reads the archive, which does not change). There was no
896
+ # way out of that state from inside the app.
897
+ #
898
+ # NOT A WALK-DOWN from 3 to 2 to 1 taking the first count that works.
899
+ # On that same thread `keep_recent_turns=2` yields cutoff 1, which folds
900
+ # turn 0's 253 tokens, writes a checkpoint, and then answers no-progress
901
+ # forever -- a compaction that succeeds, frees 0.1%, and leaves
902
+ # advances() permanently satisfied against a watermark that helped
903
+ # nothing. Yielding all the way to the size-aware floors is what makes
904
+ # the model call worth spending.
905
+ #
906
+ # Still guaranteed afterwards: the current turn verbatim, plus at least
907
+ # keep_recent_tokens of completed turns verbatim. A size-based promise,
908
+ # which is the right kind to make in answer to size pressure.
909
+ preferred = min(cutoff, len(starts) - settings["keep_recent_turns"])
910
+ if preferred >= 1:
911
+ cutoff = preferred
912
+ elif cutoff >= 1:
913
+ # A STANDING CONDITION, said once per thread (batch 221, #217).
914
+ # This runs once per compact(), which _maybe_compact calls after
915
+ # every step while the thread is over the trigger -- so at INFO it
916
+ # repeated per step for the life of the conversation, the shape #44
917
+ # took out of this module. The first time a thread yields says it;
918
+ # the rest are DEBUG, like the no-progress line that follows them.
919
+ level = (logging.DEBUG if memory.thread_id in _yield_logged
920
+ else logging.INFO)
921
+ _yield_logged.add(memory.thread_id)
922
+ logger.log(
923
+ level,
924
+ "Compaction on thread %s: keep_recent_turns=%d would have held "
925
+ "all %d turns and blocked every fold, so it yielded. The current "
926
+ "turn and keep_recent_tokens still hold.",
927
+ memory.thread_id, settings["keep_recent_turns"], len(starts))
928
+
929
+ # THE TURN-ALIGNED FOLD HAS TO BE WORTH A MODEL CALL, not merely legal.
930
+ # Batch 134, and this is the gate `cutoff < 1` used to be.
931
+ #
932
+ # `completed_turns()` is `len(turn_start_ids) - 1`, so it is at least 1
933
+ # for any thread with two user rows, and the token floor's loop assigns
934
+ # `token_cutoff = index` BEFORE testing -- so it stops at
935
+ # `len(starts) - 1` as soon as the last turn alone exceeds
936
+ # keep_recent_tokens. Between them, `cutoff` was >= 1 on every thread
937
+ # with a second user row, and the sub-turn path behind `cutoff < 1` was
938
+ # unreachable there. The boundary was then the best TURN-ALIGNED one,
939
+ # which on a thread whose current turn is almost all of it is the end of
940
+ # the first turn.
941
+ #
942
+ # MEASURED on one 40-step turn, with and without one short exchange in
943
+ # front of it:
944
+ #
945
+ # 40 steps, one user row SUB-TURN at row 73 folds 90.0%
946
+ # one exchange, then the same turn-aligned row 2 folds 0.0%
947
+ # two exchanges, then the same turn-aligned row 4 folds 0.1%
948
+ # five exchanges, then the same turn-aligned row 6 folds 0.1%
949
+ #
950
+ # And it was a DEAD END rather than a slow start, because the useless
951
+ # fold satisfies `advances()` permanently: every later evaluation
952
+ # answered `no-progress`, at `logger.debug`, with `kind=None` so
953
+ # `_maybe_compact` appended no notice -- nothing on screen while the
954
+ # thread ran toward the provider's limit, where `headroom_for`'s
955
+ # docstring says the loop retries the identical oversized prompt three
956
+ # times and the turn dies. A sweep of every legal
957
+ # keep_recent_tokens x keep_recent_turns pair reached a sub-turn
958
+ # boundary 0 times in 30 accepted settings.
959
+ #
960
+ # This is the trap item 39 names and says it avoided -- "a successful
961
+ # compaction that cannot be repeated" -- arriving on the DEFAULT
962
+ # configuration with no settings edit at all, and item 40's own measured
963
+ # scenario (two turns, 97.2% available) is the case the shipped gate
964
+ # excluded.
965
+ #
966
+ # THE FLOOR IS keep_recent_tokens, which is the thing the fold exists to
967
+ # make room behind: a boundary that frees less than the tail it must
968
+ # keep has not answered the size pressure it was called for. Below it,
969
+ # fall through to the sub-turn path exactly as `cutoff < 1` does -- and
970
+ # that branch is kept untouched, so the single-turn case batch 120 fixed
971
+ # is unchanged. `_sub_turn_span` itself needs nothing: with
972
+ # `current_turn_start >= 1` it already resolves `opener = starts[index]`
973
+ # to the current turn's user row and scans forward from there.
974
+ worth_it = True
975
+ if cutoff >= 1:
976
+ freed = sum(_row_chars(rows[i])
977
+ for i in range(starts[cutoff])) // CHARS_PER_TOKEN
978
+ worth_it = freed >= settings["keep_recent_tokens"]
979
+ if not worth_it:
980
+ logger.info(
981
+ "Compaction on thread %s: the turn-aligned boundary would "
982
+ "free %d proxy tokens against a keep_recent_tokens of %d, "
983
+ "which is not worth a model call, so the boundary is inside "
984
+ "the current turn instead.",
985
+ memory.thread_id, freed, settings["keep_recent_tokens"])
986
+
987
+ if cutoff < 1 or not worth_it:
988
+ # NO TURN-ALIGNED FOLD WORTH MAKING. Batch 120 tries one inside the
989
+ # turn before giving up -- see _sub_turn_span for what that costs
990
+ # and why M4 holds. Last, and only here: a turn-aligned boundary is
991
+ # always preferable WHEN IT FREES ENOUGH, since it needs no row
992
+ # re-added and no amendment to M5 at all.
993
+ inside = _sub_turn_span(rows, starts, settings, current_turn_start)
994
+ if inside is not None:
995
+ if cutoff < 1:
996
+ logger.info(
997
+ "Compaction on thread %s: no turn-aligned fold exists "
998
+ "(%d turns, all held), so the boundary is inside the "
999
+ "current turn. Its request and the recent steps stay "
1000
+ "verbatim.",
1001
+ memory.thread_id, len(starts))
1002
+ return inside
1003
+ # A sub-turn fold is not available either (fewer than two steps in
1004
+ # the current turn). A small turn-aligned fold still beats nothing,
1005
+ # so fall back to it rather than refusing to compact at all.
1006
+ if cutoff < 1:
1007
+ return None
403
1008
  # Fold everything strictly before turn `cutoff`, so the watermark is
404
1009
  # the row just before that turn's first row. cutoff >= 1 guarantees
405
1010
  # there is one.
@@ -431,39 +1036,81 @@ def _as_text(messages) -> str:
431
1036
  body = f"{body}\n[called: {', '.join(c.get('name', '?') for c in calls)}]"
432
1037
  else:
433
1038
  body = str(message.get("content", ""))
1039
+ # ROADMAP_v3 §49 (SS5). A user-role row the HARNESS wrote -- a
1040
+ # background session's result -- is not something the user said,
1041
+ # and a summary that attributed program output to them would be
1042
+ # M8's mistake one layer down.
1043
+ if message.get("harness"):
1044
+ role = "harness"
434
1045
  lines.append(f"{role}: {body}")
435
1046
  return "\n\n".join(lines)
436
1047
 
437
1048
 
438
- def pin_measurements(thread_id, last_n: int) -> dict:
1049
+ def pin_measurements(thread_id, last_n: int, model: Optional[str] = None,
1050
+ provider_name: Optional[str] = None) -> dict:
439
1051
  """What #89's cap is measured against, in M10's character proxy.
440
1052
 
441
1053
  Returns three numbers:
442
1054
  requested_tokens what the last `last_n` turns' ARCHIVE rows cost --
443
1055
  the span one more `pin(last_n)` would protect
444
1056
  pinned_tokens everything ALREADY pinned on this thread
445
- max_tokens PIN_MAX_TRIGGER_FRACTION x the resolved trigger
1057
+ max_tokens PIN_MAX_TRIGGER_FRACTION x the trigger that FIRES
446
1058
 
447
1059
  Reads the ARCHIVE (the same rows compactable_span walks), so a pinned
448
1060
  prefix is counted at full weight exactly as the derived view will
449
1061
  carry it. Never compared against a provider count (M10).
1062
+
1063
+ THE PAIR IS OPTIONAL AND THE ANSWER DEPENDS ON IT (batch 148).
1064
+ #89's cap is a fraction OF THE TRIGGER, and since batch 44 the trigger
1065
+ that decides when a thread folds is derived per model -- so reading the
1066
+ configured number alone made the cap 20,000 for every model and both
1067
+ directions wrong. Measured against the trigger that actually fires:
1068
+ 250% of it on an 8,192-token window, where one pin could protect more
1069
+ than the whole trigger and floor the thread permanently above it, which
1070
+ is the state #89 exists to prevent; and 8% of it on a 262,144-token
1071
+ window, where `pin` refuses requests the stated policy allows. The
1072
+ value was unchanged by batch 44 and the PROPERTY was not, which is the
1073
+ marathon's shape 9, and AGENTS.md's "so #89's cap is unchanged" is the
1074
+ sentence that made it look settled.
1075
+
1076
+ The stated obstacle was real and is gone: this function had no
1077
+ (provider, model) to match a session override against, and
1078
+ `tools/builtin/pin.py` receives `parent_run` -- already in
1079
+ registry._INJECTABLE_PARAMS, already carrying both -- one declared
1080
+ parameter away. With the pair the trigger is resolved exactly as
1081
+ thresholds() resolves it, so a session `/trigger`, a configured one and
1082
+ the derivation all reach the cap through the chain that already exists.
1083
+ Without it the binding-free read below is the fallback, so nothing that
1084
+ calls this from outside a run moves.
450
1085
  """
451
- from storage import _ordered_rows
452
1086
  from core import config_loader
1087
+ from storage import _ordered_rows
453
1088
 
454
1089
  settings = config_loader.effective_compaction()
455
1090
  # THE MINIMUM, and the min() is load-bearing rather than defensive.
456
- # This function has no (provider, model) to match a session override
457
- # against, so it reads the binding-free one -- and takes whichever
458
- # trigger is SMALLER, which is safe under every binding. A raised
459
- # session trigger leaves the cap stricter than it needs to be; a
460
- # lowered one pulls the cap down with it. Reading the configured
461
- # trigger alone would let `/trigger 12k` leave this cap at 20k, so a
462
- # pin could protect more than the whole trigger and floor the thread
463
- # permanently above it -- which is the exact defect #89 exists to
464
- # prevent, reintroduced through a number typed in a different command.
1091
+ # With no (provider, model) to match a session override against, this
1092
+ # reads the binding-free one -- and takes whichever trigger is SMALLER,
1093
+ # which is safe under every binding. A raised session trigger leaves
1094
+ # the cap stricter than it needs to be; a lowered one pulls the cap
1095
+ # down with it. Reading the configured trigger alone would let
1096
+ # `/trigger 12k` leave this cap at 20k, so a pin could protect more
1097
+ # than the whole trigger and floor the thread permanently above it --
1098
+ # which is the exact defect #89 exists to prevent, reintroduced
1099
+ # through a number typed in a different command.
465
1100
  trigger = min(settings["trigger_tokens"],
466
1101
  session.trigger_any() or settings["trigger_tokens"])
1102
+ if model:
1103
+ try:
1104
+ trigger = thresholds(model, provider_name=provider_name)[1]
1105
+ except Exception: # noqa: BLE001 -- see below
1106
+ # _dependent_floor's posture, for the same reason: a cap that
1107
+ # cannot be computed must never become the reason a pin fails.
1108
+ # effective_compaction raises on an incoherent settings.json,
1109
+ # and the binding-free read above is still a cap.
1110
+ logger.debug(
1111
+ "Could not resolve the compaction trigger for %r on %r; "
1112
+ "measuring the pin cap against the configured trigger.",
1113
+ model, provider_name)
467
1114
  max_tokens = int(config.PIN_MAX_TRIGGER_FRACTION * trigger)
468
1115
 
469
1116
  rows = _ordered_rows(thread_id)
@@ -471,11 +1118,11 @@ def pin_measurements(thread_id, last_n: int) -> dict:
471
1118
  bounds = starts + [len(rows)]
472
1119
 
473
1120
  def _tokens(lo: int) -> int:
474
- return sum(len(rows[i]["content"])
1121
+ return sum(_row_chars(rows[i])
475
1122
  for i in range(lo, len(rows))) // CHARS_PER_TOKEN
476
1123
 
477
1124
  requested = _tokens(bounds[max(0, len(starts) - last_n)]) if starts else 0
478
- pinned = sum(len(row["content"]) for row in rows
1125
+ pinned = sum(_row_chars(row) for row in rows
479
1126
  if row["pinned"]) // CHARS_PER_TOKEN
480
1127
  return {
481
1128
  "requested_tokens": requested,
@@ -527,6 +1174,45 @@ def _input_budget(model: str, provider_name: Optional[str] = None) -> int:
527
1174
  - config.COMPACTION_PIPELINE_BACKSTOP_TOKENS)
528
1175
 
529
1176
 
1177
+ def parse_compact_args(args: str) -> tuple:
1178
+ """(overrides, error). ROADMAP_v2 §21's per-invocation override --
1179
+ applies to this one run and persists nothing, which is the last step
1180
+ of D27's resolution order.
1181
+
1182
+ `/compact`'s grammar in either shell: both parse here, so one cannot
1183
+ accept a spelling the other refuses (ROADMAP_v3 §89; moved from
1184
+ tui/app.py).
1185
+
1186
+ An unrecognised flag is an ERROR, not something to ignore. Silently
1187
+ dropping `--strenght 4` would compact at the default strength while
1188
+ the user believes they asked for something else, and the result looks
1189
+ exactly like a compaction that obeyed them.
1190
+ """
1191
+ overrides: dict = {}
1192
+ tokens = args.split()
1193
+ index = 0
1194
+ while index < len(tokens):
1195
+ token = tokens[index]
1196
+ if token in ("--strength", "-s"):
1197
+ index += 1
1198
+ if index >= len(tokens):
1199
+ return None, "--strength needs a value from 1 to 5."
1200
+ try:
1201
+ overrides["strength"] = int(tokens[index])
1202
+ except ValueError:
1203
+ return None, f"--strength needs a number, got {tokens[index]!r}."
1204
+ elif token.startswith("--strength="):
1205
+ value = token.split("=", 1)[1]
1206
+ try:
1207
+ overrides["strength"] = int(value)
1208
+ except ValueError:
1209
+ return None, f"--strength needs a number, got {value!r}."
1210
+ else:
1211
+ return None, f"Unknown option {token!r}. Usage: /compact [--strength N]"
1212
+ index += 1
1213
+ return overrides, None
1214
+
1215
+
530
1216
  def compact(memory, model: str, provider_name: str,
531
1217
  current_turn_start: Optional[int] = None,
532
1218
  overrides: Optional[dict] = None,
@@ -585,16 +1271,17 @@ def compact(memory, model: str, provider_name: str,
585
1271
  (`allowed_tools: []` in its definition), and the re-entrancy guard, so
586
1272
  it can neither call anything nor trigger a compaction of its own.
587
1273
  """
588
- global _compacting
589
-
590
1274
  from agents.manager import manager
591
- from core.loop import RunAgentLoop, DEFAULT_SYSTEM_PROMPT
1275
+ from core.loop import DEFAULT_SYSTEM_PROMPT, RunAgentLoop
592
1276
  from storage import (
593
- advances, history_through, latest_checkpoint, save_checkpoint,
1277
+ advances,
1278
+ history_through,
1279
+ latest_checkpoint,
1280
+ save_checkpoint,
594
1281
  )
595
1282
  storage_advances = advances
596
1283
 
597
- if _compacting:
1284
+ if _is_compacting():
598
1285
  return {"status": "reentrant", "kind": None,
599
1286
  "text": "A compaction is already running."}
600
1287
 
@@ -607,8 +1294,24 @@ def compact(memory, model: str, provider_name: str,
607
1294
  return {
608
1295
  "status": "blocked",
609
1296
  "kind": "compaction_blocked",
610
- "text": ("Context is full but the most recent turns fill it "
611
- "entirely -- nothing could be compacted."),
1297
+ # TWO FALSE CLAIMS IN ONE LINE, both corrected in batch 119.
1298
+ #
1299
+ # "Context is full" -- it is not. The trigger is a fraction of
1300
+ # the window, so this is read with the headroom still unspent,
1301
+ # and the owner who reported it reasonably concluded the harness
1302
+ # treats the window and the trigger as one number. The usage
1303
+ # line showing the TRIGGER as its only denominator is the other
1304
+ # half of that impression; batch 119 fixes both.
1305
+ #
1306
+ # "the most recent turns fill it entirely" -- names SIZE, and
1307
+ # what actually blocked the one reported occurrence was a turn
1308
+ # COUNT on a thread 92.5% of which was foldable. The single
1309
+ # diagnostic this message offered pointed away from its cause,
1310
+ # which is worse than offering none.
1311
+ "text": ("Over the compaction trigger, but every turn is held "
1312
+ "by the keep floors -- nothing could be folded. The "
1313
+ "thread keeps running uncompacted; a fold becomes "
1314
+ "possible once this turn completes."),
612
1315
  }
613
1316
 
614
1317
  agent = manager.get(config.COMPACTOR_AGENT)
@@ -661,13 +1364,44 @@ def compact(memory, model: str, provider_name: str,
661
1364
  return seg, text
662
1365
 
663
1366
  segment, segment_text = _build(chaining)
664
- if not segment and not chaining:
1367
+ if not segment:
665
1368
  # Everything newly foldable is pinned (#89). This used to be the
666
1369
  # one silent no-progress path -- indistinguishable, at every
667
1370
  # level, from "nothing needed doing" -- while the state it reports
668
1371
  # is exactly what compaction_blocked's text describes, reached by
669
1372
  # another door. Named now, and worded so the reader knows the way
670
1373
  # out.
1374
+ #
1375
+ # ON THE SEGMENT, NOT ON THE STRATEGY (batch 148). This
1376
+ # read `if not segment and not chaining:` from the day it was
1377
+ # written, and `chaining` is true under the SHIPPED
1378
+ # `compaction_strategy: chain` whenever a checkpoint exists -- so
1379
+ # the default strategy fell straight through the branch built for
1380
+ # this state. What it fell into is the harm the no-progress guard
1381
+ # twenty lines above names as its own reason: "chain would be
1382
+ # handed an empty segment and summarize its own previous summary,
1383
+ # degrading it a little on every attempt". That guard catches a
1384
+ # watermark that has not MOVED; it cannot catch a watermark that
1385
+ # moved across a region which is entirely pinned, because the
1386
+ # boundary genuinely advanced. This branch is the guard for that
1387
+ # case and the strategy test removed the default from its cover.
1388
+ #
1389
+ # Measured through the real compact() with three pinned turns in a
1390
+ # ten-turn thread: the compactor was handed exactly
1391
+ # "Summary of everything before this point:\n<previous summary>\n\n"
1392
+ # and nothing else, at a 379-character target, and the stored
1393
+ # checkpoint went from 2,489 characters to 23 -- reported as
1394
+ # status="folded", kind="compaction", "0 earlier messages compacted
1395
+ # into a summary". A success line for a durable 99% loss, and it
1396
+ # compounds geometrically on every repeat.
1397
+ #
1398
+ # EMPTY MEANS PINNED, exactly: history_through excludes pinned rows
1399
+ # (§21 AC2) and excludes nothing else, so an empty segment on a
1400
+ # watermark that advanced is this state and no other. The message
1401
+ # is therefore as true under chain as under rederive, and an
1402
+ # explicit /compact is refused here for the same reason the
1403
+ # no-progress guard refuses one -- degrading a summary is not a
1404
+ # tighter compaction of it.
671
1405
  return {
672
1406
  "status": "all-pinned",
673
1407
  "kind": "compaction_blocked",
@@ -705,22 +1439,33 @@ def compact(memory, model: str, provider_name: str,
705
1439
  # state the cut in both places the summary travels.
706
1440
  segment_text, truncation_notice = _truncate_oldest(segment_text, budget)
707
1441
  if truncation_notice:
1442
+ # Both, as the forced-chain branch above does (#216, batch 213).
1443
+ # The target used to stay the untruncated length's, so the
1444
+ # instruction named one length and capped against another, and
1445
+ # the strength the user set came out looser by their ratio.
708
1446
  original = len(segment_text)
1447
+ target = _target_chars(original, settings["strength"])
709
1448
 
710
- _compacting = True
1449
+ _set_compacting(True)
711
1450
  try:
712
1451
  # Batch 59. ONE span over the whole call, retries included: this is
713
1452
  # one compactor run that may take several calls to hit its target,
714
1453
  # and a shell drawing a row per attempt would report the retry loop
715
- # as a stack of compactors. Inside the `_compacting` guard rather
1454
+ # as a stack of compactors. Inside the re-entrancy guard rather
716
1455
  # than around it, so the two brackets nest instead of interleaving.
717
1456
  with agent_activity.span(activity, agent.name, depth):
718
1457
  summary = _summarize(
719
1458
  RunAgentLoop, manager, agent, DEFAULT_SYSTEM_PROMPT,
720
1459
  segment_text, target, original, model, provider_name,
721
- settings["max_retries"], authorization)
1460
+ settings["max_retries"], authorization,
1461
+ # §47. The compactor already had a span; what it did not
1462
+ # have was a way to say WHOSE conversation it is
1463
+ # summarising. It is a child of that thread in the only
1464
+ # sense that matters -- the summary is of it -- so the
1465
+ # link is not a convention, it is what happened.
1466
+ parent_thread_id=memory.thread_id)
722
1467
  finally:
723
- _compacting = False
1468
+ _set_compacting(False)
724
1469
 
725
1470
  if summary is None:
726
1471
  # #136, batch 20: this used to be kind=None -- invisible to both
@@ -797,16 +1542,17 @@ def summarize_thread(thread_id, model: str, provider_name: str,
797
1542
  referencing thread, so a proportional target would let one big thread
798
1543
  inject tens of kilobytes into every call indefinitely.
799
1544
  """
800
- global _compacting
801
-
802
1545
  from agents.manager import manager
803
- from core.loop import RunAgentLoop, DEFAULT_SYSTEM_PROMPT
1546
+ from core.loop import DEFAULT_SYSTEM_PROMPT, RunAgentLoop
804
1547
  from storage import (
805
- advances, archive_history, last_message_id, latest_thread_summary,
1548
+ advances,
1549
+ archive_history,
1550
+ last_message_id,
1551
+ latest_thread_summary,
806
1552
  save_thread_summary,
807
1553
  )
808
1554
 
809
- if _compacting:
1555
+ if _is_compacting():
810
1556
  return None
811
1557
 
812
1558
  watermark = last_message_id(thread_id)
@@ -862,7 +1608,7 @@ def summarize_thread(thread_id, model: str, provider_name: str,
862
1608
  # trigger. Its own thread is one turn long and would not trip it today,
863
1609
  # which is exactly the kind of "cannot happen yet" that stops being true
864
1610
  # quietly.
865
- _compacting = True
1611
+ _set_compacting(True)
866
1612
  try:
867
1613
  # Batch 59, and see compact() above for why one span covers the
868
1614
  # retries. §21c's /summary runs this from its own worker, so it is
@@ -873,7 +1619,7 @@ def summarize_thread(thread_id, model: str, provider_name: str,
873
1619
  thread_text, target, original, model, provider_name,
874
1620
  settings["max_retries"], authorization)
875
1621
  finally:
876
- _compacting = False
1622
+ _set_compacting(False)
877
1623
 
878
1624
  if summary is None:
879
1625
  return None
@@ -895,7 +1641,8 @@ def summarize_thread(thread_id, model: str, provider_name: str,
895
1641
 
896
1642
 
897
1643
  def _summarize(loop_cls, manager, agent, base_prompt, segment_text, target,
898
- original, model, provider_name, max_retries, authorization):
1644
+ original, model, provider_name, max_retries, authorization,
1645
+ parent_thread_id=None):
899
1646
  """Run the compactor, retrying while the summary overshoots its target.
900
1647
 
901
1648
  STRUCTURALLY §3's JSON-retry loop with a length constraint in place of
@@ -915,8 +1662,19 @@ def _summarize(loop_cls, manager, agent, base_prompt, segment_text, target,
915
1662
  discarding it would spend the calls and keep the full history, leaving
916
1663
  the thread exactly as stuck as before with less budget to fix it.
917
1664
  """
1665
+ from core.loop import advertisement_facts
1666
+
918
1667
  context = manager.active_context(agent)
919
- system_prompt = manager.system_prompt_for(agent, base_prompt, context=context)
1668
+ # #228: the catalogs this prompt carries are decided by the run it is
1669
+ # for, through the bundle that run is given below -- the reviewer's
1670
+ # rule (core/reasoning/review.py). With system_prompt_for's attended
1671
+ # defaults this was right only while compactor.md's empty
1672
+ # allowed_tools kept both catalog tools out; a widening would have
1673
+ # reopened #68 on every automatic compaction of a headless run.
1674
+ callable_only, granted = advertisement_facts(authorization)
1675
+ system_prompt = manager.system_prompt_for(
1676
+ agent, base_prompt, context=context,
1677
+ callable_only=callable_only, granted=granted)
920
1678
  tolerance = int(target * (1 + config.COMPACTION_RATIO_TOLERANCE))
921
1679
 
922
1680
  response = loop_cls.run_agent_conversation(
@@ -934,6 +1692,12 @@ def _summarize(loop_cls, manager, agent, base_prompt, segment_text, target,
934
1692
  # served them. The compactor is an agent, so it is labelled like the
935
1693
  # other agent-shaped runs.
936
1694
  thread_kind=THREAD_KIND_SUBAGENT,
1695
+ # §47. No parent CALL -- no tool started this -- so the run is
1696
+ # findable by the conversation it belongs to and by nothing
1697
+ # else, which is the honest shape for a thread the harness
1698
+ # opened on its own initiative.
1699
+ thread_parent=parent_thread_id,
1700
+ thread_agent=agent.name,
937
1701
  )
938
1702
  summary = (response.text or "").strip()
939
1703