@yahyahammad/venastine 0.2.0 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/.env.example +9 -4
  2. package/CONFIG_ARCHITECTURE.md +2030 -0
  3. package/LICENSE +201 -201
  4. package/NOTICE +1 -1
  5. package/README.md +1034 -895
  6. package/agents/builtin/build.md +73 -0
  7. package/agents/builtin/compactor.md +75 -75
  8. package/agents/builtin/explore.md +77 -77
  9. package/agents/builtin/general.md +80 -0
  10. package/agents/builtin/pipeline-reviewer.md +100 -100
  11. package/agents/builtin/plan.md +106 -88
  12. package/agents/builtin/review.md +74 -74
  13. package/agents/builtin/test.md +65 -0
  14. package/agents/builtin/writer.md +67 -0
  15. package/agents/manager.py +68 -2
  16. package/agents/subagent_tool.py +472 -238
  17. package/agents/tui_commands.py +238 -151
  18. package/bin/venastine.mjs +150 -4
  19. package/bin/venastine.py +71 -0
  20. package/cli_commands.py +838 -0
  21. package/config.py +95 -1198
  22. package/config.yaml +1103 -0
  23. package/config_edit.py +1301 -0
  24. package/config_schema.py +971 -0
  25. package/config_update.py +749 -0
  26. package/core/agent_activity.py +256 -118
  27. package/core/ansi.py +452 -0
  28. package/core/approval.py +137 -115
  29. package/core/call_view.py +364 -0
  30. package/core/cancel.py +121 -0
  31. package/core/chatgpt_auth.py +747 -0
  32. package/core/claims_view.py +107 -0
  33. package/core/client.py +2715 -1559
  34. package/core/commands.py +460 -0
  35. package/core/compaction.py +1735 -971
  36. package/core/config_loader.py +1982 -1563
  37. package/core/connect.py +686 -0
  38. package/core/events.py +123 -61
  39. package/core/file_mentions.py +280 -0
  40. package/core/interaction.py +411 -328
  41. package/core/line_pattern.py +128 -0
  42. package/core/loop.py +2566 -1490
  43. package/core/memory.py +482 -370
  44. package/core/pipeline_models.py +190 -176
  45. package/core/provider_errors.py +234 -0
  46. package/core/reasoning/authorization.py +130 -130
  47. package/core/reasoning/base.py +219 -219
  48. package/core/reasoning/confidence_scoring.py +484 -468
  49. package/core/reasoning/events.py +149 -149
  50. package/core/reasoning/json_retry.py +19 -1
  51. package/core/reasoning/orchestrator.py +1561 -1382
  52. package/core/reasoning/output_writer.py +345 -286
  53. package/core/reasoning/payload_validation.py +524 -398
  54. package/core/reasoning/pipeline_storage.py +88 -22
  55. package/core/reasoning/review.py +659 -615
  56. package/core/reasoning/scholar.py +458 -434
  57. package/core/reasoning/source_corpus.py +433 -327
  58. package/core/reasoning/source_scoring.py +855 -855
  59. package/core/replay.py +315 -183
  60. package/core/session.py +36 -12
  61. package/core/session_view.py +259 -0
  62. package/core/session_wake.py +259 -0
  63. package/core/shell_sessions.py +1553 -0
  64. package/core/workspace_trust.py +1 -1
  65. package/credentials.py +148 -20
  66. package/database.py +333 -252
  67. package/{PRIVACY.md → docs/PRIVACY.md} +78 -66
  68. package/docs/SECURITY.md +108 -0
  69. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  70. package/env_secrets.py +50 -9
  71. package/json_store.py +343 -123
  72. package/logging_setup.py +295 -185
  73. package/main.py +3433 -1961
  74. package/mcp_client/client.py +733 -499
  75. package/mcp_client/config.py +126 -21
  76. package/mcp_client/registration.py +343 -182
  77. package/memories/manager.py +147 -132
  78. package/memories/tui_commands.py +319 -244
  79. package/package.json +303 -73
  80. package/project_init/config_files.py +282 -278
  81. package/project_init/generator.py +728 -564
  82. package/project_init/manifest.py +18 -6
  83. package/project_init/tui_commands.py +147 -144
  84. package/prompts/critic_pass.md +24 -24
  85. package/prompts/revalidate.md +25 -25
  86. package/prompts/revise.md +2 -0
  87. package/prompts/source_grounding.md +66 -66
  88. package/prompts/system_prompts.py +316 -262
  89. package/providers.json.example +4 -2
  90. package/pyproject.toml +149 -100
  91. package/requirements.txt +295 -72
  92. package/safety/policy_enforcement.py +1082 -746
  93. package/security/askpass.py +419 -0
  94. package/security/capability.py +207 -177
  95. package/security/confine.py +484 -0
  96. package/security/hosted.py +182 -0
  97. package/security/permissions.py +237 -190
  98. package/security/posture.py +124 -12
  99. package/security/protected_paths.py +501 -309
  100. package/security/sandbox.py +4108 -985
  101. package/security/secrets.py +855 -0
  102. package/skills/builtin/math/formal-specification.md +67 -67
  103. package/skills/builtin/math/numerical-methods.md +66 -66
  104. package/skills/builtin/research/experiment-design.md +62 -62
  105. package/skills/builtin/research/reproducibility.md +55 -55
  106. package/skills/builtin/research/source-evaluation.md +70 -70
  107. package/skills/builtin/research/statistical-inference.md +63 -63
  108. package/skills/builtin/research/technical-writing.md +59 -59
  109. package/skills/builtin/software/code-review.md +61 -61
  110. package/skills/builtin/software/debugging.md +61 -61
  111. package/skills/builtin/software/test-design.md +67 -67
  112. package/skills/tui_commands.py +110 -111
  113. package/storage.py +1338 -1005
  114. package/tools/base.py +161 -1
  115. package/tools/builtin/_math_common.py +667 -377
  116. package/tools/builtin/_net_common.py +225 -19
  117. package/tools/builtin/arxiv.py +497 -39
  118. package/tools/builtin/ask_user.py +231 -213
  119. package/tools/builtin/discrete_math.py +35 -4
  120. package/tools/builtin/fetch_url.py +371 -153
  121. package/tools/builtin/file_ops.py +720 -412
  122. package/tools/builtin/geometry.py +48 -12
  123. package/tools/builtin/get_time.py +2 -2
  124. package/tools/builtin/linear_algebra.py +66 -23
  125. package/tools/builtin/load_skill.py +76 -66
  126. package/tools/builtin/logic.py +171 -12
  127. package/tools/builtin/pin.py +227 -193
  128. package/tools/builtin/probability_stats.py +12 -16
  129. package/tools/builtin/project_docs.py +464 -405
  130. package/tools/builtin/remember.py +175 -158
  131. package/tools/builtin/shell.py +980 -461
  132. package/tools/builtin/shell_sessions.py +613 -0
  133. package/tools/builtin/symbolic_math.py +80 -10
  134. package/tools/builtin/todo.py +149 -147
  135. package/tools/builtin/web_search.py +191 -13
  136. package/tools/context.py +159 -136
  137. package/tools/isolation.py +107 -34
  138. package/tools/registry.py +1242 -925
  139. package/tui/app.py +8318 -4100
  140. package/tui/app.tcss +747 -556
  141. package/tui/commands.py +26 -170
  142. package/tui/diffs.py +34 -13
  143. package/tui/history.py +128 -128
  144. package/tui/hosted_driver.py +102 -0
  145. package/tui/markdown.py +1329 -1279
  146. package/tui/meters.py +364 -268
  147. package/tui/ravens.py +146 -146
  148. package/tui/screens.py +1255 -943
  149. package/tui/themes.py +599 -559
  150. package/tui/widgets.py +3813 -2332
  151. package/SECURITY.md +0 -88
  152. package/THIRD_PARTY_NOTICES.md +0 -100
package/core/memory.py CHANGED
@@ -1,370 +1,482 @@
1
- """
2
- core/memory.py
3
-
4
- Keeps the running message history for one agent conversation, backed by
5
- real persistence. Stores history in a PROVIDER-NEUTRAL shape -- not
6
- Anthropic's content-block format, not OpenAI's tool_calls format -- so
7
- this file has no idea which provider is in use. core/client.py is the
8
- only place that translates this neutral shape into whatever wire format
9
- a specific provider's API actually wants, the same way it already
10
- translates tool schemas per provider.
11
-
12
- Neutral shapes stored in self._messages:
13
- user: {"role": "user", "content": "text"}
14
- assistant: {"role": "assistant", "text": "...", "tool_calls": [
15
- {"id": "...", "name": "...", "input": {...}}, ...]}
16
- tool: {"role": "tool", "tool_call_id": "...", "content": "..."}
17
- (one entry per individual tool result)
18
-
19
- PERSISTENCE, AND WHY THERE'S NO NORMALIZATION STEP HERE:
20
- Each add_* method passes storage.save_message() only the PAYLOAD that
21
- role actually needs -- plain text for user, {"text", "tool_calls"} for
22
- assistant, a plain result string for tool -- never the whole neutral
23
- entry dict, which would redundantly re-wrap "role" (and, for tool rows,
24
- "tool_call_id") a second time inside the persisted content itself.
25
- storage.get_session_history() reconstructs the exact neutral shape
26
- above directly, per msg.role, so what comes back on resume is
27
- identical to what a fresh add_* call would have produced -- no
28
- post-hoc reshaping step is needed here, for this role or any future one.
29
-
30
- An earlier version of this file had exactly the bug this design avoids:
31
- add_assistant_message's persisted shape was nested and lossy on
32
- resume, a read-side "_normalize_resumed_history()" function was added
33
- to fix it -- but only for role == "assistant". add_tool_result had the
34
- identical root cause (same "pass the whole entry dict" pattern) and was
35
- never touched, because the fix lived one layer above where the actual
36
- divergence was produced. Fixing what gets WRITTEN, once, here, closes
37
- this for every role without a matching special case at every point
38
- it's read back. See tests/test_memory_write_through.py for the
39
- regression tests (now covering the tool row's `content`, not just its
40
- `role`/`tool_call_id`, which is exactly the assertion that was missing
41
- before).
42
- """
43
-
44
- from typing import Optional
45
- from uuid import UUID
46
-
47
- from storage import (
48
- create_thread, get_thread, save_message, get_session_history,
49
- update_thread_extra, latest_checkpoint, pinned_through,
50
- message_ids_from, set_pinned, turn_start_ids, THREAD_KIND_CHAT,
51
- )
52
-
53
- # ROADMAP_v2 §21 (M8). How a compaction summary is introduced to the model.
54
- #
55
- # The neutral shape has three roles -- user, assistant, tool -- and no
56
- # "system" among them, because the system prompt travels separately. A
57
- # leading USER message is therefore the only form all three providers
58
- # accept in the message list, and it is where a conversation naturally
59
- # starts anyway.
60
- #
61
- # It is NEVER persisted as a MessageLog row. Doing so would put derived
62
- # content in an append-only archive whose whole value is being original,
63
- # and the next compaction would then summarize the previous summary as if
64
- # it were something the user said -- chaining by accident, on a code path
65
- # that means to re-derive.
66
- SUMMARY_PREFIX = (
67
- "[Summary of the earlier part of this conversation, compacted to save "
68
- "context. The full history is preserved and unedited; this is what is "
69
- "being shown to you now.]\n\n"
70
- )
71
-
72
-
73
- class ConversationMemory:
74
- def __init__(self, thread_id: Optional[UUID] = None,
75
- kind: str = THREAD_KIND_CHAT) -> None:
76
- """`kind` (§27 T1) says what a NEW thread is, and is forwarded to
77
- storage.create_thread and nowhere else.
78
-
79
- This class does not learn what a kind means, store it, or read it
80
- back. It owns active in-run state and must stay unaware of what
81
- data exists -- that boundary is storage.py's. Ignored when
82
- `thread_id` is given, because resuming does not reclassify: a
83
- thread's kind was decided when it was created.
84
- """
85
- # Session-scoped billing (#4): zero on EVERY construction, fresh or
86
- # resumed -- see record_billed for why this one is deliberately not
87
- # extra_data like its neighbour.
88
- self.billed_tokens = 0
89
- if thread_id is None:
90
- self.thread_id = create_thread(kind=kind)
91
- self._messages: list[dict] = []
92
- self._extra: dict = {}
93
- self._kind = kind
94
- else:
95
- thread = get_thread(thread_id)
96
- if thread is None:
97
- raise ValueError(f"No conversation thread found with id {thread_id}")
98
- self.thread_id = thread_id
99
- self._extra = dict(thread.get("extra_data") or {})
100
- self._messages = self._derived_view()
101
- # The kind fallback (missing column on a database whose ALTER
102
- # has not run) lives inside get_thread's copy now; resuming is
103
- # not reclassifying, so the stored kind is simply reported.
104
- self._kind = thread.get("kind") or THREAD_KIND_CHAT
105
-
106
- @property
107
- def kind(self) -> str:
108
- """What this thread IS (§27 T1): "chat", "research_pass" or
109
- "subagent".
110
-
111
- A deliberate boundary bend, recorded rather than slipped in
112
- (#43, batch 16). §27 says this class "learns nothing from kind" --
113
- it forwards the kwarg at creation and ignores it on resume --
114
- because kind SEMANTICS are storage's to store and the loop's to
115
- decide. That still holds: nothing here branches on the value.
116
- What changed is that the loop can now ASK, because batch 16 gave
117
- it a decision to make from the answer -- which compaction mode a
118
- re-entered thread runs under derives from what the thread IS
119
- (research_pass and subagent threads get M6's backstop wherever
120
- they are resumed), and the memory is the object holding the
121
- thread. Reporting a stored fact is not interpreting it.
122
- """
123
- return self._kind
124
-
125
- # -- the derived view (§21) --------------------------------------------
126
-
127
- def _derived_view(self) -> list[dict]:
128
- """What the agent sees this turn: a view computed over the archive,
129
- not a second copy of it.
130
-
131
- Three parts, in order:
132
- 1. the latest checkpoint's summary, as a leading user message
133
- 2. pinned rows from inside the span that summary covers (M9),
134
- verbatim -- the summary skipped them, and one watermark
135
- cannot say "everything through K except these"
136
- 3. every archive row after the watermark, verbatim
137
-
138
- A thread that has never been compacted has no checkpoint, so this
139
- returns exactly what get_session_history() always returned. That
140
- equivalence is what keeps every pre-§21 caller untouched until the
141
- trigger first fires on a thread, and it is asserted directly.
142
- """
143
- checkpoint = latest_checkpoint(self.thread_id)
144
- if checkpoint is None:
145
- return get_session_history(self.thread_id)
146
-
147
- watermark = checkpoint["covers_up_to_message_id"]
148
- summary = {
149
- "role": "user",
150
- "content": f"{SUMMARY_PREFIX}{checkpoint['summary_text']}",
151
- }
152
- return (
153
- [summary]
154
- + pinned_through(self.thread_id, watermark)
155
- + get_session_history(self.thread_id, after_message_id=watermark)
156
- )
157
-
158
- def apply_checkpoint(self) -> None:
159
- """Rebuild the in-memory view after a compaction was recorded.
160
-
161
- Deliberately re-reads rather than splicing the summary into the
162
- list it already holds. The assembly rules (a leading summary,
163
- pinned re-inclusion, the tail) then live in exactly one place, so
164
- a mid-run compaction and a resumed thread cannot produce different
165
- views of the same checkpoint -- which is the divergence a second
166
- assembly path would eventually introduce.
167
- """
168
- self._messages = self._derived_view()
169
-
170
- # -- measured context size (§21) ----------------------------------------
171
-
172
- @property
173
- def last_input_tokens(self) -> int:
174
- """The provider's own count of everything it was last sent on this
175
- thread, or 0 if it has never been measured.
176
-
177
- This is what §21's compaction trigger reads. It is exact and it
178
- comes from the provider rather than from a local tokenizer that
179
- would have to be right per model -- but it is retrospective, which
180
- is why the trigger is evaluated after a response and acts before
181
- the next call.
182
-
183
- PERSISTED, via the same extra_data write-through goal mode uses.
184
- §21 says a resumed thread needs a rough character estimate before
185
- its first call because nothing has been measured yet; carrying the
186
- last real measurement across the process boundary means the
187
- estimate is only ever needed for a thread written before this
188
- section existed.
189
- """
190
- return int(self._extra.get("last_input_tokens") or 0)
191
-
192
- def record_input_tokens(self, tokens: int) -> None:
193
- """Store the measurement from the call that just returned. Ignores
194
- a zero, which is what D21's raise exists to prevent reaching here
195
- and what a provider reporting no usage would otherwise write over
196
- a real earlier number."""
197
- if tokens:
198
- self.set_extra("last_input_tokens", int(tokens))
199
-
200
- # -- session billing (#4) -------------------------------------------------
201
-
202
- def record_billed(self, tokens: int) -> None:
203
- """Accumulate this thread's billed input+output for the SESSION.
204
-
205
- Deliberately NOT persisted, unlike last_input_tokens beside it:
206
- the scope the TUI's usage line shows is THREAD-SINCE-RESUME. A
207
- lifetime total would need a schema column to mean anything (a
208
- resumed thread would silently restart at zero anyway, reading as
209
- a reset), and #4's figure exists for relative reasoning about
210
- this session, not as a ledger.
211
- """
212
- self.billed_tokens += int(tokens or 0)
213
-
214
- def completed_turns(self) -> int:
215
- """How many turns of this thread are already finished (§21 M5/M11).
216
-
217
- COUNTED FROM THE ARCHIVE, never from `self._messages`. M8 makes a
218
- checkpoint summary a `role: "user"` message, so a compacted view
219
- holds one user message that is not a turn -- and this number is
220
- compared against archive-space turn indices in
221
- compaction.compactable_span(). Counting the view was §21a's shipped
222
- defect: the floor came out several turns too low, the computed
223
- watermark went BACKWARDS, and the second automatic compaction of a
224
- thread un-compacted it. See tests/test_storage_e2e.py.
225
-
226
- Lives here rather than in core/loop.py because this is the file
227
- that owns the thread's persistence -- the loop imports no storage
228
- at all, and should not start.
229
-
230
- Called before the first model call of a turn, and every wrapper
231
- adds the user message first, so the turn about to run is the last
232
- entry and is excluded. That off-by-one IS the structural floor:
233
- everything from this index on is the current turn and can never be
234
- folded.
235
-
236
- Archive-space also makes the value STABLE for a whole turn. The
237
- archive never changes under compaction, so a count taken at the top
238
- of a turn stays correct after a mid-turn compaction rebuilds the
239
- view; the view-based version silently went stale at exactly that
240
- point.
241
- """
242
- return len(turn_start_ids(self.thread_id)) - 1
243
-
244
- # -- pinning (§21) ------------------------------------------------------
245
-
246
- def pin_last(self, turns: int = 1) -> int:
247
- """Mark the last `turns` turns compaction-exempt. Returns how many
248
- rows were newly pinned.
249
-
250
- ORDINAL, not an id. §21 keeps MessageLog.id out of the neutral
251
- shape -- the model never sees one and so cannot name one -- and
252
- adding it would put a persistence concept into the provider-neutral
253
- contract the whole memory design rests on, with three translation
254
- branches then needing to strip it before the wire. "Pin the last N
255
- turns" needs no id and is how the tool would actually be used.
256
-
257
- A turn starts at a user message and runs to just before the next
258
- one, so pinning a turn keeps the question together with its answer
259
- and any tool results between them. Asking for more turns than the
260
- thread has pins the whole thread rather than erroring: the intent
261
- ("keep this conversation") is unambiguous and there is nothing to
262
- clarify.
263
- """
264
- if turns < 1:
265
- return 0
266
- starts = turn_start_ids(self.thread_id)
267
- if not starts:
268
- return 0
269
- first = starts[max(0, len(starts) - turns)]
270
- return set_pinned(message_ids_from(self.thread_id, first))
271
-
272
- def unpin_last(self, turns: int = 1) -> int:
273
- """Release the protection on the last `turns` turns (#89). The
274
- mirror of pin_last, and what D26's "reversible" premise needed to
275
- become true: set_pinned's pinned=False branch was written since
276
- §21 and unreachable until this existed.
277
-
278
- Same ordinal interface, same clamp (asking for more turns than
279
- the thread has releases everything -- the intent is unambiguous),
280
- same return contract (rows actually flipped, so "nothing was
281
- pinned" reads as zero)."""
282
- if turns < 1:
283
- return 0
284
- starts = turn_start_ids(self.thread_id)
285
- if not starts:
286
- return 0
287
- first = starts[max(0, len(starts) - turns)]
288
- return set_pinned(message_ids_from(self.thread_id, first),
289
- pinned=False)
290
-
291
- @property
292
- def extra(self) -> dict:
293
- """Snapshot of the thread's extra_data (goal mode §18). A copy --
294
- mutate via set_extra(), which persists write-through like the
295
- message methods."""
296
- return dict(self._extra)
297
-
298
- def set_extra(self, key: str, value) -> None:
299
- """Write one extra_data key (value=None deletes), persisting
300
- immediately so a goal set from the TUI is visible to the next
301
- run in any shell."""
302
- update_thread_extra(self.thread_id, key, value)
303
- if value is None:
304
- self._extra.pop(key, None)
305
- else:
306
- self._extra[key] = value
307
-
308
- def add_user_message(self, text: str) -> None:
309
- entry = {"role": "user", "content": text}
310
- self._messages.append(entry)
311
- save_message(self.thread_id, role="user", content=text)
312
-
313
- def add_assistant_message(self, response) -> None:
314
- """
315
- Takes the NORMALIZED ModelResponse from
316
- core.client.call_model_stream() directly -- not a raw SDK object.
317
- This is what eliminated the previous Anthropic-only
318
- `response.raw.content` reach-through in loop.py; all three
319
- provider branches produce the same ModelResponse shape (.text,
320
- .tool_calls), so this method never needs to know which provider
321
- generated it.
322
-
323
- This used to name `call_model()`, which had no production caller
324
- (#38) -- so the one sentence telling a reader where this argument
325
- comes from named a function that never produced one. `raw` went
326
- with it: it was set only there, so it was None on every live path
327
- while its comment said it held the SDK response.
328
- """
329
- tool_calls = [
330
- {"id": tc.id, "name": tc.name, "input": tc.input}
331
- for tc in response.tool_calls
332
- ]
333
- entry = {"role": "assistant", "text": response.text, "tool_calls": tool_calls}
334
- # §44: the turn's reasoning, in the provider's own blocks. Read
335
- # off the response rather than accumulated from the display
336
- # deltas, because an Anthropic block carries a signature the
337
- # prose does not -- see ModelResponse.thinking. Attached to the
338
- # in-memory entry as well as persisted, so a live turn and a
339
- # resumed one hand _messages_for_provider the same shape; that is
340
- # the whole point, and it is why this is one field and not two
341
- # code paths.
342
- thinking = getattr(response, "thinking", None)
343
- if thinking:
344
- entry["thinking"] = thinking
345
- self._messages.append(entry)
346
- # Persist only the assistant-specific payload, not the whole
347
- # entry -- "role" is already carried by save_message's own
348
- # `role=` argument, and duplicating it inside `content` is
349
- # exactly the redundancy that caused the resume-shape bug. The
350
- # reasoning has its own column for the same reason: it is not
351
- # part of what the model said.
352
- save_message(
353
- self.thread_id,
354
- role="assistant",
355
- content={"text": response.text, "tool_calls": tool_calls},
356
- thinking=thinking,
357
- )
358
-
359
- def add_tool_result(self, tool_call_id: str, result: dict) -> None:
360
- result_text = str(result)
361
- entry = {"role": "tool", "tool_call_id": tool_call_id, "content": result_text}
362
- self._messages.append(entry)
363
- # Persist only the result string. tool_call_id already has its
364
- # own column (passed below) -- it doesn't need to also be
365
- # duplicated inside content.
366
- save_message(self.thread_id, role="tool", content=result_text, tool_call_id=tool_call_id)
367
-
368
- @property
369
- def messages(self) -> list[dict]:
370
- return self._messages
1
+ """
2
+ core/memory.py
3
+
4
+ Keeps the running message history for one agent conversation, backed by
5
+ real persistence. Stores history in a PROVIDER-NEUTRAL shape -- not
6
+ Anthropic's content-block format, not OpenAI's tool_calls format -- so
7
+ this file has no idea which provider is in use. core/client.py is the
8
+ only place that translates this neutral shape into whatever wire format
9
+ a specific provider's API actually wants, the same way it already
10
+ translates tool schemas per provider.
11
+
12
+ Neutral shapes stored in self._messages:
13
+ user: {"role": "user", "content": "text"}
14
+ assistant: {"role": "assistant", "text": "...", "tool_calls": [
15
+ {"id": "...", "name": "...", "input": {...}}, ...]}
16
+ tool: {"role": "tool", "tool_call_id": "...", "content": "..."}
17
+ (one entry per individual tool result; plus "images": [
18
+ {"media_type": "...", "data": "<base64>"}, ...] only when a
19
+ tool attached one -- ROADMAP_v3 §81)
20
+
21
+ PERSISTENCE, AND WHY THERE'S NO NORMALIZATION STEP HERE:
22
+ Each add_* method passes storage.save_message() only the PAYLOAD that
23
+ role actually needs -- plain text for user, {"text", "tool_calls"} for
24
+ assistant, a plain result string for tool -- never the whole neutral
25
+ entry dict, which would redundantly re-wrap "role" (and, for tool rows,
26
+ "tool_call_id") a second time inside the persisted content itself.
27
+ storage.get_session_history() reconstructs the exact neutral shape
28
+ above directly, per msg.role, so what comes back on resume is
29
+ identical to what a fresh add_* call would have produced -- no
30
+ post-hoc reshaping step is needed here, for this role or any future one.
31
+
32
+ An earlier version of this file had exactly the bug this design avoids:
33
+ add_assistant_message's persisted shape was nested and lossy on
34
+ resume, a read-side "_normalize_resumed_history()" function was added
35
+ to fix it -- but only for role == "assistant". add_tool_result had the
36
+ identical root cause (same "pass the whole entry dict" pattern) and was
37
+ never touched, because the fix lived one layer above where the actual
38
+ divergence was produced. Fixing what gets WRITTEN, once, here, closes
39
+ this for every role without a matching special case at every point
40
+ it's read back. See tests/test_memory_write_through.py for the
41
+ regression tests (now covering the tool row's `content`, not just its
42
+ `role`/`tool_call_id`, which is exactly the assertion that was missing
43
+ before).
44
+ """
45
+
46
+ from typing import Optional
47
+ from uuid import UUID
48
+
49
+ from storage import (
50
+ THREAD_KIND_CHAT,
51
+ create_thread,
52
+ get_session_history,
53
+ get_thread,
54
+ latest_checkpoint,
55
+ mark_turn_failed,
56
+ message_ids_from,
57
+ pinned_through,
58
+ save_message,
59
+ set_pinned,
60
+ turn_start_ids,
61
+ update_thread_extra,
62
+ )
63
+
64
+ # ROADMAP_v2 §21 (M8). How a compaction summary is introduced to the model.
65
+ #
66
+ # The neutral shape has three roles -- user, assistant, tool -- and no
67
+ # "system" among them, because the system prompt travels separately. A
68
+ # leading USER message is therefore the only form all three providers
69
+ # accept in the message list, and it is where a conversation naturally
70
+ # starts anyway.
71
+ #
72
+ # It is NEVER persisted as a MessageLog row. Doing so would put derived
73
+ # content in an append-only archive whose whole value is being original,
74
+ # and the next compaction would then summarize the previous summary as if
75
+ # it were something the user said -- chaining by accident, on a code path
76
+ # that means to re-derive.
77
+ SUMMARY_PREFIX = (
78
+ "[Summary of the earlier part of this conversation, compacted to save "
79
+ "context. The full history is preserved and unedited; this is what is "
80
+ "being shown to you now.]\n\n"
81
+ )
82
+
83
+
84
+ def _require_harness_record(harness) -> None:
85
+ if not isinstance(harness, dict) or not harness.get("kind"):
86
+ raise ValueError(
87
+ "a harness message needs its record -- a dict with a `kind` -- "
88
+ "or it is indistinguishable from something the person wrote")
89
+
90
+
91
+ def append_harness_row(thread_id: UUID, text: str, harness: dict) -> None:
92
+ """ROADMAP_v3 §49 (SS6). The same row `add_harness_message` writes, for a
93
+ thread no ConversationMemory in this process holds.
94
+
95
+ The quit path needs it: a session belongs to the thread that STARTED it,
96
+ which need not be the one on screen, and building a ConversationMemory
97
+ just to append one row would load and derive that thread's whole view to
98
+ throw it away. Like every writer here, it knows no SQL.
99
+ """
100
+ _require_harness_record(harness)
101
+ save_message(thread_id, role="user", content=text, harness=harness)
102
+
103
+
104
+ class ConversationMemory:
105
+ def __init__(self, thread_id: Optional[UUID] = None,
106
+ kind: str = THREAD_KIND_CHAT, *,
107
+ parent_thread_id: Optional[UUID] = None,
108
+ parent_call_id: Optional[str] = None,
109
+ agent_name: Optional[str] = None) -> None:
110
+ """`kind` (§27 T1) says what a NEW thread is, and is forwarded to
111
+ storage.create_thread and nowhere else.
112
+
113
+ This class does not learn what a kind means, store it, or read it
114
+ back. It owns active in-run state and must stay unaware of what
115
+ data exists -- that boundary is storage.py's. Ignored when
116
+ `thread_id` is given, because resuming does not reclassify: a
117
+ thread's kind was decided when it was created.
118
+
119
+ The three lineage arguments (§47) are forwarded the same way and
120
+ are ignored on a resume for the same reason: who spawned a thread
121
+ was settled when it was created, and reopening it settles nothing.
122
+ Keyword-only, so a caller that does not know it is creating a
123
+ child cannot say it is by filling a positional slot.
124
+ """
125
+ # Session-scoped billing (#4): zero on EVERY construction, fresh or
126
+ # resumed -- see record_billed for why this one is deliberately not
127
+ # extra_data like its neighbour.
128
+ self.billed_tokens = 0
129
+ # ROADMAP_v3 §81 (IM3). Images a tool attached for a call whose
130
+ # result has not been written yet, keyed by the call's id.
131
+ # add_tool_result takes them into the row it writes; nothing else
132
+ # reads this, and it is never persisted on its own.
133
+ self._pending_images: dict[str, list] = {}
134
+ if thread_id is None:
135
+ self.thread_id = create_thread(
136
+ kind=kind, parent_thread_id=parent_thread_id,
137
+ parent_call_id=parent_call_id, agent_name=agent_name)
138
+ self._messages: list[dict] = []
139
+ self._extra: dict = {}
140
+ self._kind = kind
141
+ else:
142
+ thread = get_thread(thread_id)
143
+ if thread is None:
144
+ raise ValueError(f"No conversation thread found with id {thread_id}")
145
+ self.thread_id = thread_id
146
+ self._extra = dict(thread.get("extra_data") or {})
147
+ self._messages = self._derived_view()
148
+ # The kind fallback (missing column on a database whose ALTER
149
+ # has not run) lives inside get_thread's copy now; resuming is
150
+ # not reclassifying, so the stored kind is simply reported.
151
+ self._kind = thread.get("kind") or THREAD_KIND_CHAT
152
+
153
+ @property
154
+ def kind(self) -> str:
155
+ """What this thread IS (§27 T1): "chat", "research_pass" or
156
+ "subagent".
157
+
158
+ A deliberate boundary bend, recorded rather than slipped in
159
+ (#43, batch 16). §27 says this class "learns nothing from kind" --
160
+ it forwards the kwarg at creation and ignores it on resume --
161
+ because kind SEMANTICS are storage's to store and the loop's to
162
+ decide. That still holds: nothing here branches on the value.
163
+ What changed is that the loop can now ASK, because batch 16 gave
164
+ it a decision to make from the answer -- which compaction mode a
165
+ re-entered thread runs under derives from what the thread IS
166
+ (research_pass and subagent threads get M6's backstop wherever
167
+ they are resumed), and the memory is the object holding the
168
+ thread. Reporting a stored fact is not interpreting it.
169
+ """
170
+ return self._kind
171
+
172
+ # -- the derived view (§21) --------------------------------------------
173
+
174
+ def _derived_view(self) -> list[dict]:
175
+ """What the agent sees this turn: a view computed over the archive,
176
+ not a second copy of it.
177
+
178
+ Three parts, in order:
179
+ 1. the latest checkpoint's summary, as a leading user message
180
+ 2. pinned rows from inside the span that summary covers (M9),
181
+ verbatim -- the summary skipped them, and one watermark
182
+ cannot say "everything through K except these"
183
+ 3. every archive row after the watermark, verbatim
184
+
185
+ A thread that has never been compacted has no checkpoint, so this
186
+ returns exactly what get_session_history() always returned. That
187
+ equivalence is what keeps every pre-§21 caller untouched until the
188
+ trigger first fires on a thread, and it is asserted directly.
189
+ """
190
+ checkpoint = latest_checkpoint(self.thread_id)
191
+ if checkpoint is None:
192
+ return get_session_history(self.thread_id)
193
+
194
+ watermark = checkpoint["covers_up_to_message_id"]
195
+ summary = {
196
+ "role": "user",
197
+ "content": f"{SUMMARY_PREFIX}{checkpoint['summary_text']}",
198
+ }
199
+ return (
200
+ [summary]
201
+ + pinned_through(self.thread_id, watermark)
202
+ + get_session_history(self.thread_id, after_message_id=watermark)
203
+ )
204
+
205
+ def apply_checkpoint(self) -> None:
206
+ """Rebuild the in-memory view after a compaction was recorded.
207
+
208
+ Deliberately re-reads rather than splicing the summary into the
209
+ list it already holds. The assembly rules (a leading summary,
210
+ pinned re-inclusion, the tail) then live in exactly one place, so
211
+ a mid-run compaction and a resumed thread cannot produce different
212
+ views of the same checkpoint -- which is the divergence a second
213
+ assembly path would eventually introduce.
214
+ """
215
+ self._messages = self._derived_view()
216
+
217
+ # -- measured context size (§21) ----------------------------------------
218
+
219
+ @property
220
+ def last_input_tokens(self) -> int:
221
+ """The provider's own count of everything it was last sent on this
222
+ thread, or 0 if it has never been measured.
223
+
224
+ This is what §21's compaction trigger reads. It is exact and it
225
+ comes from the provider rather than from a local tokenizer that
226
+ would have to be right per model -- but it is retrospective, which
227
+ is why the trigger is evaluated after a response and acts before
228
+ the next call.
229
+
230
+ PERSISTED, via the same extra_data write-through goal mode uses.
231
+ §21 says a resumed thread needs a rough character estimate before
232
+ its first call because nothing has been measured yet; carrying the
233
+ last real measurement across the process boundary means the
234
+ estimate is only ever needed for a thread written before this
235
+ section existed.
236
+ """
237
+ return int(self._extra.get("last_input_tokens") or 0)
238
+
239
+ def record_input_tokens(self, tokens: int) -> None:
240
+ """Store the measurement from the call that just returned. Ignores
241
+ a zero, which is what D21's raise exists to prevent reaching here
242
+ and what a provider reporting no usage would otherwise write over
243
+ a real earlier number."""
244
+ if tokens:
245
+ self.set_extra("last_input_tokens", int(tokens))
246
+
247
+ # -- session billing (#4) -------------------------------------------------
248
+
249
+ def record_billed(self, tokens: int) -> None:
250
+ """Accumulate this thread's billed input+output for the SESSION.
251
+
252
+ Deliberately NOT persisted, unlike last_input_tokens beside it:
253
+ the scope the TUI's usage line shows is THREAD-SINCE-RESUME. A
254
+ lifetime total would need a schema column to mean anything (a
255
+ resumed thread would silently restart at zero anyway, reading as
256
+ a reset), and #4's figure exists for relative reasoning about
257
+ this session, not as a ledger.
258
+ """
259
+ self.billed_tokens += int(tokens or 0)
260
+
261
+ def completed_turns(self) -> int:
262
+ """How many turns of this thread are already finished (§21 M5/M11).
263
+
264
+ COUNTED FROM THE ARCHIVE, never from `self._messages`. M8 makes a
265
+ checkpoint summary a `role: "user"` message, so a compacted view
266
+ holds one user message that is not a turn -- and this number is
267
+ compared against archive-space turn indices in
268
+ compaction.compactable_span(). Counting the view was §21a's shipped
269
+ defect: the floor came out several turns too low, the computed
270
+ watermark went BACKWARDS, and the second automatic compaction of a
271
+ thread un-compacted it. See tests/test_storage_e2e.py.
272
+
273
+ Lives here rather than in core/loop.py because this is the file
274
+ that owns the thread's persistence -- the loop imports no storage
275
+ at all, and should not start.
276
+
277
+ Called before the first model call of a turn, and every wrapper
278
+ adds the user message first, so the turn about to run is the last
279
+ entry and is excluded. That off-by-one IS the structural floor:
280
+ everything from this index on is the current turn and can never be
281
+ folded.
282
+
283
+ Archive-space also makes the value STABLE for a whole turn. The
284
+ archive never changes under compaction, so a count taken at the top
285
+ of a turn stays correct after a mid-turn compaction rebuilds the
286
+ view; the view-based version silently went stale at exactly that
287
+ point.
288
+ """
289
+ return len(turn_start_ids(self.thread_id)) - 1
290
+
291
+ # -- pinning (§21) ------------------------------------------------------
292
+
293
+ def pin_last(self, turns: int = 1) -> int:
294
+ """Mark the last `turns` turns compaction-exempt. Returns how many
295
+ rows were newly pinned.
296
+
297
+ ORDINAL, not an id. §21 keeps MessageLog.id out of the neutral
298
+ shape -- the model never sees one and so cannot name one -- and
299
+ adding it would put a persistence concept into the provider-neutral
300
+ contract the whole memory design rests on, with three translation
301
+ branches then needing to strip it before the wire. "Pin the last N
302
+ turns" needs no id and is how the tool would actually be used.
303
+
304
+ A turn starts at a user message and runs to just before the next
305
+ one, so pinning a turn keeps the question together with its answer
306
+ and any tool results between them. Asking for more turns than the
307
+ thread has pins the whole thread rather than erroring: the intent
308
+ ("keep this conversation") is unambiguous and there is nothing to
309
+ clarify.
310
+ """
311
+ if turns < 1:
312
+ return 0
313
+ starts = turn_start_ids(self.thread_id)
314
+ if not starts:
315
+ return 0
316
+ first = starts[max(0, len(starts) - turns)]
317
+ return set_pinned(message_ids_from(self.thread_id, first))
318
+
319
+ def unpin_last(self, turns: int = 1) -> int:
320
+ """Release the protection on the last `turns` turns (#89). The
321
+ mirror of pin_last, and what D26's "reversible" premise needed to
322
+ become true: set_pinned's pinned=False branch was written since
323
+ §21 and unreachable until this existed.
324
+
325
+ Same ordinal interface, same clamp (asking for more turns than
326
+ the thread has releases everything -- the intent is unambiguous),
327
+ same return contract (rows actually flipped, so "nothing was
328
+ pinned" reads as zero)."""
329
+ if turns < 1:
330
+ return 0
331
+ starts = turn_start_ids(self.thread_id)
332
+ if not starts:
333
+ return 0
334
+ first = starts[max(0, len(starts) - turns)]
335
+ return set_pinned(message_ids_from(self.thread_id, first),
336
+ pinned=False)
337
+
338
+ @property
339
+ def extra(self) -> dict:
340
+ """Snapshot of the thread's extra_data (goal mode §18). A copy --
341
+ mutate via set_extra(), which persists write-through like the
342
+ message methods."""
343
+ return dict(self._extra)
344
+
345
+ def set_extra(self, key: str, value) -> None:
346
+ """Write one extra_data key (value=None deletes), persisting
347
+ immediately so a goal set from the TUI is visible to the next
348
+ run in any shell."""
349
+ update_thread_extra(self.thread_id, key, value)
350
+ if value is None:
351
+ self._extra.pop(key, None)
352
+ else:
353
+ self._extra[key] = value
354
+
355
+ def mark_turn_failed(self, error: str) -> None:
356
+ """Batch 91. Record that the run this thread's newest user message
357
+ started FAILED, and why -- write-through, like every add_* method.
358
+
359
+ On the user row because that is what names the turn; see
360
+ storage.MessageLog.error for why it is a column and not a row. The
361
+ in-memory entry carries it too, for §44's reason on `thinking`: a
362
+ live turn and a resumed one must hand every reader the same shape,
363
+ and a resumed thread gets the key from storage._to_neutral. Nothing
364
+ that builds a request reads it -- _messages_for_provider takes a user
365
+ row's `content` alone.
366
+ """
367
+ mark_turn_failed(self.thread_id, error)
368
+ for entry in reversed(self._messages):
369
+ if entry.get("role") == "user":
370
+ entry["error"] = error
371
+ break
372
+
373
+ def add_user_message(self, text: str) -> None:
374
+ entry = {"role": "user", "content": text}
375
+ self._messages.append(entry)
376
+ save_message(self.thread_id, role="user", content=text)
377
+
378
+ def add_harness_message(self, text: str, harness: dict) -> None:
379
+ """ROADMAP_v3 §49 (SS5). A message the HARNESS wrote to start a turn --
380
+ a background session's result, or the record that one was killed --
381
+ write-through like every add_* method.
382
+
383
+ A user-role entry, because that is the only shape the model can be
384
+ handed between turns (storage.MessageLog.harness says why), marked so
385
+ replay and the compactor can tell it from the person. Nothing that
386
+ builds a request reads the mark, and it never carries a tool_call_id.
387
+
388
+ The record is REQUIRED. A harness message without one would be a
389
+ user message the harness wrote and nobody could tell apart -- the
390
+ exact confusion M8 names for the compaction summary.
391
+ """
392
+ _require_harness_record(harness)
393
+ entry = {"role": "user", "content": text, "harness": harness}
394
+ self._messages.append(entry)
395
+ save_message(self.thread_id, role="user", content=text,
396
+ harness=harness)
397
+
398
+ def add_assistant_message(self, response) -> None:
399
+ """
400
+ Takes the NORMALIZED ModelResponse from
401
+ core.client.call_model_stream() directly -- not a raw SDK object.
402
+ This is what eliminated the previous Anthropic-only
403
+ `response.raw.content` reach-through in loop.py; all three
404
+ provider branches produce the same ModelResponse shape (.text,
405
+ .tool_calls), so this method never needs to know which provider
406
+ generated it.
407
+
408
+ This used to name `call_model()`, which had no production caller
409
+ (#38) -- so the one sentence telling a reader where this argument
410
+ comes from named a function that never produced one. `raw` went
411
+ with it: it was set only there, so it was None on every live path
412
+ while its comment said it held the SDK response.
413
+ """
414
+ tool_calls = [
415
+ {"id": tc.id, "name": tc.name, "input": tc.input}
416
+ for tc in response.tool_calls
417
+ ]
418
+ entry = {"role": "assistant", "text": response.text, "tool_calls": tool_calls}
419
+ # §44: the turn's reasoning, in the provider's own blocks. Read
420
+ # off the response rather than accumulated from the display
421
+ # deltas, because an Anthropic block carries a signature the
422
+ # prose does not -- see ModelResponse.thinking. Attached to the
423
+ # in-memory entry as well as persisted, so a live turn and a
424
+ # resumed one hand _messages_for_provider the same shape; that is
425
+ # the whole point, and it is why this is one field and not two
426
+ # code paths.
427
+ thinking = getattr(response, "thinking", None)
428
+ if thinking:
429
+ entry["thinking"] = thinking
430
+ self._messages.append(entry)
431
+ # Persist only the assistant-specific payload, not the whole
432
+ # entry -- "role" is already carried by save_message's own
433
+ # `role=` argument, and duplicating it inside `content` is
434
+ # exactly the redundancy that caused the resume-shape bug. The
435
+ # reasoning has its own column for the same reason: it is not
436
+ # part of what the model said.
437
+ save_message(
438
+ self.thread_id,
439
+ role="assistant",
440
+ content={"text": response.text, "tool_calls": tool_calls},
441
+ thinking=thinking,
442
+ )
443
+
444
+ def attach_image(self, tool_call_id: str, media_type: str,
445
+ data: str) -> None:
446
+ """ROADMAP_v3 §81 (IM3). Hold an image for the result of
447
+ `tool_call_id`, to be written WITH that result's row.
448
+
449
+ The bytes reach the thread this way rather than inside the tool's
450
+ result dict, and that is the design: a result passes the output
451
+ redactor, the tool_result event and every shell's transcript, all of
452
+ which read text. A base64 payload run through the secret patterns can
453
+ be altered -- and an altered image is one the provider rejects -- so
454
+ the exemption from redaction is structural: the bytes never enter
455
+ that path. Only first-party code holding this object can attach; an
456
+ MCP server's result is a dict and cannot.
457
+
458
+ `data` is base64. Nothing is written here: a call that never gets a
459
+ result row never gets its image either.
460
+ """
461
+ self._pending_images.setdefault(tool_call_id, []).append(
462
+ {"media_type": media_type, "data": data})
463
+
464
+ def add_tool_result(self, tool_call_id: str, result: dict) -> None:
465
+ result_text = str(result)
466
+ entry = {"role": "tool", "tool_call_id": tool_call_id, "content": result_text}
467
+ # §81 (IM2). Taken whatever the result says: the tool attaches only
468
+ # on its success path, immediately before returning, so an image
469
+ # pending here belongs to this result.
470
+ images = self._pending_images.pop(tool_call_id, None)
471
+ if images:
472
+ entry["images"] = images
473
+ self._messages.append(entry)
474
+ # Persist only the result string. tool_call_id already has its
475
+ # own column (passed below) -- it doesn't need to also be
476
+ # duplicated inside content. The images have their own column too.
477
+ save_message(self.thread_id, role="tool", content=result_text,
478
+ tool_call_id=tool_call_id, images=images)
479
+
480
+ @property
481
+ def messages(self) -> list[dict]:
482
+ return self._messages