@yahyahammad/venastine 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/.env.example +14 -0
  2. package/LICENSE +201 -0
  3. package/NOTICE +14 -0
  4. package/PRIVACY.md +66 -0
  5. package/README.md +878 -0
  6. package/SECURITY.md +88 -0
  7. package/THIRD_PARTY_NOTICES.md +100 -0
  8. package/agents/builtin/compactor.md +75 -0
  9. package/agents/builtin/explore.md +77 -0
  10. package/agents/builtin/grill-me.md +27 -0
  11. package/agents/builtin/initializer.md +94 -0
  12. package/agents/builtin/pipeline-reviewer.md +100 -0
  13. package/agents/builtin/plan.md +88 -0
  14. package/agents/builtin/review.md +74 -0
  15. package/agents/manager.py +195 -0
  16. package/agents/subagent_tool.py +215 -0
  17. package/agents/tui_commands.py +136 -0
  18. package/bin/venastine.mjs +382 -0
  19. package/config.py +1198 -0
  20. package/core/__init__.py +0 -0
  21. package/core/approval.py +115 -0
  22. package/core/client.py +1548 -0
  23. package/core/compaction.py +959 -0
  24. package/core/config_loader.py +1563 -0
  25. package/core/events.py +61 -0
  26. package/core/interaction.py +328 -0
  27. package/core/loop.py +1353 -0
  28. package/core/memory.py +370 -0
  29. package/core/model_windows.py +175 -0
  30. package/core/pipeline_models.py +176 -0
  31. package/core/reasoning/__init__.py +0 -0
  32. package/core/reasoning/authorization.py +130 -0
  33. package/core/reasoning/base.py +219 -0
  34. package/core/reasoning/confidence_scoring.py +468 -0
  35. package/core/reasoning/events.py +149 -0
  36. package/core/reasoning/json_retry.py +219 -0
  37. package/core/reasoning/orchestrator.py +1382 -0
  38. package/core/reasoning/output_writer.py +286 -0
  39. package/core/reasoning/payload_validation.py +398 -0
  40. package/core/reasoning/pipeline_storage.py +361 -0
  41. package/core/reasoning/review.py +615 -0
  42. package/core/reasoning/scholar.py +434 -0
  43. package/core/reasoning/source_corpus.py +327 -0
  44. package/core/reasoning/source_scoring.py +855 -0
  45. package/core/replay.py +167 -0
  46. package/core/session.py +153 -0
  47. package/core/workspace_trust.py +207 -0
  48. package/credentials.py +180 -0
  49. package/database.py +252 -0
  50. package/env_secrets.py +77 -0
  51. package/json_store.py +123 -0
  52. package/logging_setup.py +185 -0
  53. package/main.py +1959 -0
  54. package/mcp_client/__init__.py +15 -0
  55. package/mcp_client/client.py +499 -0
  56. package/mcp_client/config.py +379 -0
  57. package/mcp_client/registration.py +182 -0
  58. package/memories/manager.py +132 -0
  59. package/memories/tui_commands.py +232 -0
  60. package/package.json +73 -0
  61. package/project_init/config_files.py +278 -0
  62. package/project_init/doc_sets.py +330 -0
  63. package/project_init/generator.py +564 -0
  64. package/project_init/manifest.py +406 -0
  65. package/project_init/tui_commands.py +144 -0
  66. package/prompts/__init__.py +0 -0
  67. package/prompts/annotate.md +14 -0
  68. package/prompts/assumption_audit.md +27 -0
  69. package/prompts/claim_extraction.md +34 -0
  70. package/prompts/completeness.md +18 -0
  71. package/prompts/confidence_tiers.md +23 -0
  72. package/prompts/critic_pass.md +24 -0
  73. package/prompts/final_synthesis.md +13 -0
  74. package/prompts/initial_generation.md +14 -0
  75. package/prompts/preliminary_plan.md +23 -0
  76. package/prompts/revalidate.md +25 -0
  77. package/prompts/revise.md +15 -0
  78. package/prompts/source_grounding.md +66 -0
  79. package/prompts/system_prompts.py +262 -0
  80. package/prompts/universal_system_prompt +7 -0
  81. package/prompts/untrusted_content +1 -0
  82. package/providers.json.example +92 -0
  83. package/pyproject.toml +100 -0
  84. package/requirements.txt +72 -0
  85. package/safety/__init__.py +0 -0
  86. package/safety/policy_enforcement.py +682 -0
  87. package/security/__init__.py +0 -0
  88. package/security/capability.py +177 -0
  89. package/security/permissions.py +190 -0
  90. package/security/posture.py +302 -0
  91. package/security/protected_paths.py +309 -0
  92. package/security/sandbox.py +985 -0
  93. package/skills/builtin/crypto/cryptography-verification.md +66 -0
  94. package/skills/builtin/math/formal-specification.md +67 -0
  95. package/skills/builtin/math/numerical-methods.md +66 -0
  96. package/skills/builtin/math/proof-writing.md +61 -0
  97. package/skills/builtin/research/experiment-design.md +62 -0
  98. package/skills/builtin/research/literature-review.md +65 -0
  99. package/skills/builtin/research/reproducibility.md +55 -0
  100. package/skills/builtin/research/source-evaluation.md +70 -0
  101. package/skills/builtin/research/statistical-inference.md +63 -0
  102. package/skills/builtin/research/technical-writing.md +59 -0
  103. package/skills/builtin/security/cybersecurity-research.md +63 -0
  104. package/skills/builtin/software/code-review.md +61 -0
  105. package/skills/builtin/software/debugging.md +61 -0
  106. package/skills/builtin/software/test-design.md +67 -0
  107. package/skills/manager.py +138 -0
  108. package/skills/tui_commands.py +111 -0
  109. package/storage.py +1005 -0
  110. package/tools/base.py +305 -0
  111. package/tools/builtin/__init__.py +0 -0
  112. package/tools/builtin/_math_common.py +377 -0
  113. package/tools/builtin/_net_common.py +61 -0
  114. package/tools/builtin/arxiv.py +223 -0
  115. package/tools/builtin/ask_user.py +213 -0
  116. package/tools/builtin/discrete_math.py +134 -0
  117. package/tools/builtin/fetch_url.py +153 -0
  118. package/tools/builtin/file_ops.py +412 -0
  119. package/tools/builtin/geometry.py +148 -0
  120. package/tools/builtin/get_time.py +20 -0
  121. package/tools/builtin/linear_algebra.py +155 -0
  122. package/tools/builtin/load_skill.py +66 -0
  123. package/tools/builtin/logic.py +96 -0
  124. package/tools/builtin/pin.py +193 -0
  125. package/tools/builtin/probability_stats.py +228 -0
  126. package/tools/builtin/project_docs.py +405 -0
  127. package/tools/builtin/remember.py +158 -0
  128. package/tools/builtin/shell.py +461 -0
  129. package/tools/builtin/symbolic_math.py +119 -0
  130. package/tools/builtin/todo.py +147 -0
  131. package/tools/builtin/web_search.py +151 -0
  132. package/tools/context.py +136 -0
  133. package/tools/isolation.py +314 -0
  134. package/tools/registry.py +897 -0
  135. package/tui/app.py +3423 -0
  136. package/tui/app.tcss +458 -0
  137. package/tui/commands.py +61 -0
  138. package/tui/diffs.py +265 -0
  139. package/tui/preferences.py +182 -0
  140. package/tui/ravens.py +146 -0
  141. package/tui/screens.py +943 -0
  142. package/tui/themes.py +366 -0
  143. package/tui/widgets.py +1239 -0
package/storage.py ADDED
@@ -0,0 +1,1005 @@
1
+ import json
2
+ import logging
3
+ from datetime import datetime, timezone
4
+ from typing import List, Dict, Any, Optional
5
+ from uuid import UUID, uuid4
6
+
7
+ from sqlmodel import Field, SQLModel, JSON, Session, select
8
+ from sqlalchemy import func as sa_func # real sqlalchemy; sqlmodel is faked in tests, this is not
9
+
10
+ from database import engine # your SQLAlchemy engine, assumed to exist here
11
+
12
+ logger = logging.getLogger(__name__)
13
+
14
+
15
+ class ConversationThread(SQLModel, table=True):
16
+ id: UUID = Field(default_factory=uuid4, primary_key=True)
17
+ created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
18
+ extra_data: Dict[str, Any] = Field(default_factory=dict, sa_type=JSON)
19
+ # ROADMAP_v2 §27 (T1). What this thread IS: "chat" (a conversation a
20
+ # human had), "research_pass" (one pass of the ten-pass pipeline) or
21
+ # "subagent" (a spawned agent, the §20 reviewer, or the compactor).
22
+ #
23
+ # A COLUMN, not an extra_data key. That field exists and would need no
24
+ # migration, but it is an opaque JSON blob -- filtering on it means
25
+ # loading every row and testing in Python, and list_threads() is the
26
+ # one query that must stay cheap on a database gaining ~10 rows per
27
+ # research run.
28
+ #
29
+ # NOT indexed, deliberately. ensure_columns() ALTERs in a column and
30
+ # does not build indexes, and create_all() will not add one to a table
31
+ # that already exists -- so index=True here would produce an index on
32
+ # fresh databases and none on migrated ones. A divergence that shows
33
+ # up only as a performance difference is worse than no index at all;
34
+ # the WHERE clause is what T1 was buying, not the index.
35
+ kind: str = Field(default="chat")
36
+ # (#32) When this thread was last written to -- ANY archived row,
37
+ # user, assistant or tool alike. The picker orders by this rather
38
+ # than by creation, because "most recent first" should surface the
39
+ # conversation you were just in, not the one you merely opened last.
40
+ #
41
+ # Nullable deliberately, twice over. Semantically: NULL means "never
42
+ # received a message", and the reader falls back to created_at --
43
+ # which is also what every row predating this column reads as, since
44
+ # the additive migration never backfills (M7). Mechanically: the
45
+ # honest default is per-row Python (a timestamp), which has no SQL
46
+ # literal, so the migrator could not have added it NOT NULL anyway
47
+ # (#26's WARNING path). save_message is the single writer of
48
+ # MessageLog rows in production and stamps this in the same session
49
+ # it writes the message, so the two cannot drift.
50
+ last_activity_at: Optional[datetime] = Field(default=None)
51
+
52
+
53
+ class MessageLog(SQLModel, table=True):
54
+ id: UUID = Field(default_factory=uuid4, primary_key=True)
55
+ thread_id: UUID = Field(foreign_key="conversationthread.id", index=True)
56
+ role: str = Field(description="system, user, assistant, or tool")
57
+ content: str # always JSON-encoded, regardless of the original type
58
+ name: Optional[str] = None
59
+ tool_call_id: Optional[str] = None
60
+ created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
61
+ # ROADMAP_v2 §21. Compaction-exempt within this thread (D26's pin
62
+ # tool). This is the first field this project has ever added to a
63
+ # table that already exists on disk, which is why database.py grew
64
+ # ensure_columns() -- create_all() would have left every existing
65
+ # app.db without it and failed at the next SELECT (M7).
66
+ pinned: bool = Field(default=False)
67
+ # §44. The turn's reasoning, JSON-encoded, or None -- assistant rows
68
+ # only, and only where the provider returned any.
69
+ #
70
+ # A SEPARATE COLUMN rather than a key inside `content`, because these
71
+ # are different KINDS of thing with different rules. `content` is what
72
+ # the model said; this is the provider's own signed blocks, which go
73
+ # back on the wire unchanged or not at all, and which every reader
74
+ # that measures the conversation (the compactor's input, /summary's
75
+ # distillation) has to be able to leave out without parsing around
76
+ # them. Nullable and additive, so database.ensure_columns() adds it to
77
+ # an existing database exactly as it added `pinned` -- see the note on
78
+ # that field's migration in _ordered_rows.
79
+ thinking: Optional[str] = None
80
+
81
+
82
+ class CompactionCheckpoint(SQLModel, table=True):
83
+ """One compaction of one thread (ROADMAP_v2 §21).
84
+
85
+ THE ARCHIVE IS NEVER EDITED. Compaction only ever ADDS a row here;
86
+ MessageLog is append-only and stays complete forever, which is what
87
+ §21 AC1 is about and what makes an aggressive compaction trigger safe --
88
+ nothing is ever destroyed, only hidden from the next call. See
89
+ archive_history() for the read that proves it.
90
+
91
+ `summary_text` represents every message in this thread through
92
+ `covers_up_to_message_id` EXCEPT the pinned ones (§21 AC2), which is why
93
+ the derived view has to re-include them separately (M9) -- a single
94
+ watermark cannot express "everything up to K except rows 5 and 6".
95
+
96
+ `strategy` records which way this summary was produced: "rederive"
97
+ (summarized the original messages, so exactly one summarization step
98
+ sits between an original message and what the model sees) or "chain"
99
+ (summarized the previous summary plus what followed it, bounded cost,
100
+ compounding loss). M2 makes rederive the default and chain the
101
+ configurable/fallback case; storing it per row means a thread's
102
+ history of the two is auditable rather than inferred from settings
103
+ that may since have changed.
104
+ """
105
+
106
+ id: UUID = Field(default_factory=uuid4, primary_key=True)
107
+ thread_id: UUID = Field(foreign_key="conversationthread.id", index=True)
108
+ summary_text: str
109
+ covers_up_to_message_id: UUID
110
+ strategy: str = Field(default="rederive")
111
+ created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
112
+
113
+
114
+ class ThreadSummary(SQLModel, table=True):
115
+ """A DESCRIPTION of one whole thread (ROADMAP_v2 §21c, M18).
116
+
117
+ NOT A COMPACTION, and the distinction is the entire reason this is a
118
+ separate table rather than a row in CompactionCheckpoint with a
119
+ different `strategy`. The two carry almost the same columns and mean
120
+ opposite things:
121
+
122
+ * a CompactionCheckpoint CHANGES WHAT THE MODEL SEES. `latest_checkpoint`
123
+ resolves by timestamp and feeds ConversationMemory._derived_view(), so
124
+ writing one is an edit to the live conversation.
125
+ * a ThreadSummary changes nothing. It is read by `/summary` to show a
126
+ person, and by `/ref` to inject into a DIFFERENT thread. The thread it
127
+ describes is untouched.
128
+
129
+ So a summary row in the checkpoint table would silently compact the
130
+ thread it was only meant to describe -- and it would win, because it
131
+ would be the newest row. That is the failure this table exists to make
132
+ impossible rather than to warn about.
133
+
134
+ `covers_up_to_message_id` is the last archive row the summary accounts
135
+ for, which is what makes staleness answerable: storage.advances() already
136
+ knows whether a thread has moved past a given position, so a `/ref` of an
137
+ unchanged thread costs no model call and a `/ref` of a grown one is
138
+ re-summarized. Same column name as the checkpoint's on purpose -- it means
139
+ the same thing there, and `advances()` serves both.
140
+ """
141
+
142
+ id: UUID = Field(default_factory=uuid4, primary_key=True)
143
+ thread_id: UUID = Field(foreign_key="conversationthread.id", index=True)
144
+ summary_text: str
145
+ covers_up_to_message_id: UUID
146
+ created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
147
+
148
+
149
+ class UserMemory(SQLModel, table=True):
150
+ """One durable fact, carried BETWEEN threads (ROADMAP_v2 §21b).
151
+
152
+ Unlike everything else in this file, a row here is not part of any one
153
+ conversation: `source_thread_id` records where it was learned, but the
154
+ row outlives that thread and is injected into later ones. That is the
155
+ whole point, and it is also why D26 gates `remember` and leaves `pin`
156
+ ungated -- a wrong pin costs some context budget inside a conversation
157
+ the user is watching, while a wrong memory silently shapes
158
+ conversations that have not started yet.
159
+
160
+ SCOPE IS TWO COLUMNS, not one (M12). §21's schema block carries only
161
+ `scope`, but D25 says a project-scoped memory is "keyed to the same
162
+ resolved project path the workspace-trust store uses" -- and with no
163
+ path recorded, `"project"` cannot tell two projects apart, which makes
164
+ §21 AC6 untestable rather than merely unenforced. `project_path` is the
165
+ realpath for `scope="project"` and None for `scope="global"`, taken
166
+ from config_loader.get_project_path() so "which project" means one
167
+ thing across trust, config and memory.
168
+
169
+ `category` is free text the model may set for its own grouping. It is
170
+ deliberately not an enum: the harness has no basis to decide what
171
+ kinds of fact are worth distinguishing, and a wrong enum would push
172
+ every real memory into "other".
173
+ """
174
+
175
+ id: UUID = Field(default_factory=uuid4, primary_key=True)
176
+ content: str
177
+ category: Optional[str] = None
178
+ # D25: "project" (default) or "global". Narrow by default because the
179
+ # two error directions are not symmetric -- a memory that should have
180
+ # been project-scoped but landed globally contaminates every later
181
+ # conversation with a stale fact about a codebase you may not be
182
+ # working in, and presents as the model simply being confident.
183
+ scope: str = Field(default="project", index=True)
184
+ project_path: Optional[str] = Field(default=None, index=True)
185
+ source_thread_id: UUID = Field(foreign_key="conversationthread.id")
186
+ created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
187
+
188
+
189
+ #: ROADMAP_v2 §27 (T1). The three things a thread can be. Not an Enum:
190
+ #: the column is a plain string so an unmigrated row reads back as data
191
+ #: rather than raising, and config.py's "plain values only" rule means a
192
+ #: vocabulary shared by storage and its callers belongs beside the table
193
+ #: that stores it.
194
+ THREAD_KIND_CHAT = "chat"
195
+ THREAD_KIND_RESEARCH_PASS = "research_pass"
196
+ THREAD_KIND_SUBAGENT = "subagent"
197
+
198
+
199
+ def create_thread(kind: str = THREAD_KIND_CHAT) -> UUID:
200
+ """Starts a brand-new conversation thread and returns its id.
201
+
202
+ `kind` defaults to "chat" (§27 AC1), so every existing caller keeps
203
+ creating conversations and only the paths that know they are creating
204
+ something else pass anything here.
205
+ """
206
+ with Session(engine) as session:
207
+ thread = ConversationThread(kind=kind)
208
+ session.add(thread)
209
+ session.commit()
210
+ session.refresh(thread)
211
+ return thread.id
212
+
213
+
214
+ def get_thread(thread_id: UUID) -> Optional[dict]:
215
+ """The thread row as a plain column dict, or None if unknown (#31).
216
+
217
+ Used to confirm a thread_id is real before resuming it, and by
218
+ ConversationMemory to read extra_data and kind on resume. A PLAIN
219
+ DICT, like every other public read here: returning a detached ORM
220
+ instance worked only because nothing had expired it -- what a
221
+ detached instance hands back depends on what happened to be loaded,
222
+ which is exactly the question _ordered_rows' docstring says copying
223
+ columns removes.
224
+
225
+ The kind fallback lives HERE rather than at the caller: a row read
226
+ from a database whose ALTER has not run yet has no attribute at all,
227
+ and the caller should not have to know that.
228
+ """
229
+ with Session(engine) as session:
230
+ thread = session.get(ConversationThread, thread_id)
231
+ if thread is None:
232
+ return None
233
+ return {
234
+ "id": thread.id,
235
+ "created_at": thread.created_at,
236
+ "extra_data": dict(thread.extra_data or {}),
237
+ "kind": getattr(thread, "kind", None) or THREAD_KIND_CHAT,
238
+ "last_activity_at": getattr(thread, "last_activity_at", None),
239
+ }
240
+
241
+
242
+ def get_thread_extra(thread_id: UUID) -> Dict[str, Any]:
243
+ """Copy of the thread's extra_data JSON column (goal mode §18, todo
244
+ list §23). Raises ValueError for an unknown thread, same contract as
245
+ the other accessors."""
246
+ with Session(engine) as session:
247
+ thread = session.get(ConversationThread, thread_id)
248
+ if thread is None:
249
+ raise ValueError(f"No conversation thread found with id {thread_id}")
250
+ return dict(thread.extra_data or {})
251
+
252
+
253
+ def update_thread_extra(thread_id: UUID, key: str, value: Any) -> None:
254
+ """Set one key of the thread's extra_data; value=None deletes it.
255
+ Reassigns a fresh dict rather than mutating in place so SQLModel sees
256
+ the JSON column change."""
257
+ with Session(engine) as session:
258
+ thread = session.get(ConversationThread, thread_id)
259
+ if thread is None:
260
+ raise ValueError(f"No conversation thread found with id {thread_id}")
261
+ extra = dict(thread.extra_data or {})
262
+ if value is None:
263
+ extra.pop(key, None)
264
+ else:
265
+ extra[key] = value
266
+ thread.extra_data = extra
267
+ session.add(thread)
268
+ session.commit()
269
+
270
+
271
+ def save_message(
272
+ thread_id: UUID,
273
+ role: str,
274
+ content: Any,
275
+ name: Optional[str] = None,
276
+ tool_call_id: Optional[str] = None,
277
+ thinking: Optional[dict] = None,
278
+ ) -> None:
279
+ """Append one archived row and stamp the thread's activity (#32).
280
+
281
+ THE single production writer of MessageLog rows, which is what makes
282
+ it the right place to maintain `last_activity_at`: the stamp rides in
283
+ the same session and commit as the insert, so a message can never be
284
+ persisted without its thread moving to the top of the picker. Any
285
+ role counts (Q8b) -- a thread whose last event was tool output is
286
+ still the one you were just in. An unknown thread_id writes the
287
+ orphan message exactly as before; there is simply no row to stamp.
288
+ """
289
+ with Session(engine) as session:
290
+ new_message = MessageLog(
291
+ thread_id=thread_id,
292
+ role=role,
293
+ content=json.dumps(content),
294
+ name=name,
295
+ tool_call_id=tool_call_id,
296
+ # Encoded separately from `content` and stored as NULL when
297
+ # absent, so "this turn did no reasoning" costs no bytes and
298
+ # reads back as None rather than as an empty structure.
299
+ thinking=json.dumps(thinking) if thinking else None,
300
+ )
301
+ session.add(new_message)
302
+ thread = session.get(ConversationThread, thread_id)
303
+ if thread is not None:
304
+ thread.last_activity_at = datetime.now(timezone.utc)
305
+ session.add(thread)
306
+ session.commit()
307
+
308
+
309
+ def list_threads(
310
+ kind: Optional[str] = THREAD_KIND_CHAT, limit: Optional[int] = None,
311
+ ) -> List[dict]:
312
+ """Conversation threads, most recently ACTIVE first (#32).
313
+
314
+ Each entry: ``{"id": UUID, "created_at": datetime, "kind": str,
315
+ "preview": str}``. Used by the CLI / TUI layer for thread browsing —
316
+ core/memory.py does NOT call this.
317
+
318
+ `limit` caps the result AFTER ordering (#30/#32 together). The default
319
+ of None is uncached and changes nothing for existing callers -- a
320
+ caller that wants everything still gets everything. The TUI picker
321
+ passes a cap because a modal listing thousands of conversations is its
322
+ own usability defect; with activity ordering the cap costs the least,
323
+ since the threads most likely to be wanted are exactly the ones that
324
+ survive it. Anything older stays reachable by id (/resume, --thread).
325
+
326
+ The order key is `last_activity_at`, falling back to `created_at`
327
+ when it is NULL -- never-received-a-message threads, and every row
328
+ that predates the column (the additive migration never backfills).
329
+ §27 made the picker correct by filtering and identifiable by preview;
330
+ ordering is the third axis: resume yesterday's long thread, work in
331
+ it an hour, and it stays pinned at the top rather than sinking below
332
+ three threads opened and abandoned this morning.
333
+
334
+ ROADMAP_v2 §27 AC2: CONVERSATIONS ONLY by default. One research run
335
+ creates ~10 pass threads plus up to 4 retry-round threads plus the
336
+ reviewer's, and every automatic compaction creates one more — so the
337
+ unfiltered list was mostly machinery, which is what made the picker
338
+ unusable. `kind=None` returns everything, for a caller that genuinely
339
+ wants the machinery (see §27's note on reaching a run's pass threads).
340
+
341
+ `preview` is the thread's first user message, truncated (§27's picker
342
+ decision). A row of bare uuid + timestamp is filterable but still not
343
+ recognisable, and identifying a conversation was the actual complaint.
344
+ It costs ONE extra query for the whole list rather than one per row --
345
+ and ONE ROW per thread, not one row per user message (#30).
346
+ """
347
+ with Session(engine) as session:
348
+ statement = select(ConversationThread)
349
+ if kind is not None:
350
+ statement = statement.where(ConversationThread.kind == kind)
351
+ statement = statement.order_by(
352
+ sa_func.coalesce(
353
+ ConversationThread.last_activity_at,
354
+ ConversationThread.created_at,
355
+ ).desc(),
356
+ ConversationThread.created_at.desc(),
357
+ )
358
+ if limit is not None:
359
+ statement = statement.limit(limit)
360
+ threads = session.exec(statement).all()
361
+ previews = _first_user_messages(session, [t.id for t in threads])
362
+ return [
363
+ {
364
+ "id": t.id,
365
+ "created_at": t.created_at,
366
+ # getattr, not t.kind: a row read back from a database
367
+ # whose ALTER has not run yet has no attribute at all, and
368
+ # a picker that raises is worse than one showing "chat".
369
+ "kind": getattr(t, "kind", None) or THREAD_KIND_CHAT,
370
+ "preview": previews.get(t.id, ""),
371
+ }
372
+ for t in threads
373
+ ]
374
+
375
+
376
+ #: How much of a thread's first user message list_threads() carries. Long
377
+ #: enough to tell two conversations apart, short enough for one row.
378
+ _PREVIEW_CHARS = 70
379
+
380
+
381
+ def thread_preview(thread_id: UUID) -> str:
382
+ """The picker's preview for ONE thread (#171).
383
+
384
+ The same first-user-message text `list_threads()` computes -- read
385
+ from the ARCHIVE, so a compacted thread is labelled by what the user
386
+ actually said, never by §21a's synthesized summary (the reason T3
387
+ makes replay do the same) -- through the same one-row window query,
388
+ so a caller with a single id pays one row too.
389
+
390
+ A PUBLIC wrapper rather than a list_threads() scan because the CLI's
391
+ --ref takes ANY thread id (#171): the picker list is chat-kind-
392
+ filtered and capped at 200, and a pass thread or an older
393
+ conversation is exactly the thing worth referencing by name. Unknown
394
+ id -> "", which the reader renders as "(no preview)".
395
+ """
396
+ with Session(engine) as session:
397
+ return _first_user_messages(session, [thread_id]).get(thread_id, "")
398
+
399
+
400
+ def _first_user_messages(session, thread_ids: List[UUID]) -> Dict[UUID, str]:
401
+ """thread id -> truncated first user message, for the given threads.
402
+
403
+ ONE query for the whole list, walked oldest-first so the first row seen
404
+ per thread is the earliest one. Reads the ARCHIVE (every MessageLog row
405
+ of the thread) rather than the derived view, for the same reason §27's
406
+ T3 makes replay do it: on a compacted thread the view begins with the
407
+ synthesized summary, and previewing THAT would label harness-generated
408
+ text as something the user said.
409
+
410
+ The "one query" claim is now true of ROWS too (#30): the first-user
411
+ message is picked by `ROW_NUMBER() OVER (PARTITION BY thread_id ...)`,
412
+ so SQLite materializes one user message per thread instead of loading
413
+ every user message of every thread into ORM objects and discarding all
414
+ but one in Python -- measured at 40x over-read on a realistic database
415
+ (~98% of the picker's cost). A plain LIMIT-per-thread loop would need
416
+ one query per thread; the window function keeps the count at one.
417
+ """
418
+ if not thread_ids:
419
+ return {}
420
+ ranked = (
421
+ select(
422
+ MessageLog.thread_id.label("thread_id"),
423
+ MessageLog.content.label("content"),
424
+ sa_func.row_number().over(
425
+ partition_by=MessageLog.thread_id,
426
+ order_by=(MessageLog.created_at.asc(), MessageLog.id.asc()),
427
+ ).label("rn"),
428
+ )
429
+ .where(MessageLog.role == "user")
430
+ .where(MessageLog.thread_id.in_(thread_ids))
431
+ .cte("first_user_messages_ranked")
432
+ )
433
+ rows = session.exec(
434
+ select(ranked.c.thread_id, ranked.c.content)
435
+ .where(ranked.c.rn == 1)
436
+ ).all()
437
+ out: Dict[UUID, str] = {}
438
+ for thread_id, raw in rows:
439
+ try:
440
+ text = json.loads(raw)
441
+ except (TypeError, ValueError):
442
+ text = raw
443
+ text = " ".join(str(text).split())
444
+ out[thread_id] = (
445
+ text[:_PREVIEW_CHARS] + "…" if len(text) > _PREVIEW_CHARS else text
446
+ )
447
+ return out
448
+
449
+
450
+ def _decode_thinking(raw):
451
+ """A stored thinking record, or None for anything unusable.
452
+
453
+ Degrades rather than raising, like every other read on this path: the
454
+ column is a convenience for replay and for echoing blocks back to the
455
+ model that wrote them, and a row whose JSON cannot be parsed must cost
456
+ that convenience rather than the ability to resume the thread at all.
457
+ """
458
+ if not raw:
459
+ return None
460
+ try:
461
+ value = json.loads(raw)
462
+ except (TypeError, ValueError):
463
+ return None
464
+ return value if isinstance(value, dict) and value.get("blocks") else None
465
+
466
+
467
+ def _to_neutral(msg: dict) -> dict:
468
+ """Reconstructs the exact neutral shape core/memory.py's add_* methods
469
+ write, per msg.role -- this is the counterpart to memory.py only ever
470
+ persisting the role-specific payload (see that file's docstring for
471
+ why there's no separate normalization step on the read side either).
472
+
473
+ ONE reconstruction point. Every read below is a filter over the same
474
+ rows fed through this function, so a change to the neutral shape
475
+ cannot reach one read and miss another -- which is the same reason
476
+ memory.py fixes shapes on the write side rather than per-role at each
477
+ reader. tests/conftest.py's FakeStorage mirrors this function; if it
478
+ changes, that changes identically.
479
+ """
480
+ decoded = json.loads(msg["content"])
481
+ role = msg["role"]
482
+
483
+ if role == "assistant":
484
+ # decoded is {"text": ..., "tool_calls": [...]} -- exactly
485
+ # what add_assistant_message persisted, nothing nested.
486
+ payload = {
487
+ "role": "assistant",
488
+ "text": decoded.get("text", ""),
489
+ "tool_calls": decoded.get("tool_calls", []),
490
+ }
491
+ # §44, from its own column rather than from `content`. Present
492
+ # ONLY when the row carried reasoning, so every message this
493
+ # project has ever written reconstructs to exactly the shape it
494
+ # did before -- which is what keeps _messages_for_provider,
495
+ # compactable_span and the fake's mirror unchanged for them.
496
+ thinking = _decode_thinking(msg.get("thinking"))
497
+ if thinking is not None:
498
+ payload["thinking"] = thinking
499
+ elif role == "tool":
500
+ # decoded is the plain result string add_tool_result persisted.
501
+ payload = {"role": "tool", "tool_call_id": msg["tool_call_id"], "content": decoded}
502
+ else:
503
+ # user (and any future plain-content role): decoded is
504
+ # already the right value for "content".
505
+ payload = {"role": role, "content": decoded}
506
+ if msg["tool_call_id"]:
507
+ payload["tool_call_id"] = msg["tool_call_id"]
508
+
509
+ if msg["name"]:
510
+ payload["name"] = msg["name"]
511
+
512
+ return payload
513
+
514
+
515
+ def _ordered_rows(thread_id: UUID) -> List[dict]:
516
+ """Every row of a thread as plain column values, oldest first.
517
+
518
+ Ordered by (created_at, id) rather than created_at alone. Timestamps
519
+ have microsecond resolution so ties are unlikely, but "unlikely" is
520
+ not an ordering guarantee, and §21's watermark arithmetic slices this
521
+ list by position -- an unstable sort there would move the fold
522
+ boundary between two reads of the same thread.
523
+
524
+ Plain dicts rather than ORM instances because every caller below
525
+ reads these after the Session has closed, and what a detached
526
+ instance will still hand back depends on what happened to be loaded.
527
+ Copying six columns removes the question.
528
+
529
+ `pinned` is coerced to bool: a database migrated by
530
+ database.ensure_columns() gets the column with a DEFAULT, but a row
531
+ written by an older build through some other path could still read
532
+ back NULL, and None is not the same thing as "not pinned" to a
533
+ caller doing `if row["pinned"]`.
534
+ """
535
+ with Session(engine) as session:
536
+ statement = (
537
+ select(MessageLog)
538
+ .where(MessageLog.thread_id == thread_id)
539
+ .order_by(MessageLog.created_at.asc(), MessageLog.id.asc())
540
+ )
541
+ return [
542
+ {
543
+ "id": m.id, "role": m.role, "content": m.content,
544
+ "name": m.name, "tool_call_id": m.tool_call_id,
545
+ "pinned": bool(m.pinned),
546
+ # Not coerced, unlike `pinned`: NULL is the ordinary state
547
+ # of this column and _decode_thinking answers None for it.
548
+ "thinking": m.thinking,
549
+ }
550
+ for m in session.exec(statement).all()
551
+ ]
552
+
553
+
554
+ def _split_at(rows: List[dict], message_id: Optional[UUID]) -> int:
555
+ """Index just past `message_id` in `rows`, or 0 when it isn't there.
556
+
557
+ Resolving a watermark by POSITION in the already-loaded, already-ordered
558
+ list rather than by a `(created_at, id) > (t, i)` SQL predicate. The
559
+ comparison a tuple-ordered watermark needs is awkward to express
560
+ portably over a UUID column, and a thread is at most a few thousand
561
+ rows that this function's callers are loading anyway. Simplicity wins
562
+ over a query that has to be right about UUID collation.
563
+
564
+ A watermark that names a row this thread does not have returns 0 --
565
+ i.e. nothing is treated as covered. That is the safe direction: the
566
+ caller shows the whole thread rather than silently hiding a prefix
567
+ on the strength of an id it could not find.
568
+ """
569
+ if message_id is None:
570
+ return 0
571
+ for index, row in enumerate(rows):
572
+ if row["id"] == message_id:
573
+ return index + 1
574
+ return 0
575
+
576
+
577
+ def get_session_history(
578
+ thread_id: UUID, after_message_id: Optional[UUID] = None,
579
+ ) -> List[dict]:
580
+ """The thread's messages in neutral shape, oldest first.
581
+
582
+ `after_message_id` (ROADMAP_v2 §21) loads only the UNCOMPACTED TAIL --
583
+ everything strictly after that row. The rows before it are not gone;
584
+ they are represented by a CompactionCheckpoint summary that
585
+ core/memory.py prepends, and they remain readable in full via
586
+ archive_history(). None (the default) returns the whole thread, which
587
+ is what every pre-§21 caller gets and why their behaviour is
588
+ unchanged.
589
+ """
590
+ rows = _ordered_rows(thread_id)
591
+ return [_to_neutral(m) for m in rows[_split_at(rows, after_message_id):]]
592
+
593
+
594
+ def archive_history(thread_id: UUID) -> List[dict]:
595
+ """Every message ever written to this thread, compaction or not.
596
+
597
+ This is ROADMAP_v2 §21 AC1 made greppable: compaction adds
598
+ CompactionCheckpoint rows and never edits or deletes a MessageLog
599
+ row, so this read is complete no matter how many times a thread has
600
+ been compacted. It is a distinct name rather than a comment on
601
+ get_session_history because the guarantee is what callers want to
602
+ find -- and because a future watermark default on that function
603
+ would silently take this one with it.
604
+ """
605
+ return get_session_history(thread_id)
606
+
607
+
608
+ def pinned_through(thread_id: UUID, message_id: Optional[UUID]) -> List[dict]:
609
+ """Pinned rows at or before the watermark, in neutral shape (§21 M9).
610
+
611
+ A summary covers a contiguous span, but §21 AC2 says a pinned message is
612
+ never summarized -- so pinned rows INSIDE the covered span have to
613
+ come back verbatim, and a single `covers_up_to_message_id` cannot say
614
+ "everything through K except these". The derived view is therefore
615
+ summary + these rows + tail.
616
+
617
+ Empty when nothing in the span is pinned, which is every thread until
618
+ someone calls pin().
619
+
620
+ THE PINNED SPAN IS MADE WHOLE HERE, which is M4's rule applied to the
621
+ second boundary. The derived view has two of them and only the
622
+ checkpoint watermark is turn-aligned: `pin` is dispatched from inside a
623
+ turn, so the set it writes always ends mid-turn. D20 has already
624
+ persisted the assistant message carrying the `tool_use` for `pin`
625
+ itself, and the answering `tool` row is written after dispatch returns
626
+ -- so it is never pinned. While the pin is still in the uncompacted
627
+ tail this is invisible, because the tail carries both rows. The first
628
+ compaction whose watermark passes the pinned region activates it: the
629
+ assistant row comes back with an unanswered `tool_use`, its result
630
+ having gone into the summary text instead, and a `tool_result` with no
631
+ preceding `tool_use` is an HTTP 400 on Anthropic -- on a thread that
632
+ was working a moment ago, with no visible connection to a `pin` several
633
+ turns earlier. Durable, too: the checkpoint is a persisted row, so
634
+ every later call and every resume rebuilds the same unsendable view.
635
+
636
+ Fixed at the PRODUCER, where a pinned span becomes view content.
637
+ Stopping pin_last at the last complete turn would change what "pin the
638
+ last N turns" means and leave any other non-turn-aligned pinned set
639
+ open; dropping unanswered tool_calls in _derived_view is consumer-side
640
+ -- this project's canonical bug shape -- and would hide from the model
641
+ a call it actually made.
642
+ """
643
+ rows = _ordered_rows(thread_id)
644
+ span = rows[:_split_at(rows, message_id)]
645
+ neutral = [_to_neutral(m) for m in span]
646
+
647
+ keep = [i for i, m in enumerate(span) if m["pinned"]]
648
+ # `have`, the set() below and the role check are all BACKSTOPS, and the
649
+ # measurement says so: each was deleted separately against the full
650
+ # suite and each survived. What actually keeps the result duplicate-free
651
+ # is the `not m["pinned"]` filter -- a pinned tool row is already in
652
+ # `keep` and can never be re-added -- and reaching any of the three
653
+ # needs two rows sharing one tool_call_id, or a non-tool row carrying
654
+ # one, neither of which has a producer: add_tool_result writes exactly
655
+ # one row per call, and add_user_message passes no tool_call_id.
656
+ #
657
+ # Kept rather than trimmed, and the reasoning recorded rather than
658
+ # asserted: the failure they guard against is a duplicate tool_result,
659
+ # which is the same wire error from the other direction as the one this
660
+ # function exists to prevent. That is the wrong place to trade a free
661
+ # backstop for a reachability argument.
662
+ have = {neutral[i].get("tool_call_id") for i in keep}
663
+ need = {call["id"] for i in keep
664
+ for call in (neutral[i].get("tool_calls") or [])
665
+ if call["id"] not in have}
666
+ keep += [i for i, m in enumerate(span)
667
+ if not m["pinned"] and m["role"] == "tool"
668
+ and neutral[i].get("tool_call_id") in need]
669
+
670
+ return [neutral[i] for i in sorted(set(keep))]
671
+
672
+
673
+ def history_through(
674
+ thread_id: UUID, message_id: Optional[UUID],
675
+ after_message_id: Optional[UUID] = None,
676
+ ) -> List[dict]:
677
+ """The span a compaction is about to summarize, in neutral shape.
678
+
679
+ Pinned rows are EXCLUDED (§21 AC2: a pinned message is never included in
680
+ what the compactor is asked to summarize). `after_message_id` is what
681
+ makes chaining possible -- M2's chain strategy summarizes only what
682
+ followed the previous checkpoint, while rederive passes None and
683
+ summarizes the whole span from the top.
684
+ """
685
+ rows = _ordered_rows(thread_id)
686
+ start = _split_at(rows, after_message_id)
687
+ end = _split_at(rows, message_id)
688
+ return [_to_neutral(m) for m in rows[start:end] if not m["pinned"]]
689
+
690
+
691
+ def turn_start_ids(thread_id: UUID) -> List[UUID]:
692
+ """Ids of the rows that BEGIN each turn, oldest first.
693
+
694
+ A turn starts at a user message: the wrapper adds one, then the loop
695
+ appends assistant and tool rows until it stops. §21's M4 needs this
696
+ because a fold boundary that lands anywhere else can separate an
697
+ assistant message from its tool results, and a tool_result with no
698
+ preceding tool_use is an HTTP 400 on Anthropic -- a hard wire error,
699
+ not a degraded answer. M5's "keep the last N turns" needs it too.
700
+ """
701
+ return [m["id"] for m in _ordered_rows(thread_id) if m["role"] == "user"]
702
+
703
+
704
+ def message_ids_from(thread_id: UUID, message_id: UUID) -> List[UUID]:
705
+ """Ids of `message_id` and everything after it. Resolves pin()'s
706
+ ordinal ("the last N turns") into the real row ids the pinned flag is
707
+ written against -- §21 keeps MessageLog.id an implementation detail
708
+ the model never sees, so the ordinal-to-id mapping has to happen
709
+ somewhere below the tool."""
710
+ rows = _ordered_rows(thread_id)
711
+ start = _split_at(rows, message_id)
712
+ if not start:
713
+ return []
714
+ return [message_id] + [m["id"] for m in rows[start:]]
715
+
716
+
717
+ def set_pinned(message_ids: List[UUID], pinned: bool = True) -> int:
718
+ """Flag rows compaction-exempt. Returns how many rows changed."""
719
+ if not message_ids:
720
+ return 0
721
+ changed = 0
722
+ with Session(engine) as session:
723
+ for message_id in message_ids:
724
+ row = session.get(MessageLog, message_id)
725
+ if row is None or row.pinned == pinned:
726
+ continue
727
+ row.pinned = pinned
728
+ session.add(row)
729
+ changed += 1
730
+ session.commit()
731
+ return changed
732
+
733
+
734
+ def _current_watermark(thread_id: UUID) -> Optional[UUID]:
735
+ """The live checkpoint's watermark, or None if never compacted."""
736
+ checkpoint = latest_checkpoint(thread_id)
737
+ return checkpoint["covers_up_to_message_id"] if checkpoint else None
738
+
739
+
740
+ def advances(thread_id: UUID, current: Optional[UUID], proposed: UUID) -> bool:
741
+ """Is `proposed` strictly later in this thread than `current`?
742
+
743
+ The ordering test behind §21's one real monotonic invariant: a
744
+ compaction watermark only ever moves FORWARD, and -- #27's half --
745
+ it must name a row THIS THREAD actually has. `current` of None
746
+ (no checkpoint yet) skips the ordering comparison but not the
747
+ containment one: a watermark we cannot place is one we must not
748
+ write, and "first compaction" is exactly when nothing else would
749
+ catch a foreign id.
750
+
751
+ A `proposed` this thread does not contain returns False -- the safe
752
+ direction, since a watermark we cannot place is one we must not write.
753
+ """
754
+ rows = _ordered_rows(thread_id)
755
+ if current is None:
756
+ return _split_at(rows, proposed) > 0
757
+ return _split_at(rows, proposed) > _split_at(rows, current) > 0
758
+
759
+
760
+ def latest_checkpoint(thread_id: UUID) -> Optional[dict]:
761
+ """The most recent compaction of this thread as a plain column dict,
762
+ or None if it has never been compacted (which is every thread until
763
+ the trigger first fires).
764
+
765
+ A dict rather than a detached ORM instance (#31): two of the three
766
+ callers read it after their Session has closed, and one of them is
767
+ the compaction path -- the one place in the project where a wrong
768
+ read rewrites a live conversation. Copying the columns removes the
769
+ question of what a detached instance still hands back.
770
+ """
771
+ with Session(engine) as session:
772
+ statement = (
773
+ select(CompactionCheckpoint)
774
+ .where(CompactionCheckpoint.thread_id == thread_id)
775
+ .order_by(CompactionCheckpoint.created_at.desc(),
776
+ CompactionCheckpoint.id.desc())
777
+ )
778
+ row = next(iter(session.exec(statement).all()), None)
779
+ if row is None:
780
+ return None
781
+ return {
782
+ "id": row.id,
783
+ "thread_id": row.thread_id,
784
+ "summary_text": row.summary_text,
785
+ "covers_up_to_message_id": row.covers_up_to_message_id,
786
+ "strategy": row.strategy,
787
+ "created_at": row.created_at,
788
+ }
789
+
790
+
791
+ def save_checkpoint(
792
+ thread_id: UUID, summary_text: str, covers_up_to_message_id: UUID,
793
+ strategy: str = "rederive",
794
+ ) -> UUID:
795
+ """Record one compaction. Adds a row; touches nothing else (§21 AC1).
796
+
797
+ REFUSES A WATERMARK THAT DOES NOT ADVANCE, returning the existing
798
+ checkpoint's id instead. `latest_checkpoint` resolves ties by
799
+ timestamp, so a backwards watermark written here does not merely fail
800
+ to help -- it becomes the live one, and the thread UN-COMPACTS: the
801
+ derived view jumps back to showing everything after the earlier
802
+ boundary. That is what §21a shipped, and the caller-side guard in
803
+ core/compaction.py is not enough on its own, because it only covers
804
+ the one caller that exists today.
805
+
806
+ Enforced rather than documented, for the same reason D24's declared-
807
+ permission check is: the invariant is cheap to state and its violation
808
+ is silent and durable.
809
+ """
810
+ if not advances(thread_id, _current_watermark(thread_id),
811
+ covers_up_to_message_id):
812
+ existing = latest_checkpoint(thread_id)
813
+ logger.warning(
814
+ "Refused a compaction checkpoint for thread %s whose watermark "
815
+ "does not advance on the existing one; keeping the existing "
816
+ "checkpoint. This would have made the thread larger, not "
817
+ "smaller.", thread_id)
818
+ return existing["id"] if existing is not None else None
819
+
820
+ with Session(engine) as session:
821
+ checkpoint = CompactionCheckpoint(
822
+ thread_id=thread_id,
823
+ summary_text=summary_text,
824
+ covers_up_to_message_id=covers_up_to_message_id,
825
+ strategy=strategy,
826
+ )
827
+ session.add(checkpoint)
828
+ session.commit()
829
+ session.refresh(checkpoint)
830
+ return checkpoint.id
831
+
832
+
833
+ # ---------------------------------------------------------------------------
834
+ # ---- Thread summaries (ROADMAP_v2 §21c) -----------------------------------
835
+ # ---------------------------------------------------------------------------
836
+
837
+ def last_message_id(thread_id: UUID) -> Optional[UUID]:
838
+ """The id of this thread's most recent archive row, or None if empty.
839
+
840
+ §21c needs a position meaning "everything, as of now", to store as a
841
+ summary's watermark and to compare against a stored one. Nothing existing
842
+ answers it: `message_ids_from` needs a start id, `turn_start_ids` returns
843
+ turn starts (and the last row is usually not one), and `_ordered_rows` is
844
+ private for the reasons its docstring gives.
845
+
846
+ Reads the ARCHIVE, like everything else here -- the ordering is
847
+ `(created_at, id)`, so this is the same last row `advances()` compares
848
+ positions within.
849
+ """
850
+ rows = _ordered_rows(thread_id)
851
+ return rows[-1]["id"] if rows else None
852
+
853
+
854
+ def latest_thread_summary(thread_id: UUID) -> Optional[dict]:
855
+ """The most recent summary of this thread as a plain column dict, or
856
+ None if never summarized (#31 -- same reasoning as latest_checkpoint).
857
+
858
+ Newest by `(created_at, id)`, matching `latest_checkpoint` -- but note
859
+ that the resemblance ends at the query. A stale row winning here shows
860
+ someone an out-of-date description, which `advances()` then detects and
861
+ corrects; a stale row winning in `latest_checkpoint` silently rewrites a
862
+ live conversation. See ThreadSummary's docstring.
863
+ """
864
+ with Session(engine) as session:
865
+ statement = (
866
+ select(ThreadSummary)
867
+ .where(ThreadSummary.thread_id == thread_id)
868
+ .order_by(ThreadSummary.created_at.desc(), ThreadSummary.id.desc())
869
+ )
870
+ row = next(iter(session.exec(statement).all()), None)
871
+ if row is None:
872
+ return None
873
+ return {
874
+ "id": row.id,
875
+ "thread_id": row.thread_id,
876
+ "summary_text": row.summary_text,
877
+ "covers_up_to_message_id": row.covers_up_to_message_id,
878
+ "created_at": row.created_at,
879
+ }
880
+
881
+
882
+ def save_thread_summary(
883
+ thread_id: UUID, summary_text: str, covers_up_to_message_id: UUID,
884
+ ) -> UUID:
885
+ """Record one summary of one thread. Adds a row; touches nothing else.
886
+
887
+ NO ADVANCE GUARD, deliberately, and this is the one place the difference
888
+ from `save_checkpoint` above is load-bearing rather than decorative. That
889
+ function REFUSES a watermark that does not advance, because a backwards
890
+ compaction watermark becomes the live one and un-compacts the thread. A
891
+ summary row changes what NO model sees: the worst a redundant or
892
+ backwards one can do is describe the thread as it was a moment ago, which
893
+ the next `advances()` check notices.
894
+
895
+ So the guard is absent because it would be cargo -- not because the
896
+ monotonic property does not matter here. Do not add one, and do not route
897
+ a summary through `save_checkpoint` to get one.
898
+ """
899
+ with Session(engine) as session:
900
+ summary = ThreadSummary(
901
+ thread_id=thread_id,
902
+ summary_text=summary_text,
903
+ covers_up_to_message_id=covers_up_to_message_id,
904
+ )
905
+ session.add(summary)
906
+ session.commit()
907
+ session.refresh(summary)
908
+ return summary.id
909
+
910
+
911
+ # ---------------------------------------------------------------------------
912
+ # ---- Durable memory (ROADMAP_v2 §21b) -------------------------------------
913
+ # ---------------------------------------------------------------------------
914
+
915
+ def save_memory(
916
+ content: str, source_thread_id: UUID, scope: str = "project",
917
+ category: Optional[str] = None, project_path: Optional[str] = None,
918
+ ) -> UUID:
919
+ """Record one durable fact. Returns its id.
920
+
921
+ `project_path` is required in substance for scope="project" and
922
+ meaningless for "global" -- the caller resolves it, because this file
923
+ knows nothing about config_loader or the trust store. memories/manager.py
924
+ is the one place that decides.
925
+ """
926
+ with Session(engine) as session:
927
+ row = UserMemory(
928
+ content=content,
929
+ category=category,
930
+ scope=scope,
931
+ project_path=project_path if scope == "project" else None,
932
+ source_thread_id=source_thread_id,
933
+ )
934
+ session.add(row)
935
+ session.commit()
936
+ session.refresh(row)
937
+ return row.id
938
+
939
+
940
+ def list_memories(
941
+ project_path: Optional[str] = None, scope: Optional[str] = None,
942
+ ) -> List[dict]:
943
+ """Memories visible from `project_path`, NEWEST FIRST.
944
+
945
+ Returns global memories plus the project ones recorded against exactly
946
+ this path -- which is §21 AC6: a memory written in one project does not
947
+ surface in a thread opened from another. `project_path=None` returns
948
+ global memories only, which is the honest answer when no project has
949
+ been resolved rather than a reason to show everything.
950
+
951
+ `scope` filters to one kind, for the management commands. Absent means
952
+ both, which is what prompt assembly wants.
953
+
954
+ Both predicates are pushed INTO THE QUERY (#28): `UserMemory` declares
955
+ indexes on `scope` and `project_path`, and this function used to load
956
+ every row and filter in Python -- the exact pattern T1 rejected, run
957
+ against two columns whose indexes had no reader. SQLite answers the
958
+ visibility rule below from those indexes now.
959
+
960
+ Plain dicts rather than ORM instances, for the same reason
961
+ _ordered_rows returns them: every caller reads these after the Session
962
+ has closed.
963
+ """
964
+ with Session(engine) as session:
965
+ statement = select(UserMemory)
966
+ if scope is not None:
967
+ statement = statement.where(UserMemory.scope == scope)
968
+ # Visibility: anything not project-scoped travels; a project memory
969
+ # only matches its own path. A NULL path therefore matches nothing
970
+ # except when the caller has no resolved project either -- same as
971
+ # the Python filter this replaced, edge included deliberately.
972
+ statement = statement.where(
973
+ (UserMemory.scope != "project")
974
+ | (UserMemory.project_path == project_path))
975
+ statement = statement.order_by(
976
+ UserMemory.created_at.desc(), UserMemory.id.desc())
977
+ return [
978
+ {
979
+ "id": row.id, "content": row.content,
980
+ "category": row.category, "scope": row.scope,
981
+ "project_path": row.project_path,
982
+ "source_thread_id": row.source_thread_id,
983
+ "created_at": row.created_at,
984
+ }
985
+ for row in session.exec(statement).all()
986
+ ]
987
+
988
+
989
+ def forget_memory(memory_id: UUID) -> bool:
990
+ """Delete one memory. True if a row was removed.
991
+
992
+ A REAL DELETE, unlike everything else here. MessageLog is append-only
993
+ because a conversation is a record of what happened; a UserMemory is a
994
+ standing assertion the harness keeps repeating, and "this is no longer
995
+ true" has no useful archived form. Returning False for an unknown id
996
+ rather than raising: the id came from a human reading a list, and a
997
+ typo deserves "no such memory", not a traceback.
998
+ """
999
+ with Session(engine) as session:
1000
+ row = session.get(UserMemory, memory_id)
1001
+ if row is None:
1002
+ return False
1003
+ session.delete(row)
1004
+ session.commit()
1005
+ return True