@yahyahammad/venastine 0.3.8 → 0.3.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/.env.example +19 -19
  2. package/CONFIG_ARCHITECTURE.md +2139 -2002
  3. package/LICENSE +201 -201
  4. package/README.md +1043 -1034
  5. package/agents/builtin/compactor.md +75 -75
  6. package/agents/builtin/explore.md +77 -77
  7. package/agents/builtin/pipeline-reviewer.md +100 -100
  8. package/agents/builtin/plan.md +106 -106
  9. package/agents/builtin/review.md +74 -74
  10. package/agents/manager.py +261 -261
  11. package/agents/subagent_tool.py +472 -472
  12. package/agents/tui_commands.py +238 -146
  13. package/bin/venastine.mjs +584 -528
  14. package/bin/venastine.py +73 -71
  15. package/cli_commands.py +848 -835
  16. package/config.yaml +64 -0
  17. package/config_edit.py +1301 -1301
  18. package/config_schema.py +987 -966
  19. package/config_update.py +749 -749
  20. package/core/agent_activity.py +256 -256
  21. package/core/ansi.py +452 -452
  22. package/core/approval.py +137 -137
  23. package/core/call_view.py +364 -364
  24. package/core/cancel.py +121 -121
  25. package/core/chatgpt_auth.py +747 -747
  26. package/core/claims_view.py +107 -107
  27. package/core/client.py +2756 -2715
  28. package/core/commands.py +464 -454
  29. package/core/compaction.py +1735 -1735
  30. package/core/config_loader.py +1982 -1982
  31. package/core/connect.py +686 -686
  32. package/core/events.py +123 -123
  33. package/core/file_mentions.py +280 -280
  34. package/core/interaction.py +411 -411
  35. package/core/loop.py +2566 -2566
  36. package/core/memory.py +482 -482
  37. package/core/pipeline_models.py +190 -190
  38. package/core/provider_errors.py +274 -234
  39. package/core/reasoning/authorization.py +130 -130
  40. package/core/reasoning/base.py +219 -219
  41. package/core/reasoning/confidence_scoring.py +484 -484
  42. package/core/reasoning/events.py +149 -149
  43. package/core/reasoning/orchestrator.py +1561 -1561
  44. package/core/reasoning/output_writer.py +345 -345
  45. package/core/reasoning/payload_validation.py +524 -524
  46. package/core/reasoning/pipeline_storage.py +427 -427
  47. package/core/reasoning/scholar.py +467 -458
  48. package/core/reasoning/source_corpus.py +433 -433
  49. package/core/reasoning/source_scoring.py +855 -855
  50. package/core/replay.py +315 -315
  51. package/core/shell_sessions.py +1553 -1553
  52. package/credentials.py +307 -307
  53. package/database.py +333 -333
  54. package/docs/PRIVACY.md +78 -78
  55. package/docs/SECURITY.md +114 -108
  56. package/docs/THIRD_PARTY_NOTICES.md +148 -148
  57. package/env_secrets.py +118 -118
  58. package/json_store.py +343 -343
  59. package/logging_setup.py +295 -295
  60. package/main.py +3433 -3433
  61. package/mcp_client/client.py +780 -733
  62. package/mcp_client/config.py +484 -484
  63. package/mcp_client/registration.py +343 -343
  64. package/memories/manager.py +147 -147
  65. package/memories/tui_commands.py +319 -239
  66. package/package.json +303 -303
  67. package/project_init/config_files.py +282 -282
  68. package/project_init/generator.py +728 -728
  69. package/project_init/manifest.py +418 -418
  70. package/project_init/tui_commands.py +147 -147
  71. package/prompts/critic_pass.md +24 -24
  72. package/prompts/revalidate.md +25 -25
  73. package/prompts/revise.md +17 -17
  74. package/prompts/source_grounding.md +66 -66
  75. package/prompts/system_prompts.py +316 -316
  76. package/providers.json.example +94 -94
  77. package/pyproject.toml +149 -149
  78. package/requirements.txt +314 -295
  79. package/safety/policy_enforcement.py +1082 -1082
  80. package/security/askpass.py +419 -419
  81. package/security/capability.py +207 -207
  82. package/security/confine.py +484 -484
  83. package/security/hosted.py +182 -182
  84. package/security/permissions.py +237 -237
  85. package/security/protected_paths.py +501 -501
  86. package/security/sandbox.py +4108 -4108
  87. package/skills/builtin/math/formal-specification.md +67 -67
  88. package/skills/builtin/math/numerical-methods.md +66 -66
  89. package/skills/builtin/research/experiment-design.md +62 -62
  90. package/skills/builtin/research/reproducibility.md +55 -55
  91. package/skills/builtin/research/source-evaluation.md +70 -70
  92. package/skills/builtin/research/statistical-inference.md +63 -63
  93. package/skills/builtin/research/technical-writing.md +59 -59
  94. package/skills/builtin/software/code-review.md +61 -61
  95. package/skills/builtin/software/debugging.md +61 -61
  96. package/skills/builtin/software/test-design.md +67 -67
  97. package/skills/tui_commands.py +110 -110
  98. package/storage.py +1338 -1338
  99. package/tools/base.py +465 -465
  100. package/tools/builtin/_math_common.py +667 -667
  101. package/tools/builtin/_net_common.py +1145 -267
  102. package/tools/builtin/_page_text.py +490 -0
  103. package/tools/builtin/_pdf_text.py +113 -0
  104. package/tools/builtin/arxiv.py +685 -681
  105. package/tools/builtin/fetch_url.py +639 -371
  106. package/tools/builtin/file_ops.py +720 -720
  107. package/tools/builtin/geometry.py +184 -184
  108. package/tools/builtin/load_skill.py +76 -76
  109. package/tools/builtin/pin.py +227 -227
  110. package/tools/builtin/probability_stats.py +224 -224
  111. package/tools/builtin/project_docs.py +464 -464
  112. package/tools/builtin/remember.py +175 -175
  113. package/tools/builtin/shell.py +980 -980
  114. package/tools/builtin/todo.py +149 -149
  115. package/tools/builtin/web_search.py +402 -328
  116. package/tools/context.py +159 -159
  117. package/tools/isolation.py +387 -387
  118. package/tools/registry.py +1252 -1242
  119. package/tui/app.py +8372 -8307
  120. package/tui/app.tcss +754 -739
  121. package/tui/commands.py +26 -26
  122. package/tui/diffs.py +286 -286
  123. package/tui/history.py +128 -128
  124. package/tui/hosted_driver.py +102 -102
  125. package/tui/markdown.py +1329 -1329
  126. package/tui/meters.py +364 -364
  127. package/tui/ravens.py +146 -146
  128. package/tui/screens.py +1255 -1197
  129. package/tui/themes.py +599 -599
  130. package/tui/widgets.py +3861 -3796
package/storage.py CHANGED
@@ -1,1338 +1,1338 @@
1
- import json
2
- import logging
3
- from datetime import UTC, datetime
4
- from typing import Any, Optional
5
- from uuid import UUID, uuid4
6
-
7
- from sqlalchemy import (
8
- func as sa_func, # real sqlalchemy; sqlmodel is faked in tests, this is not
9
- )
10
- from sqlmodel import JSON, Field, Session, SQLModel, select
11
-
12
- from database import engine # your SQLAlchemy engine, assumed to exist here
13
-
14
- logger = logging.getLogger(__name__)
15
-
16
-
17
- class ConversationThread(SQLModel, table=True):
18
- id: UUID = Field(default_factory=uuid4, primary_key=True)
19
- created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
20
- extra_data: dict[str, Any] = Field(default_factory=dict, sa_type=JSON)
21
- # ROADMAP_v2 §27 (T1). What this thread IS: "chat" (a conversation a
22
- # human had), "research_pass" (one pass of the ten-pass pipeline) or
23
- # "subagent" (a spawned agent, the §20 reviewer, or the compactor).
24
- #
25
- # A COLUMN, not an extra_data key. That field exists and would need no
26
- # migration, but it is an opaque JSON blob -- filtering on it means
27
- # loading every row and testing in Python, and list_threads() is the
28
- # one query that must stay cheap on a database gaining ~10 rows per
29
- # research run.
30
- #
31
- # NOT indexed, deliberately. ensure_columns() ALTERs in a column and
32
- # does not build indexes, and create_all() will not add one to a table
33
- # that already exists -- so index=True here would produce an index on
34
- # fresh databases and none on migrated ones. A divergence that shows
35
- # up only as a performance difference is worse than no index at all;
36
- # the WHERE clause is what T1 was buying, not the index.
37
- kind: str = Field(default="chat")
38
- # (#32) When this thread was last written to -- ANY archived row,
39
- # user, assistant or tool alike. The picker orders by this rather
40
- # than by creation, because "most recent first" should surface the
41
- # conversation you were just in, not the one you merely opened last.
42
- #
43
- # Nullable deliberately, twice over. Semantically: NULL means "never
44
- # received a message", and the reader falls back to created_at --
45
- # which is also what every row predating this column reads as, since
46
- # the additive migration never backfills (M7). Mechanically: the
47
- # honest default is per-row Python (a timestamp), which has no SQL
48
- # literal, so the migrator could not have added it NOT NULL anyway
49
- # (#26's WARNING path). save_message is the single writer of
50
- # MessageLog rows in production and stamps this in the same session
51
- # it writes the message, so the two cannot drift.
52
- last_activity_at: Optional[datetime] = Field(default=None)
53
- # ROADMAP_v2 §47. WHO spawned this thread, and at which call.
54
- #
55
- # Before these, the only trace of the edge was the child's uuid
56
- # stringified inside the parent's tool-result row -- a repr inside a
57
- # JSON blob, not a field, not queryable, and skipped by replay's T4
58
- # anyway. A sidebar row and a `▸ spawn_subagent` line could describe
59
- # a run nobody could then open.
60
- #
61
- # COLUMNS, not `extra_data`, for `kind`'s reason exactly (T1): the
62
- # question asked of them is "which threads are children of this
63
- # one", and answering that from an opaque JSON blob means loading
64
- # every row and testing in Python. Not indexed, also for T1's
65
- # reason -- ensure_columns() ALTERs a column in and builds no index,
66
- # and create_all() adds none to a table that already exists, so
67
- # index=True would mean an index on fresh databases and none on
68
- # migrated ones.
69
- #
70
- # Nullable and additive, so database.ensure_columns() adds them to an
71
- # existing app.db exactly as it added `pinned`, `kind`, `thinking`
72
- # and `last_activity_at`. NULL is the honest value for every thread
73
- # that predates them and for every one nothing spawned.
74
- parent_thread_id: Optional[UUID] = Field(default=None)
75
- # The parent's `tool_call_id` for the call that made this thread --
76
- # the SAME id MessageLog stores on the parent's side, which is what
77
- # lets one specific `▸ spawn_subagent` line find one specific child
78
- # when a turn spawned three. None for a run no tool call made: the
79
- # compactor, the reviewer, /init's initializer.
80
- parent_call_id: Optional[str] = Field(default=None)
81
- # Which agent ran, for a reader who has only the row. `kind` says
82
- # what a thread IS; this says who it was. A viewer opening a stored
83
- # thread cannot ask the span, because the span is long gone.
84
- agent_name: Optional[str] = Field(default=None)
85
-
86
-
87
- class MessageLog(SQLModel, table=True):
88
- id: UUID = Field(default_factory=uuid4, primary_key=True)
89
- thread_id: UUID = Field(foreign_key="conversationthread.id", index=True)
90
- role: str = Field(description="system, user, assistant, or tool")
91
- content: str # always JSON-encoded, regardless of the original type
92
- name: Optional[str] = None
93
- tool_call_id: Optional[str] = None
94
- created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
95
- # ROADMAP_v2 §21. Compaction-exempt within this thread (D26's pin
96
- # tool). This is the first field this project has ever added to a
97
- # table that already exists on disk, which is why database.py grew
98
- # ensure_columns() -- create_all() would have left every existing
99
- # app.db without it and failed at the next SELECT (M7).
100
- pinned: bool = Field(default=False)
101
- # §44. The turn's reasoning, JSON-encoded, or None -- assistant rows
102
- # only, and only where the provider returned any.
103
- #
104
- # A SEPARATE COLUMN rather than a key inside `content`, because these
105
- # are different KINDS of thing with different rules. `content` is what
106
- # the model said; this is the provider's own signed blocks, which go
107
- # back on the wire unchanged or not at all, and which every reader
108
- # that measures the conversation (the compactor's input, /summary's
109
- # distillation) has to be able to leave out without parsing around
110
- # them. Nullable and additive, so database.ensure_columns() adds it to
111
- # an existing database exactly as it added `pinned` -- see the note on
112
- # that field's migration in _ordered_rows.
113
- thinking: Optional[str] = None
114
- # Batch 91. Why the run this row STARTED failed, or None -- user rows
115
- # only, written by mark_turn_failed when an exception escapes the loop.
116
- #
117
- # A column on the user row rather than a row of its own, and that is
118
- # the whole design. A failed run leaves no assistant turn behind (D20
119
- # persists one only when a call returns), so without this a thread
120
- # whose first call failed read as a request nobody answered. An `error`
121
- # ROLE would have been a harness notice stored as conversation -- the
122
- # compactor's `_as_text` and /summary render every role, so it would
123
- # have been summarized into what the model later reads, which is M8's
124
- # mistake. A column is read by nothing that measures the conversation:
125
- # provider translation and the compactor take `content` alone. On the
126
- # user row because that is what says WHICH turn failed, which a
127
- # thread-level field could not for a chat. Nullable and additive, so
128
- # database.ensure_columns() adds it like `thinking`.
129
- error: Optional[str] = None
130
- # ROADMAP_v3 §49 (SS5). What wrote this row when the HARNESS did, as a
131
- # JSON object -- {"kind": "session_wake" | "session_killed_at_quit" |
132
- # "session_held", "sessions": [{"id", "call_id", "shape"}, ...]} -- or
133
- # None for everything a person or the model wrote.
134
- #
135
- # A column on a USER row, for batch 91's reason one field over. A
136
- # finished background session has to reach the model as a message that
137
- # starts a turn, and a tool result cannot carry it: every tool_use is
138
- # answered in the step that made it (D20, NA11), long before the session
139
- # ends. A user-role row is the one shape every provider accepts there,
140
- # and a compacted thread already sends one the person never wrote (M8's
141
- # summary). A `harness` ROLE would need translating and summarizing as
142
- # something new; a column is read by nothing that builds a request, so
143
- # the row goes out as the plain user message it is, while replay and
144
- # the compactor can still tell it was not the person.
145
- #
146
- # NEVER beside a tool_call_id. `pinned_through` treats any kept row's
147
- # tool_call_id as an answered call, so a wake row carrying its session's
148
- # start call id would hide the real tool_result -- the HTTP 400 M4
149
- # exists to prevent. The call id lives inside this JSON instead.
150
- harness: Optional[str] = None
151
- # ROADMAP_v3 §81 (IM2). The images a tool result carried, as a JSON list
152
- # of {"media_type", "data"} with `data` base64 -- tool rows only, and only
153
- # where `view_image` attached one. NULL everywhere else.
154
- #
155
- # A COLUMN rather than blocks inside `content`, for `thinking`'s reason:
156
- # `content` is read as a string by every consumer of the conversation --
157
- # replay, the transcript, /copy, the compactor, the summarizer -- and
158
- # only two readers have any use for the bytes, provider translation and
159
- # the compactor's size proxy. In the database rather than in a blob store
160
- # or as a path to the workspace file: a path would make a resumed thread
161
- # send whatever the file holds NOW, and a second store would be a second
162
- # place docs/PRIVACY.md's "wipe everything" has to name. Nullable and
163
- # additive, so database.ensure_columns() adds it like `thinking`.
164
- images: Optional[str] = None
165
-
166
-
167
- class CompactionCheckpoint(SQLModel, table=True):
168
- """One compaction of one thread (ROADMAP_v2 §21).
169
-
170
- THE ARCHIVE IS NEVER EDITED. Compaction only ever ADDS a row here;
171
- MessageLog is append-only and stays complete forever, which is what
172
- §21 AC1 is about and what makes an aggressive compaction trigger safe --
173
- nothing is ever destroyed, only hidden from the next call. See
174
- archive_history() for the read that proves it.
175
-
176
- `summary_text` represents every message in this thread through
177
- `covers_up_to_message_id` EXCEPT the pinned ones (§21 AC2), which is why
178
- the derived view has to re-include them separately (M9) -- a single
179
- watermark cannot express "everything up to K except rows 5 and 6".
180
-
181
- `strategy` records which way this summary was produced: "rederive"
182
- (summarized the original messages, so exactly one summarization step
183
- sits between an original message and what the model sees) or "chain"
184
- (summarized the previous summary plus what followed it, bounded cost,
185
- compounding loss). M2 makes rederive the default and chain the
186
- configurable/fallback case; storing it per row means a thread's
187
- history of the two is auditable rather than inferred from settings
188
- that may since have changed.
189
- """
190
-
191
- id: UUID = Field(default_factory=uuid4, primary_key=True)
192
- thread_id: UUID = Field(foreign_key="conversationthread.id", index=True)
193
- summary_text: str
194
- covers_up_to_message_id: UUID
195
- strategy: str = Field(default="rederive")
196
- created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
197
-
198
-
199
- class ThreadSummary(SQLModel, table=True):
200
- """A DESCRIPTION of one whole thread (ROADMAP_v2 §21c, M18).
201
-
202
- NOT A COMPACTION, and the distinction is the entire reason this is a
203
- separate table rather than a row in CompactionCheckpoint with a
204
- different `strategy`. The two carry almost the same columns and mean
205
- opposite things:
206
-
207
- * a CompactionCheckpoint CHANGES WHAT THE MODEL SEES. `latest_checkpoint`
208
- resolves by timestamp and feeds ConversationMemory._derived_view(), so
209
- writing one is an edit to the live conversation.
210
- * a ThreadSummary changes nothing. It is read by `/summary` to show a
211
- person, and by `/ref` to inject into a DIFFERENT thread. The thread it
212
- describes is untouched.
213
-
214
- So a summary row in the checkpoint table would silently compact the
215
- thread it was only meant to describe -- and it would win, because it
216
- would be the newest row. That is the failure this table exists to make
217
- impossible rather than to warn about.
218
-
219
- `covers_up_to_message_id` is the last archive row the summary accounts
220
- for, which is what makes staleness answerable: storage.advances() already
221
- knows whether a thread has moved past a given position, so a `/ref` of an
222
- unchanged thread costs no model call and a `/ref` of a grown one is
223
- re-summarized. Same column name as the checkpoint's on purpose -- it means
224
- the same thing there, and `advances()` serves both.
225
- """
226
-
227
- id: UUID = Field(default_factory=uuid4, primary_key=True)
228
- thread_id: UUID = Field(foreign_key="conversationthread.id", index=True)
229
- summary_text: str
230
- covers_up_to_message_id: UUID
231
- created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
232
-
233
-
234
- class UserMemory(SQLModel, table=True):
235
- """One durable fact, carried BETWEEN threads (ROADMAP_v2 §21b).
236
-
237
- Unlike everything else in this file, a row here is not part of any one
238
- conversation: `source_thread_id` records where it was learned, but the
239
- row outlives that thread and is injected into later ones. That is the
240
- whole point, and it is also why D26 gates `remember` and leaves `pin`
241
- ungated -- a wrong pin costs some context budget inside a conversation
242
- the user is watching, while a wrong memory silently shapes
243
- conversations that have not started yet.
244
-
245
- SCOPE IS TWO COLUMNS, not one (M12). §21's schema block carries only
246
- `scope`, but D25 says a project-scoped memory is "keyed to the same
247
- resolved project path the workspace-trust store uses" -- and with no
248
- path recorded, `"project"` cannot tell two projects apart, which makes
249
- §21 AC6 untestable rather than merely unenforced. `project_path` is the
250
- realpath for `scope="project"` and None for `scope="global"`, taken
251
- from config_loader.get_project_path() so "which project" means one
252
- thing across trust, config and memory.
253
-
254
- `category` is free text the model may set for its own grouping. It is
255
- deliberately not an enum: the harness has no basis to decide what
256
- kinds of fact are worth distinguishing, and a wrong enum would push
257
- every real memory into "other".
258
- """
259
-
260
- id: UUID = Field(default_factory=uuid4, primary_key=True)
261
- content: str
262
- category: Optional[str] = None
263
- # D25: "project" (default) or "global". Narrow by default because the
264
- # two error directions are not symmetric -- a memory that should have
265
- # been project-scoped but landed globally contaminates every later
266
- # conversation with a stale fact about a codebase you may not be
267
- # working in, and presents as the model simply being confident.
268
- scope: str = Field(default="project", index=True)
269
- project_path: Optional[str] = Field(default=None, index=True)
270
- source_thread_id: UUID = Field(foreign_key="conversationthread.id")
271
- created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
272
-
273
-
274
- #: ROADMAP_v2 §27 (T1). The three things a thread can be. Not an Enum:
275
- #: the column is a plain string so an unmigrated row reads back as data
276
- #: rather than raising, and config.py's "plain values only" rule means a
277
- #: vocabulary shared by storage and its callers belongs beside the table
278
- #: that stores it.
279
- THREAD_KIND_CHAT = "chat"
280
- THREAD_KIND_RESEARCH_PASS = "research_pass"
281
- THREAD_KIND_SUBAGENT = "subagent"
282
-
283
-
284
- def create_thread(kind: str = THREAD_KIND_CHAT, *,
285
- parent_thread_id: Optional[UUID] = None,
286
- parent_call_id: Optional[str] = None,
287
- agent_name: Optional[str] = None) -> UUID:
288
- """Starts a brand-new conversation thread and returns its id.
289
-
290
- `kind` defaults to "chat" (§27 AC1), so every existing caller keeps
291
- creating conversations and only the paths that know they are creating
292
- something else pass anything here.
293
-
294
- The three lineage arguments (§47) are keyword-only and all default to
295
- None, for the same reason: a caller that does not know it is creating
296
- a child is not one, and should not be able to say so by accident in a
297
- positional slot.
298
- """
299
- with Session(engine) as session:
300
- thread = ConversationThread(
301
- kind=kind, parent_thread_id=parent_thread_id,
302
- parent_call_id=parent_call_id, agent_name=agent_name)
303
- session.add(thread)
304
- session.commit()
305
- session.refresh(thread)
306
- return thread.id
307
-
308
-
309
- def get_thread(thread_id: UUID) -> Optional[dict]:
310
- """The thread row as a plain column dict, or None if unknown (#31).
311
-
312
- Used to confirm a thread_id is real before resuming it, and by
313
- ConversationMemory to read extra_data and kind on resume. A PLAIN
314
- DICT, like every other public read here: returning a detached ORM
315
- instance worked only because nothing had expired it -- what a
316
- detached instance hands back depends on what happened to be loaded,
317
- which is exactly the question _ordered_rows' docstring says copying
318
- columns removes.
319
-
320
- THE `getattr` FALLBACKS COVER A NULL, NOT A MISSING COLUMN, and this
321
- paragraph said the second thing until batch 75. It read: "a row read
322
- from a database whose ALTER has not run yet has no attribute at all".
323
- It cannot -- `session.get` SELECTs every mapped column, so a database
324
- missing one raises OperationalError before any attribute is read, and
325
- the class always has the attribute anyway. What these actually absorb
326
- is a NULL on a row written before the column existed (and a hand-built
327
- stand-in in a test), which is why each pairs the lookup with a default
328
- the caller can use. The migration case belongs to
329
- `database.ensure_columns()`, which runs at startup.
330
-
331
- `kind` IS THE EXCEPTION, and the sentence above was wrong about it
332
- until #257. ensure_columns has added `kind` with DEFAULT 'chat' since
333
- §27 shipped it, and SQLite reads every row older than that ALTER as
334
- the default -- so a migrated row arrives as 'chat' before any reader
335
- runs (the lineage columns and `last_activity_at` have no SQL default
336
- and really are NULL there). A NULL `kind` takes SQL written by hand,
337
- on a database first migrated between §27 and #26 (2026-08-06 to
338
- 08-22), where the column was added nullable. That is all the
339
- `or THREAD_KIND_CHAT` here, in child_threads and in list_threads
340
- absorbs; tests/test_storage_e2e.py builds that database and pins both
341
- the default and the fallback. list_threads filters in SQL, so its
342
- default picker never meets such a row at all.
343
- """
344
- with Session(engine) as session:
345
- thread = session.get(ConversationThread, thread_id)
346
- if thread is None:
347
- return None
348
- return {
349
- "id": thread.id,
350
- "created_at": thread.created_at,
351
- "extra_data": dict(thread.extra_data or {}),
352
- "kind": getattr(thread, "kind", None) or THREAD_KIND_CHAT,
353
- "last_activity_at": getattr(thread, "last_activity_at", None),
354
- # getattr for the kind fallback's reason, one column over --
355
- # see the docstring for what that reason actually is. NULL on
356
- # a row written before §47 means "no known parent", and a
357
- # viewer showing that is better than one that raises.
358
- "parent_thread_id": getattr(thread, "parent_thread_id", None),
359
- "parent_call_id": getattr(thread, "parent_call_id", None),
360
- "agent_name": getattr(thread, "agent_name", None),
361
- }
362
-
363
-
364
- def child_threads(parent_thread_id: UUID) -> list[dict]:
365
- """The threads spawned by this one, oldest first (§47).
366
-
367
- Each entry: ``{"id", "created_at", "kind", "parent_call_id",
368
- "agent_name"}``. Empty for a thread that spawned nothing, which is
369
- most of them.
370
-
371
- ONE QUERY, not one per spawn line. A resumed conversation resolves
372
- every `▸ spawn_subagent` line it draws, and asking per line would be
373
- a query per row of a screen.
374
-
375
- Oldest first because that is the order the calls happened in, so a
376
- reader comparing this against the transcript walks both the same
377
- way. `parent_call_id` is what pairs an entry to a specific line;
378
- ordering is for the human, not for the lookup.
379
-
380
- ORDERED BY `(created_at, id)`, which is `_ordered_rows`' rule and was
381
- missing here (batch 75). That rule reads: timestamps have microsecond
382
- resolution so ties are unlikely, but "unlikely" is not an ordering
383
- guarantee. This is the query where it stopped being merely unlikely --
384
- §47's NA9 makes three `create_thread` calls land from three threads at
385
- once, which is the one place in this file that can actually tie.
386
-
387
- NO `kind` FILTER, deliberately. Every child of a chat thread is
388
- machinery of some sort, and which kinds exist is §27's question
389
- rather than this one's -- a filter here would have to be widened
390
- again the moment a sixth source appears, which is how §27's own
391
- picker missed the compactor.
392
- """
393
- with Session(engine) as session:
394
- statement = (
395
- select(ConversationThread)
396
- .where(ConversationThread.parent_thread_id == parent_thread_id)
397
- .order_by(ConversationThread.created_at.asc(),
398
- ConversationThread.id.asc())
399
- )
400
- return [
401
- {
402
- "id": t.id,
403
- "created_at": t.created_at,
404
- # A hand-written NULL only (#257); see get_thread.
405
- "kind": getattr(t, "kind", None) or THREAD_KIND_CHAT,
406
- "parent_call_id": getattr(t, "parent_call_id", None),
407
- "agent_name": getattr(t, "agent_name", None),
408
- }
409
- for t in session.exec(statement).all()
410
- ]
411
-
412
-
413
- def get_thread_extra(thread_id: UUID) -> dict[str, Any]:
414
- """Copy of the thread's extra_data JSON column (goal mode §18, todo
415
- list §23). Raises ValueError for an unknown thread, same contract as
416
- the other accessors."""
417
- with Session(engine) as session:
418
- thread = session.get(ConversationThread, thread_id)
419
- if thread is None:
420
- raise ValueError(f"No conversation thread found with id {thread_id}")
421
- return dict(thread.extra_data or {})
422
-
423
-
424
- def update_thread_extra(thread_id: UUID, key: str, value: Any) -> None:
425
- """Set one key of the thread's extra_data; value=None deletes it.
426
- Reassigns a fresh dict rather than mutating in place so SQLModel sees
427
- the JSON column change."""
428
- with Session(engine) as session:
429
- thread = session.get(ConversationThread, thread_id)
430
- if thread is None:
431
- raise ValueError(f"No conversation thread found with id {thread_id}")
432
- extra = dict(thread.extra_data or {})
433
- if value is None:
434
- extra.pop(key, None)
435
- else:
436
- extra[key] = value
437
- thread.extra_data = extra
438
- session.add(thread)
439
- session.commit()
440
-
441
-
442
- def save_message(
443
- thread_id: UUID,
444
- role: str,
445
- content: Any,
446
- name: Optional[str] = None,
447
- tool_call_id: Optional[str] = None,
448
- thinking: Optional[dict] = None,
449
- harness: Optional[dict] = None,
450
- images: Optional[list] = None,
451
- ) -> None:
452
- """Append one archived row and stamp the thread's activity (#32).
453
-
454
- THE single production writer of MessageLog rows, which is what makes
455
- it the right place to maintain `last_activity_at`: the stamp rides in
456
- the same session and commit as the insert, so a message can never be
457
- persisted without its thread moving to the top of the picker. Any
458
- role counts (Q8b) -- a thread whose last event was tool output is
459
- still the one you were just in. An unknown thread_id writes the
460
- orphan message exactly as before; there is simply no row to stamp.
461
- """
462
- with Session(engine) as session:
463
- new_message = MessageLog(
464
- thread_id=thread_id,
465
- role=role,
466
- content=json.dumps(content),
467
- name=name,
468
- tool_call_id=tool_call_id,
469
- # Encoded separately from `content` and stored as NULL when
470
- # absent, so "this turn did no reasoning" costs no bytes and
471
- # reads back as None rather than as an empty structure.
472
- thinking=json.dumps(thinking) if thinking else None,
473
- # ROADMAP_v3 §49 (SS5), on `thinking`'s terms: NULL for every
474
- # row a person or the model wrote, which is nearly all of them.
475
- harness=json.dumps(harness) if harness else None,
476
- # ROADMAP_v3 §81 (IM2), on `thinking`'s terms again: NULL for
477
- # every row that attached no image.
478
- images=json.dumps(images) if images else None,
479
- )
480
- session.add(new_message)
481
- thread = session.get(ConversationThread, thread_id)
482
- if thread is not None:
483
- thread.last_activity_at = datetime.now(UTC)
484
- session.add(thread)
485
- session.commit()
486
-
487
-
488
- def list_threads(
489
- kind: Optional[str] = THREAD_KIND_CHAT, limit: Optional[int] = None,
490
- ) -> list[dict]:
491
- """Conversation threads, most recently ACTIVE first (#32).
492
-
493
- Each entry: ``{"id": UUID, "created_at": datetime, "kind": str,
494
- "preview": str}``. Used by the CLI / TUI layer for thread browsing —
495
- core/memory.py does NOT call this.
496
-
497
- `limit` caps the result AFTER ordering (#30/#32 together). The default
498
- of None is uncached and changes nothing for existing callers -- a
499
- caller that wants everything still gets everything. The TUI picker
500
- passes a cap because a modal listing thousands of conversations is its
501
- own usability defect; with activity ordering the cap costs the least,
502
- since the threads most likely to be wanted are exactly the ones that
503
- survive it. Anything older stays reachable by id (/resume, --thread).
504
-
505
- The order key is `last_activity_at`, falling back to `created_at`
506
- when it is NULL -- never-received-a-message threads, and every row
507
- that predates the column (the additive migration never backfills).
508
- §27 made the picker correct by filtering and identifiable by preview;
509
- ordering is the third axis: resume yesterday's long thread, work in
510
- it an hour, and it stays pinned at the top rather than sinking below
511
- three threads opened and abandoned this morning.
512
-
513
- ROADMAP_v2 §27 AC2: CONVERSATIONS ONLY by default. One research run
514
- creates ~10 pass threads plus up to 4 retry-round threads plus the
515
- reviewer's, and every automatic compaction creates one more — so the
516
- unfiltered list was mostly machinery, which is what made the picker
517
- unusable. `kind=None` returns everything, for a caller that genuinely
518
- wants the machinery (see §27's note on reaching a run's pass threads).
519
-
520
- `preview` is the thread's first user message, truncated (§27's picker
521
- decision). A row of bare uuid + timestamp is filterable but still not
522
- recognisable, and identifying a conversation was the actual complaint.
523
- It costs ONE extra query for the whole list rather than one per row --
524
- and ONE ROW per thread, not one row per user message (#30).
525
- """
526
- with Session(engine) as session:
527
- statement = select(ConversationThread)
528
- if kind is not None:
529
- statement = statement.where(ConversationThread.kind == kind)
530
- statement = statement.order_by(
531
- sa_func.coalesce(
532
- ConversationThread.last_activity_at,
533
- ConversationThread.created_at,
534
- ).desc(),
535
- ConversationThread.created_at.desc(),
536
- )
537
- if limit is not None:
538
- statement = statement.limit(limit)
539
- threads = session.exec(statement).all()
540
- previews = _first_user_messages(session, [t.id for t in threads])
541
- return [
542
- {
543
- "id": t.id,
544
- "created_at": t.created_at,
545
- # The fallback absorbs a hand-written NULL only: the
546
- # migration fills a row older than the column with 'chat'
547
- # (#257). get_thread's docstring has the whole of it. NOT a
548
- # database whose ALTER has not run: the mapped class always
549
- # has the attribute, and a database missing the column
550
- # raises OperationalError at the SELECT above, before any
551
- # attribute is read.
552
- "kind": getattr(t, "kind", None) or THREAD_KIND_CHAT,
553
- "preview": previews.get(t.id, ""),
554
- }
555
- for t in threads
556
- ]
557
-
558
-
559
- #: How much of a thread's first user message list_threads() carries. Long
560
- #: enough to tell two conversations apart, short enough for one row.
561
- _PREVIEW_CHARS = 70
562
-
563
-
564
- def thread_preview(thread_id: UUID) -> str:
565
- """The picker's preview for ONE thread (#171).
566
-
567
- The same first-user-message text `list_threads()` computes -- read
568
- from the ARCHIVE, so a compacted thread is labelled by what the user
569
- actually said, never by §21a's synthesized summary (the reason T3
570
- makes replay do the same) -- through the same one-row window query,
571
- so a caller with a single id pays one row too.
572
-
573
- A PUBLIC wrapper rather than a list_threads() scan because the CLI's
574
- --ref takes ANY thread id (#171): the picker list is chat-kind-
575
- filtered and capped at 200, and a pass thread or an older
576
- conversation is exactly the thing worth referencing by name. Unknown
577
- id -> "", which the reader renders as "(no preview)".
578
- """
579
- with Session(engine) as session:
580
- return _first_user_messages(session, [thread_id]).get(thread_id, "")
581
-
582
-
583
- def _first_user_messages(session, thread_ids: list[UUID]) -> dict[UUID, str]:
584
- """thread id -> truncated first user message, for the given threads.
585
-
586
- ONE query for the whole list, walked oldest-first so the first row seen
587
- per thread is the earliest one. Reads the ARCHIVE (every MessageLog row
588
- of the thread) rather than the derived view, for the same reason §27's
589
- T3 makes replay do it: on a compacted thread the view begins with the
590
- synthesized summary, and previewing THAT would label harness-generated
591
- text as something the user said.
592
-
593
- The "one query" claim is now true of ROWS too (#30): the first-user
594
- message is picked by `ROW_NUMBER() OVER (PARTITION BY thread_id ...)`,
595
- so SQLite materializes one user message per thread instead of loading
596
- every user message of every thread into ORM objects and discarding all
597
- but one in Python -- measured at 40x over-read on a realistic database
598
- (~98% of the picker's cost). A plain LIMIT-per-thread loop would need
599
- one query per thread; the window function keeps the count at one.
600
- """
601
- if not thread_ids:
602
- return {}
603
- ranked = (
604
- select(
605
- MessageLog.thread_id.label("thread_id"),
606
- MessageLog.content.label("content"),
607
- sa_func.row_number().over(
608
- partition_by=MessageLog.thread_id,
609
- order_by=(MessageLog.created_at.asc(), MessageLog.id.asc()),
610
- ).label("rn"),
611
- )
612
- .where(MessageLog.role == "user")
613
- .where(MessageLog.thread_id.in_(thread_ids))
614
- .cte("first_user_messages_ranked")
615
- )
616
- rows = session.exec(
617
- select(ranked.c.thread_id, ranked.c.content)
618
- .where(ranked.c.rn == 1)
619
- ).all()
620
- out: dict[UUID, str] = {}
621
- for thread_id, raw in rows:
622
- try:
623
- text = json.loads(raw)
624
- except (TypeError, ValueError):
625
- text = raw
626
- text = " ".join(str(text).split())
627
- out[thread_id] = (
628
- text[:_PREVIEW_CHARS] + "…" if len(text) > _PREVIEW_CHARS else text
629
- )
630
- return out
631
-
632
-
633
- def _decode_thinking(raw):
634
- """A stored thinking record, or None for anything unusable.
635
-
636
- Degrades rather than raising, like every other read on this path: the
637
- column is a convenience for replay and for echoing blocks back to the
638
- model that wrote them, and a row whose JSON cannot be parsed must cost
639
- that convenience rather than the ability to resume the thread at all.
640
- """
641
- if not raw:
642
- return None
643
- try:
644
- value = json.loads(raw)
645
- except (TypeError, ValueError):
646
- return None
647
- return value if isinstance(value, dict) and value.get("blocks") else None
648
-
649
-
650
- def _decode_harness(raw):
651
- """A stored harness record (ROADMAP_v3 §49, SS5), or None for anything
652
- unusable -- `_decode_thinking`'s rule for `_decode_thinking`'s reason: a
653
- mark that cannot be read costs the mark, never the thread."""
654
- if not raw:
655
- return None
656
- try:
657
- value = json.loads(raw)
658
- except (TypeError, ValueError):
659
- return None
660
- return value if isinstance(value, dict) and value.get("kind") else None
661
-
662
-
663
- def _decode_images(raw):
664
- """A tool row's stored images (ROADMAP_v3 §81, IM2), or None.
665
-
666
- `_decode_thinking`'s rule: an entry that cannot be read costs that
667
- entry, never the thread. An entry is usable when it names a media type
668
- and carries data, both as non-empty strings -- what translation needs to
669
- build a block, and nothing it would have to guess at.
670
- """
671
- if not raw:
672
- return None
673
- try:
674
- value = json.loads(raw)
675
- except (TypeError, ValueError):
676
- return None
677
- if not isinstance(value, list):
678
- return None
679
- usable = [entry for entry in value
680
- if isinstance(entry, dict)
681
- and isinstance(entry.get("media_type"), str)
682
- and entry["media_type"]
683
- and isinstance(entry.get("data"), str)
684
- and entry["data"]]
685
- return usable or None
686
-
687
-
688
- def _to_neutral(msg: dict) -> dict:
689
- """Reconstructs the exact neutral shape core/memory.py's add_* methods
690
- write, per msg.role -- this is the counterpart to memory.py only ever
691
- persisting the role-specific payload (see that file's docstring for
692
- why there's no separate normalization step on the read side either).
693
-
694
- ONE reconstruction point. Every read below is a filter over the same
695
- rows fed through this function, so a change to the neutral shape
696
- cannot reach one read and miss another -- which is the same reason
697
- memory.py fixes shapes on the write side rather than per-role at each
698
- reader. tests/conftest.py's FakeStorage mirrors this function; if it
699
- changes, that changes identically.
700
- """
701
- decoded = json.loads(msg["content"])
702
- role = msg["role"]
703
-
704
- if role == "assistant":
705
- # decoded is {"text": ..., "tool_calls": [...]} -- exactly
706
- # what add_assistant_message persisted, nothing nested.
707
- payload = {
708
- "role": "assistant",
709
- "text": decoded.get("text", ""),
710
- "tool_calls": decoded.get("tool_calls", []),
711
- }
712
- # §44, from its own column rather than from `content`. Present
713
- # ONLY when the row carried reasoning, so every message this
714
- # project has ever written reconstructs to exactly the shape it
715
- # did before -- which is what keeps _messages_for_provider,
716
- # compactable_span and the fake's mirror unchanged for them.
717
- thinking = _decode_thinking(msg.get("thinking"))
718
- if thinking is not None:
719
- payload["thinking"] = thinking
720
- elif role == "tool":
721
- # decoded is the plain result string add_tool_result persisted.
722
- payload = {"role": "tool", "tool_call_id": msg["tool_call_id"], "content": decoded}
723
- # ROADMAP_v3 §81 (IM2), on §44's terms for `thinking`: present ONLY
724
- # when the row attached an image, so every tool row written before
725
- # reconstructs to exactly the shape it did. Read by translation and
726
- # the compactor's size proxy; never by anything that renders text.
727
- images = _decode_images(msg.get("images"))
728
- if images is not None:
729
- payload["images"] = images
730
- else:
731
- # user (and any future plain-content role): decoded is
732
- # already the right value for "content".
733
- payload = {"role": role, "content": decoded}
734
- if msg["tool_call_id"]:
735
- payload["tool_call_id"] = msg["tool_call_id"]
736
- # Batch 91, on §44's terms for `thinking`: present ONLY when the row
737
- # records a failed run, so every message written before reconstructs
738
- # to exactly the shape it did. Nothing that builds a request reads
739
- # it -- _messages_for_provider takes a user row's `content` alone,
740
- # and so does the compactor's _as_text.
741
- if msg.get("error"):
742
- payload["error"] = msg["error"]
743
- # ROADMAP_v3 §49 (SS5), on the same terms: present ONLY when the HARNESS
744
- # wrote the row, so every other message reconstructs exactly as it did.
745
- # Read by replay and the compactor's labelling; never by translation.
746
- harness = _decode_harness(msg.get("harness"))
747
- if harness is not None:
748
- payload["harness"] = harness
749
-
750
- if msg["name"]:
751
- payload["name"] = msg["name"]
752
-
753
- return payload
754
-
755
-
756
- def _ordered_rows(thread_id: UUID) -> list[dict]:
757
- """Every row of a thread as plain column values, oldest first.
758
-
759
- Ordered by (created_at, id) rather than created_at alone. Timestamps
760
- have microsecond resolution so ties are unlikely, but "unlikely" is
761
- not an ordering guarantee, and §21's watermark arithmetic slices this
762
- list by position -- an unstable sort there would move the fold
763
- boundary between two reads of the same thread.
764
-
765
- Plain dicts rather than ORM instances because every caller below
766
- reads these after the Session has closed, and what a detached
767
- instance will still hand back depends on what happened to be loaded.
768
- Copying six columns removes the question.
769
-
770
- `pinned` is coerced to bool: a database migrated by
771
- database.ensure_columns() gets the column with a DEFAULT, but a row
772
- written by an older build through some other path could still read
773
- back NULL, and None is not the same thing as "not pinned" to a
774
- caller doing `if row["pinned"]`.
775
- """
776
- with Session(engine) as session:
777
- statement = (
778
- select(MessageLog)
779
- .where(MessageLog.thread_id == thread_id)
780
- .order_by(MessageLog.created_at.asc(), MessageLog.id.asc())
781
- )
782
- return [
783
- {
784
- "id": m.id, "role": m.role, "content": m.content,
785
- "name": m.name, "tool_call_id": m.tool_call_id,
786
- "pinned": bool(m.pinned),
787
- # Not coerced, unlike `pinned`: NULL is the ordinary state
788
- # of this column and _decode_thinking answers None for it.
789
- "thinking": m.thinking,
790
- # Batch 91, and NULL is ordinary here too: a run failed.
791
- "error": m.error,
792
- # ROADMAP_v3 §49 (SS5). NULL for every row a person or the
793
- # model wrote.
794
- "harness": m.harness,
795
- # ROADMAP_v3 §81 (IM2). NULL for every row that attached no
796
- # image; the compactor's size proxy reads it (IM8).
797
- "images": m.images,
798
- }
799
- for m in session.exec(statement).all()
800
- ]
801
-
802
-
803
- def _split_at(rows: list[dict], message_id: Optional[UUID]) -> int:
804
- """Index just past `message_id` in `rows`, or 0 when it isn't there.
805
-
806
- Resolving a watermark by POSITION in the already-loaded, already-ordered
807
- list rather than by a `(created_at, id) > (t, i)` SQL predicate. The
808
- comparison a tuple-ordered watermark needs is awkward to express
809
- portably over a UUID column, and a thread is at most a few thousand
810
- rows that this function's callers are loading anyway. Simplicity wins
811
- over a query that has to be right about UUID collation.
812
-
813
- A watermark that names a row this thread does not have returns 0 --
814
- i.e. nothing is treated as covered. That is the safe direction: the
815
- caller shows the whole thread rather than silently hiding a prefix
816
- on the strength of an id it could not find.
817
- """
818
- if message_id is None:
819
- return 0
820
- for index, row in enumerate(rows):
821
- if row["id"] == message_id:
822
- return index + 1
823
- return 0
824
-
825
-
826
- def get_session_history(
827
- thread_id: UUID, after_message_id: Optional[UUID] = None,
828
- ) -> list[dict]:
829
- """The thread's messages in neutral shape, oldest first.
830
-
831
- `after_message_id` (ROADMAP_v2 §21) loads only the UNCOMPACTED TAIL --
832
- everything strictly after that row. The rows before it are not gone;
833
- they are represented by a CompactionCheckpoint summary that
834
- core/memory.py prepends, and they remain readable in full via
835
- archive_history(). None (the default) returns the whole thread, which
836
- is what every pre-§21 caller gets and why their behaviour is
837
- unchanged.
838
- """
839
- rows = _ordered_rows(thread_id)
840
- return [_to_neutral(m) for m in rows[_split_at(rows, after_message_id):]]
841
-
842
-
843
- def archive_history(thread_id: UUID) -> list[dict]:
844
- """Every message ever written to this thread, compaction or not.
845
-
846
- This is ROADMAP_v2 §21 AC1 made greppable: compaction adds
847
- CompactionCheckpoint rows and never edits or deletes a MessageLog
848
- row, so this read is complete no matter how many times a thread has
849
- been compacted. It is a distinct name rather than a comment on
850
- get_session_history because the guarantee is what callers want to
851
- find -- and because a future watermark default on that function
852
- would silently take this one with it.
853
- """
854
- return get_session_history(thread_id)
855
-
856
-
857
- def _turn_opener_index(rows: list[dict], span: list[dict]) -> list[int]:
858
- """`[i]` for the user row that OPENS the turn a MID-TURN watermark cut
859
- into, or `[]`. Batch 120, register item 40.
860
-
861
- A sub-turn fold puts the watermark inside a turn, so the turn's own
862
- request falls on the covered side and would be summarized away -- and
863
- the model would then read a paraphrase of the question it is answering
864
- rather than the question. That is M5's objection to folding the current
865
- turn at all, and re-adding the request verbatim is what answers it.
866
-
867
- The TEST IS THE FIRST KEPT ROW, not a flag on the checkpoint. When the
868
- watermark is turn-aligned the tail opens with a `user` row and nothing
869
- needs re-adding; when it is mid-turn the tail opens with an `assistant`
870
- row, because that is the only boundary _sub_turn_span will place. So
871
- the state is derivable from the rows, which means no column, no
872
- migration, and no checkpoint written before this existed that lies
873
- about its own shape.
874
- """
875
- if not span or len(span) >= len(rows):
876
- return []
877
- if rows[len(span)]["role"] == "user":
878
- return []
879
- for index in range(len(span) - 1, -1, -1):
880
- if span[index]["role"] == "user":
881
- return [index]
882
- return []
883
-
884
-
885
- def pinned_through(thread_id: UUID, message_id: Optional[UUID]) -> list[dict]:
886
- """Rows at or before the watermark the derived view must carry
887
- verbatim: the pinned set, made whole (§21 M9), plus the opening request
888
- of a turn a mid-turn watermark cut into (batch 120).
889
-
890
- A summary covers a contiguous span, but §21 AC2 says a pinned message is
891
- never summarized -- so pinned rows INSIDE the covered span have to
892
- come back verbatim, and a single `covers_up_to_message_id` cannot say
893
- "everything through K except these". The derived view is therefore
894
- summary + these rows + tail.
895
-
896
- Empty when nothing in the span is pinned, which is every thread until
897
- someone calls pin().
898
-
899
- THE PINNED SPAN IS MADE WHOLE HERE, which is M4's rule applied to the
900
- second boundary. The derived view has two of them and only the
901
- checkpoint watermark is turn-aligned: `pin` is dispatched from inside a
902
- turn, so the set it writes always ends mid-turn. D20 has already
903
- persisted the assistant message carrying the `tool_use` for `pin`
904
- itself, and the answering `tool` row is written after dispatch returns
905
- -- so it is never pinned. While the pin is still in the uncompacted
906
- tail this is invisible, because the tail carries both rows. The first
907
- compaction whose watermark passes the pinned region activates it: the
908
- assistant row comes back with an unanswered `tool_use`, its result
909
- having gone into the summary text instead, and a `tool_result` with no
910
- preceding `tool_use` is an HTTP 400 on Anthropic -- on a thread that
911
- was working a moment ago, with no visible connection to a `pin` several
912
- turns earlier. Durable, too: the checkpoint is a persisted row, so
913
- every later call and every resume rebuilds the same unsendable view.
914
-
915
- Fixed at the PRODUCER, where a pinned span becomes view content.
916
- Stopping pin_last at the last complete turn would change what "pin the
917
- last N turns" means and leave any other non-turn-aligned pinned set
918
- open; dropping unanswered tool_calls in _derived_view is consumer-side
919
- -- this project's canonical bug shape -- and would hide from the model
920
- a call it actually made.
921
- """
922
- rows = _ordered_rows(thread_id)
923
- span = rows[:_split_at(rows, message_id)]
924
- neutral = [_to_neutral(m) for m in span]
925
- return [neutral[i] for i in _pinned_indices(rows, span, neutral)]
926
-
927
-
928
- def _pinned_indices(rows: list, span: list, neutral: list) -> list[int]:
929
- """The positions in `span` that `pinned_through` returns, in archive
930
- order: the pinned rows, the tool results that make them whole, and the
931
- opener of a turn a mid-turn watermark cut into.
932
-
933
- A PURE FUNCTION SINCE BATCH 216 (#231), split out of `pinned_through`
934
- so `tests/test_fake_storage_mirror.py` can compare it against
935
- `FakeStorage`'s copy the way it compares `_to_neutral` and `_split_at`
936
- -- the fake transcribes this algorithm, and a transcription nothing
937
- compares is a convention somebody has to remember. `rows` and `span`
938
- need only `role` and `pinned`; `neutral` is `span` reconstructed.
939
- """
940
- keep = [i for i, m in enumerate(span) if m["pinned"]]
941
- # `have`, the set() below and the role check are all BACKSTOPS, and the
942
- # measurement says so: each was deleted separately against the full
943
- # suite and each survived. What actually keeps the result duplicate-free
944
- # is the `not m["pinned"]` filter -- a pinned tool row is already in
945
- # `keep` and can never be re-added -- and reaching any of the three
946
- # needs two rows sharing one tool_call_id, or a non-tool row carrying
947
- # one, neither of which has a producer: add_tool_result writes exactly
948
- # one row per call, and add_user_message passes no tool_call_id.
949
- #
950
- # Kept rather than trimmed, and the reasoning recorded rather than
951
- # asserted: the failure they guard against is a duplicate tool_result,
952
- # which is the same wire error from the other direction as the one this
953
- # function exists to prevent. That is the wrong place to trade a free
954
- # backstop for a reachability argument.
955
- have = {neutral[i].get("tool_call_id") for i in keep}
956
- need = {call["id"] for i in keep
957
- for call in (neutral[i].get("tool_calls") or [])
958
- if call["id"] not in have}
959
- keep += [i for i, m in enumerate(span)
960
- if not m["pinned"] and m["role"] == "tool"
961
- and neutral[i].get("tool_call_id") in need]
962
-
963
- # Batch 120. ONE INDEX, AND THE SORT BELOW PLACES IT -- which is the
964
- # whole reason the turn opener lives here rather than in a function of
965
- # its own beside this one. The view needs these rows in ARCHIVE ORDER,
966
- # and `_derived_view` has no positions to merge two lists by: a pinned
967
- # row from an older turn belongs before the request and a pinned row
968
- # from inside this turn belongs after it, and either sibling-function
969
- # arrangement gets one of those wrong. `set()` covers the opener
970
- # already being pinned.
971
- keep += _turn_opener_index(rows, span)
972
-
973
- return sorted(set(keep))
974
-
975
-
976
- def history_through(
977
- thread_id: UUID, message_id: Optional[UUID],
978
- after_message_id: Optional[UUID] = None,
979
- ) -> list[dict]:
980
- """The span a compaction is about to summarize, in neutral shape.
981
-
982
- Pinned rows are EXCLUDED (§21 AC2: a pinned message is never included in
983
- what the compactor is asked to summarize). `after_message_id` is what
984
- makes chaining possible -- M2's chain strategy summarizes only what
985
- followed the previous checkpoint, while rederive passes None and
986
- summarizes the whole span from the top.
987
- """
988
- rows = _ordered_rows(thread_id)
989
- start = _split_at(rows, after_message_id)
990
- end = _split_at(rows, message_id)
991
- return [_to_neutral(m) for m in rows[start:end] if not m["pinned"]]
992
-
993
-
994
- def turn_start_ids(thread_id: UUID) -> list[UUID]:
995
- """Ids of the rows that BEGIN each turn, oldest first.
996
-
997
- A turn starts at a user message: the wrapper adds one, then the loop
998
- appends assistant and tool rows until it stops. §21's M4 needs this
999
- because a fold boundary that lands anywhere else can separate an
1000
- assistant message from its tool results, and a tool_result with no
1001
- preceding tool_use is an HTTP 400 on Anthropic -- a hard wire error,
1002
- not a degraded answer. M5's "keep the last N turns" needs it too.
1003
- """
1004
- return [m["id"] for m in _ordered_rows(thread_id) if m["role"] == "user"]
1005
-
1006
-
1007
- def message_ids_from(thread_id: UUID, message_id: UUID) -> list[UUID]:
1008
- """Ids of `message_id` and everything after it. Resolves pin()'s
1009
- ordinal ("the last N turns") into the real row ids the pinned flag is
1010
- written against -- §21 keeps MessageLog.id an implementation detail
1011
- the model never sees, so the ordinal-to-id mapping has to happen
1012
- somewhere below the tool."""
1013
- rows = _ordered_rows(thread_id)
1014
- start = _split_at(rows, message_id)
1015
- if not start:
1016
- return []
1017
- return [message_id] + [m["id"] for m in rows[start:]]
1018
-
1019
-
1020
- def set_pinned(message_ids: list[UUID], pinned: bool = True) -> int:
1021
- """Flag rows compaction-exempt. Returns how many rows changed."""
1022
- if not message_ids:
1023
- return 0
1024
- changed = 0
1025
- with Session(engine) as session:
1026
- for message_id in message_ids:
1027
- row = session.get(MessageLog, message_id)
1028
- if row is None or row.pinned == pinned:
1029
- continue
1030
- row.pinned = pinned
1031
- session.add(row)
1032
- changed += 1
1033
- session.commit()
1034
- return changed
1035
-
1036
-
1037
- def mark_turn_failed(thread_id: UUID, error: str) -> bool:
1038
- """Record on the thread's NEWEST user row why the run it started failed.
1039
-
1040
- Batch 91; see MessageLog.error for why this is a column. The newest user
1041
- row is always the right one: every entry point writes the message that
1042
- starts a run BEFORE the loop begins -- run_agent_conversation,
1043
- continue_conversation, stream_deep_research_mode and the TUI turn -- so
1044
- by the time a failure escapes the loop, the row that started it is the
1045
- last user row there is. Ordered by (created_at, id), `_ordered_rows`'
1046
- own order, so the two can never disagree about which row is newest.
1047
-
1048
- Returns False for a thread with no user row. Does not stamp
1049
- last_activity_at the way save_message does: a failure is not a message.
1050
- """
1051
- with Session(engine) as session:
1052
- statement = (
1053
- select(MessageLog)
1054
- .where(MessageLog.thread_id == thread_id)
1055
- .where(MessageLog.role == "user")
1056
- .order_by(MessageLog.created_at.desc(), MessageLog.id.desc())
1057
- )
1058
- row = session.exec(statement).first()
1059
- if row is None:
1060
- return False
1061
- row.error = error
1062
- session.add(row)
1063
- session.commit()
1064
- return True
1065
-
1066
-
1067
- def _current_watermark(thread_id: UUID) -> Optional[UUID]:
1068
- """The live checkpoint's watermark, or None if never compacted."""
1069
- checkpoint = latest_checkpoint(thread_id)
1070
- return checkpoint["covers_up_to_message_id"] if checkpoint else None
1071
-
1072
-
1073
- def advances(thread_id: UUID, current: Optional[UUID], proposed: UUID) -> bool:
1074
- """Is `proposed` strictly later in this thread than `current`?
1075
-
1076
- The ordering test behind §21's one real monotonic invariant: a
1077
- compaction watermark only ever moves FORWARD, and -- #27's half --
1078
- it must name a row THIS THREAD actually has. `current` of None
1079
- (no checkpoint yet) skips the ordering comparison but not the
1080
- containment one: a watermark we cannot place is one we must not
1081
- write, and "first compaction" is exactly when nothing else would
1082
- catch a foreign id.
1083
-
1084
- A `proposed` this thread does not contain returns False -- the safe
1085
- direction, since a watermark we cannot place is one we must not write.
1086
- """
1087
- rows = _ordered_rows(thread_id)
1088
- if current is None:
1089
- return _split_at(rows, proposed) > 0
1090
- return _split_at(rows, proposed) > _split_at(rows, current) > 0
1091
-
1092
-
1093
- def latest_checkpoint(thread_id: UUID) -> Optional[dict]:
1094
- """The most recent compaction of this thread as a plain column dict,
1095
- or None if it has never been compacted (which is every thread until
1096
- the trigger first fires).
1097
-
1098
- A dict rather than a detached ORM instance (#31): two of the three
1099
- callers read it after their Session has closed, and one of them is
1100
- the compaction path -- the one place in the project where a wrong
1101
- read rewrites a live conversation. Copying the columns removes the
1102
- question of what a detached instance still hands back.
1103
- """
1104
- with Session(engine) as session:
1105
- statement = (
1106
- select(CompactionCheckpoint)
1107
- .where(CompactionCheckpoint.thread_id == thread_id)
1108
- .order_by(CompactionCheckpoint.created_at.desc(),
1109
- CompactionCheckpoint.id.desc())
1110
- )
1111
- row = next(iter(session.exec(statement).all()), None)
1112
- if row is None:
1113
- return None
1114
- return {
1115
- "id": row.id,
1116
- "thread_id": row.thread_id,
1117
- "summary_text": row.summary_text,
1118
- "covers_up_to_message_id": row.covers_up_to_message_id,
1119
- "strategy": row.strategy,
1120
- "created_at": row.created_at,
1121
- }
1122
-
1123
-
1124
- def save_checkpoint(
1125
- thread_id: UUID, summary_text: str, covers_up_to_message_id: UUID,
1126
- strategy: str = "rederive",
1127
- ) -> UUID:
1128
- """Record one compaction. Adds a row; touches nothing else (§21 AC1).
1129
-
1130
- REFUSES A WATERMARK THAT DOES NOT ADVANCE, returning the existing
1131
- checkpoint's id instead. `latest_checkpoint` resolves ties by
1132
- timestamp, so a backwards watermark written here does not merely fail
1133
- to help -- it becomes the live one, and the thread UN-COMPACTS: the
1134
- derived view jumps back to showing everything after the earlier
1135
- boundary. That is what §21a shipped, and the caller-side guard in
1136
- core/compaction.py is not enough on its own, because it only covers
1137
- the one caller that exists today.
1138
-
1139
- Enforced rather than documented, for the same reason D24's declared-
1140
- permission check is: the invariant is cheap to state and its violation
1141
- is silent and durable.
1142
- """
1143
- if not advances(thread_id, _current_watermark(thread_id),
1144
- covers_up_to_message_id):
1145
- existing = latest_checkpoint(thread_id)
1146
- logger.warning(
1147
- "Refused a compaction checkpoint for thread %s whose watermark "
1148
- "does not advance on the existing one; keeping the existing "
1149
- "checkpoint. This would have made the thread larger, not "
1150
- "smaller.", thread_id)
1151
- return existing["id"] if existing is not None else None
1152
-
1153
- with Session(engine) as session:
1154
- checkpoint = CompactionCheckpoint(
1155
- thread_id=thread_id,
1156
- summary_text=summary_text,
1157
- covers_up_to_message_id=covers_up_to_message_id,
1158
- strategy=strategy,
1159
- )
1160
- session.add(checkpoint)
1161
- session.commit()
1162
- session.refresh(checkpoint)
1163
- return checkpoint.id
1164
-
1165
-
1166
- # ---------------------------------------------------------------------------
1167
- # ---- Thread summaries (ROADMAP_v2 §21c) -----------------------------------
1168
- # ---------------------------------------------------------------------------
1169
-
1170
- def last_message_id(thread_id: UUID) -> Optional[UUID]:
1171
- """The id of this thread's most recent archive row, or None if empty.
1172
-
1173
- §21c needs a position meaning "everything, as of now", to store as a
1174
- summary's watermark and to compare against a stored one. Nothing existing
1175
- answers it: `message_ids_from` needs a start id, `turn_start_ids` returns
1176
- turn starts (and the last row is usually not one), and `_ordered_rows` is
1177
- private for the reasons its docstring gives.
1178
-
1179
- Reads the ARCHIVE, like everything else here -- the ordering is
1180
- `(created_at, id)`, so this is the same last row `advances()` compares
1181
- positions within.
1182
- """
1183
- rows = _ordered_rows(thread_id)
1184
- return rows[-1]["id"] if rows else None
1185
-
1186
-
1187
- def latest_thread_summary(thread_id: UUID) -> Optional[dict]:
1188
- """The most recent summary of this thread as a plain column dict, or
1189
- None if never summarized (#31 -- same reasoning as latest_checkpoint).
1190
-
1191
- Newest by `(created_at, id)`, matching `latest_checkpoint` -- but note
1192
- that the resemblance ends at the query. A stale row winning here shows
1193
- someone an out-of-date description, which `advances()` then detects and
1194
- corrects; a stale row winning in `latest_checkpoint` silently rewrites a
1195
- live conversation. See ThreadSummary's docstring.
1196
- """
1197
- with Session(engine) as session:
1198
- statement = (
1199
- select(ThreadSummary)
1200
- .where(ThreadSummary.thread_id == thread_id)
1201
- .order_by(ThreadSummary.created_at.desc(), ThreadSummary.id.desc())
1202
- )
1203
- row = next(iter(session.exec(statement).all()), None)
1204
- if row is None:
1205
- return None
1206
- return {
1207
- "id": row.id,
1208
- "thread_id": row.thread_id,
1209
- "summary_text": row.summary_text,
1210
- "covers_up_to_message_id": row.covers_up_to_message_id,
1211
- "created_at": row.created_at,
1212
- }
1213
-
1214
-
1215
- def save_thread_summary(
1216
- thread_id: UUID, summary_text: str, covers_up_to_message_id: UUID,
1217
- ) -> UUID:
1218
- """Record one summary of one thread. Adds a row; touches nothing else.
1219
-
1220
- NO ADVANCE GUARD, deliberately, and this is the one place the difference
1221
- from `save_checkpoint` above is load-bearing rather than decorative. That
1222
- function REFUSES a watermark that does not advance, because a backwards
1223
- compaction watermark becomes the live one and un-compacts the thread. A
1224
- summary row changes what NO model sees: the worst a redundant or
1225
- backwards one can do is describe the thread as it was a moment ago, which
1226
- the next `advances()` check notices.
1227
-
1228
- So the guard is absent because it would be cargo -- not because the
1229
- monotonic property does not matter here. Do not add one, and do not route
1230
- a summary through `save_checkpoint` to get one.
1231
- """
1232
- with Session(engine) as session:
1233
- summary = ThreadSummary(
1234
- thread_id=thread_id,
1235
- summary_text=summary_text,
1236
- covers_up_to_message_id=covers_up_to_message_id,
1237
- )
1238
- session.add(summary)
1239
- session.commit()
1240
- session.refresh(summary)
1241
- return summary.id
1242
-
1243
-
1244
- # ---------------------------------------------------------------------------
1245
- # ---- Durable memory (ROADMAP_v2 §21b) -------------------------------------
1246
- # ---------------------------------------------------------------------------
1247
-
1248
- def save_memory(
1249
- content: str, source_thread_id: UUID, scope: str = "project",
1250
- category: Optional[str] = None, project_path: Optional[str] = None,
1251
- ) -> UUID:
1252
- """Record one durable fact. Returns its id.
1253
-
1254
- `project_path` is required in substance for scope="project" and
1255
- meaningless for "global" -- the caller resolves it, because this file
1256
- knows nothing about config_loader or the trust store. memories/manager.py
1257
- is the one place that decides.
1258
- """
1259
- with Session(engine) as session:
1260
- row = UserMemory(
1261
- content=content,
1262
- category=category,
1263
- scope=scope,
1264
- project_path=project_path if scope == "project" else None,
1265
- source_thread_id=source_thread_id,
1266
- )
1267
- session.add(row)
1268
- session.commit()
1269
- session.refresh(row)
1270
- return row.id
1271
-
1272
-
1273
- def list_memories(
1274
- project_path: Optional[str] = None, scope: Optional[str] = None,
1275
- ) -> list[dict]:
1276
- """Memories visible from `project_path`, NEWEST FIRST.
1277
-
1278
- Returns global memories plus the project ones recorded against exactly
1279
- this path -- which is §21 AC6: a memory written in one project does not
1280
- surface in a thread opened from another. `project_path=None` returns
1281
- global memories only, which is the honest answer when no project has
1282
- been resolved rather than a reason to show everything.
1283
-
1284
- `scope` filters to one kind, for the management commands. Absent means
1285
- both, which is what prompt assembly wants.
1286
-
1287
- Both predicates are pushed INTO THE QUERY (#28): `UserMemory` declares
1288
- indexes on `scope` and `project_path`, and this function used to load
1289
- every row and filter in Python -- the exact pattern T1 rejected, run
1290
- against two columns whose indexes had no reader. SQLite answers the
1291
- visibility rule below from those indexes now.
1292
-
1293
- Plain dicts rather than ORM instances, for the same reason
1294
- _ordered_rows returns them: every caller reads these after the Session
1295
- has closed.
1296
- """
1297
- with Session(engine) as session:
1298
- statement = select(UserMemory)
1299
- if scope is not None:
1300
- statement = statement.where(UserMemory.scope == scope)
1301
- # Visibility: anything not project-scoped travels; a project memory
1302
- # only matches its own path. A NULL path therefore matches nothing
1303
- # except when the caller has no resolved project either -- same as
1304
- # the Python filter this replaced, edge included deliberately.
1305
- statement = statement.where(
1306
- (UserMemory.scope != "project")
1307
- | (UserMemory.project_path == project_path))
1308
- statement = statement.order_by(
1309
- UserMemory.created_at.desc(), UserMemory.id.desc())
1310
- return [
1311
- {
1312
- "id": row.id, "content": row.content,
1313
- "category": row.category, "scope": row.scope,
1314
- "project_path": row.project_path,
1315
- "source_thread_id": row.source_thread_id,
1316
- "created_at": row.created_at,
1317
- }
1318
- for row in session.exec(statement).all()
1319
- ]
1320
-
1321
-
1322
- def forget_memory(memory_id: UUID) -> bool:
1323
- """Delete one memory. True if a row was removed.
1324
-
1325
- A REAL DELETE, unlike everything else here. MessageLog is append-only
1326
- because a conversation is a record of what happened; a UserMemory is a
1327
- standing assertion the harness keeps repeating, and "this is no longer
1328
- true" has no useful archived form. Returning False for an unknown id
1329
- rather than raising: the id came from a human reading a list, and a
1330
- typo deserves "no such memory", not a traceback.
1331
- """
1332
- with Session(engine) as session:
1333
- row = session.get(UserMemory, memory_id)
1334
- if row is None:
1335
- return False
1336
- session.delete(row)
1337
- session.commit()
1338
- return True
1
+ import json
2
+ import logging
3
+ from datetime import UTC, datetime
4
+ from typing import Any, Optional
5
+ from uuid import UUID, uuid4
6
+
7
+ from sqlalchemy import (
8
+ func as sa_func, # real sqlalchemy; sqlmodel is faked in tests, this is not
9
+ )
10
+ from sqlmodel import JSON, Field, Session, SQLModel, select
11
+
12
+ from database import engine # your SQLAlchemy engine, assumed to exist here
13
+
14
+ logger = logging.getLogger(__name__)
15
+
16
+
17
+ class ConversationThread(SQLModel, table=True):
18
+ id: UUID = Field(default_factory=uuid4, primary_key=True)
19
+ created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
20
+ extra_data: dict[str, Any] = Field(default_factory=dict, sa_type=JSON)
21
+ # ROADMAP_v2 §27 (T1). What this thread IS: "chat" (a conversation a
22
+ # human had), "research_pass" (one pass of the ten-pass pipeline) or
23
+ # "subagent" (a spawned agent, the §20 reviewer, or the compactor).
24
+ #
25
+ # A COLUMN, not an extra_data key. That field exists and would need no
26
+ # migration, but it is an opaque JSON blob -- filtering on it means
27
+ # loading every row and testing in Python, and list_threads() is the
28
+ # one query that must stay cheap on a database gaining ~10 rows per
29
+ # research run.
30
+ #
31
+ # NOT indexed, deliberately. ensure_columns() ALTERs in a column and
32
+ # does not build indexes, and create_all() will not add one to a table
33
+ # that already exists -- so index=True here would produce an index on
34
+ # fresh databases and none on migrated ones. A divergence that shows
35
+ # up only as a performance difference is worse than no index at all;
36
+ # the WHERE clause is what T1 was buying, not the index.
37
+ kind: str = Field(default="chat")
38
+ # (#32) When this thread was last written to -- ANY archived row,
39
+ # user, assistant or tool alike. The picker orders by this rather
40
+ # than by creation, because "most recent first" should surface the
41
+ # conversation you were just in, not the one you merely opened last.
42
+ #
43
+ # Nullable deliberately, twice over. Semantically: NULL means "never
44
+ # received a message", and the reader falls back to created_at --
45
+ # which is also what every row predating this column reads as, since
46
+ # the additive migration never backfills (M7). Mechanically: the
47
+ # honest default is per-row Python (a timestamp), which has no SQL
48
+ # literal, so the migrator could not have added it NOT NULL anyway
49
+ # (#26's WARNING path). save_message is the single writer of
50
+ # MessageLog rows in production and stamps this in the same session
51
+ # it writes the message, so the two cannot drift.
52
+ last_activity_at: Optional[datetime] = Field(default=None)
53
+ # ROADMAP_v2 §47. WHO spawned this thread, and at which call.
54
+ #
55
+ # Before these, the only trace of the edge was the child's uuid
56
+ # stringified inside the parent's tool-result row -- a repr inside a
57
+ # JSON blob, not a field, not queryable, and skipped by replay's T4
58
+ # anyway. A sidebar row and a `▸ spawn_subagent` line could describe
59
+ # a run nobody could then open.
60
+ #
61
+ # COLUMNS, not `extra_data`, for `kind`'s reason exactly (T1): the
62
+ # question asked of them is "which threads are children of this
63
+ # one", and answering that from an opaque JSON blob means loading
64
+ # every row and testing in Python. Not indexed, also for T1's
65
+ # reason -- ensure_columns() ALTERs a column in and builds no index,
66
+ # and create_all() adds none to a table that already exists, so
67
+ # index=True would mean an index on fresh databases and none on
68
+ # migrated ones.
69
+ #
70
+ # Nullable and additive, so database.ensure_columns() adds them to an
71
+ # existing app.db exactly as it added `pinned`, `kind`, `thinking`
72
+ # and `last_activity_at`. NULL is the honest value for every thread
73
+ # that predates them and for every one nothing spawned.
74
+ parent_thread_id: Optional[UUID] = Field(default=None)
75
+ # The parent's `tool_call_id` for the call that made this thread --
76
+ # the SAME id MessageLog stores on the parent's side, which is what
77
+ # lets one specific `▸ spawn_subagent` line find one specific child
78
+ # when a turn spawned three. None for a run no tool call made: the
79
+ # compactor, the reviewer, /init's initializer.
80
+ parent_call_id: Optional[str] = Field(default=None)
81
+ # Which agent ran, for a reader who has only the row. `kind` says
82
+ # what a thread IS; this says who it was. A viewer opening a stored
83
+ # thread cannot ask the span, because the span is long gone.
84
+ agent_name: Optional[str] = Field(default=None)
85
+
86
+
87
+ class MessageLog(SQLModel, table=True):
88
+ id: UUID = Field(default_factory=uuid4, primary_key=True)
89
+ thread_id: UUID = Field(foreign_key="conversationthread.id", index=True)
90
+ role: str = Field(description="system, user, assistant, or tool")
91
+ content: str # always JSON-encoded, regardless of the original type
92
+ name: Optional[str] = None
93
+ tool_call_id: Optional[str] = None
94
+ created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
95
+ # ROADMAP_v2 §21. Compaction-exempt within this thread (D26's pin
96
+ # tool). This is the first field this project has ever added to a
97
+ # table that already exists on disk, which is why database.py grew
98
+ # ensure_columns() -- create_all() would have left every existing
99
+ # app.db without it and failed at the next SELECT (M7).
100
+ pinned: bool = Field(default=False)
101
+ # §44. The turn's reasoning, JSON-encoded, or None -- assistant rows
102
+ # only, and only where the provider returned any.
103
+ #
104
+ # A SEPARATE COLUMN rather than a key inside `content`, because these
105
+ # are different KINDS of thing with different rules. `content` is what
106
+ # the model said; this is the provider's own signed blocks, which go
107
+ # back on the wire unchanged or not at all, and which every reader
108
+ # that measures the conversation (the compactor's input, /summary's
109
+ # distillation) has to be able to leave out without parsing around
110
+ # them. Nullable and additive, so database.ensure_columns() adds it to
111
+ # an existing database exactly as it added `pinned` -- see the note on
112
+ # that field's migration in _ordered_rows.
113
+ thinking: Optional[str] = None
114
+ # Batch 91. Why the run this row STARTED failed, or None -- user rows
115
+ # only, written by mark_turn_failed when an exception escapes the loop.
116
+ #
117
+ # A column on the user row rather than a row of its own, and that is
118
+ # the whole design. A failed run leaves no assistant turn behind (D20
119
+ # persists one only when a call returns), so without this a thread
120
+ # whose first call failed read as a request nobody answered. An `error`
121
+ # ROLE would have been a harness notice stored as conversation -- the
122
+ # compactor's `_as_text` and /summary render every role, so it would
123
+ # have been summarized into what the model later reads, which is M8's
124
+ # mistake. A column is read by nothing that measures the conversation:
125
+ # provider translation and the compactor take `content` alone. On the
126
+ # user row because that is what says WHICH turn failed, which a
127
+ # thread-level field could not for a chat. Nullable and additive, so
128
+ # database.ensure_columns() adds it like `thinking`.
129
+ error: Optional[str] = None
130
+ # ROADMAP_v3 §49 (SS5). What wrote this row when the HARNESS did, as a
131
+ # JSON object -- {"kind": "session_wake" | "session_killed_at_quit" |
132
+ # "session_held", "sessions": [{"id", "call_id", "shape"}, ...]} -- or
133
+ # None for everything a person or the model wrote.
134
+ #
135
+ # A column on a USER row, for batch 91's reason one field over. A
136
+ # finished background session has to reach the model as a message that
137
+ # starts a turn, and a tool result cannot carry it: every tool_use is
138
+ # answered in the step that made it (D20, NA11), long before the session
139
+ # ends. A user-role row is the one shape every provider accepts there,
140
+ # and a compacted thread already sends one the person never wrote (M8's
141
+ # summary). A `harness` ROLE would need translating and summarizing as
142
+ # something new; a column is read by nothing that builds a request, so
143
+ # the row goes out as the plain user message it is, while replay and
144
+ # the compactor can still tell it was not the person.
145
+ #
146
+ # NEVER beside a tool_call_id. `pinned_through` treats any kept row's
147
+ # tool_call_id as an answered call, so a wake row carrying its session's
148
+ # start call id would hide the real tool_result -- the HTTP 400 M4
149
+ # exists to prevent. The call id lives inside this JSON instead.
150
+ harness: Optional[str] = None
151
+ # ROADMAP_v3 §81 (IM2). The images a tool result carried, as a JSON list
152
+ # of {"media_type", "data"} with `data` base64 -- tool rows only, and only
153
+ # where `view_image` attached one. NULL everywhere else.
154
+ #
155
+ # A COLUMN rather than blocks inside `content`, for `thinking`'s reason:
156
+ # `content` is read as a string by every consumer of the conversation --
157
+ # replay, the transcript, /copy, the compactor, the summarizer -- and
158
+ # only two readers have any use for the bytes, provider translation and
159
+ # the compactor's size proxy. In the database rather than in a blob store
160
+ # or as a path to the workspace file: a path would make a resumed thread
161
+ # send whatever the file holds NOW, and a second store would be a second
162
+ # place docs/PRIVACY.md's "wipe everything" has to name. Nullable and
163
+ # additive, so database.ensure_columns() adds it like `thinking`.
164
+ images: Optional[str] = None
165
+
166
+
167
+ class CompactionCheckpoint(SQLModel, table=True):
168
+ """One compaction of one thread (ROADMAP_v2 §21).
169
+
170
+ THE ARCHIVE IS NEVER EDITED. Compaction only ever ADDS a row here;
171
+ MessageLog is append-only and stays complete forever, which is what
172
+ §21 AC1 is about and what makes an aggressive compaction trigger safe --
173
+ nothing is ever destroyed, only hidden from the next call. See
174
+ archive_history() for the read that proves it.
175
+
176
+ `summary_text` represents every message in this thread through
177
+ `covers_up_to_message_id` EXCEPT the pinned ones (§21 AC2), which is why
178
+ the derived view has to re-include them separately (M9) -- a single
179
+ watermark cannot express "everything up to K except rows 5 and 6".
180
+
181
+ `strategy` records which way this summary was produced: "rederive"
182
+ (summarized the original messages, so exactly one summarization step
183
+ sits between an original message and what the model sees) or "chain"
184
+ (summarized the previous summary plus what followed it, bounded cost,
185
+ compounding loss). M2 makes rederive the default and chain the
186
+ configurable/fallback case; storing it per row means a thread's
187
+ history of the two is auditable rather than inferred from settings
188
+ that may since have changed.
189
+ """
190
+
191
+ id: UUID = Field(default_factory=uuid4, primary_key=True)
192
+ thread_id: UUID = Field(foreign_key="conversationthread.id", index=True)
193
+ summary_text: str
194
+ covers_up_to_message_id: UUID
195
+ strategy: str = Field(default="rederive")
196
+ created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
197
+
198
+
199
+ class ThreadSummary(SQLModel, table=True):
200
+ """A DESCRIPTION of one whole thread (ROADMAP_v2 §21c, M18).
201
+
202
+ NOT A COMPACTION, and the distinction is the entire reason this is a
203
+ separate table rather than a row in CompactionCheckpoint with a
204
+ different `strategy`. The two carry almost the same columns and mean
205
+ opposite things:
206
+
207
+ * a CompactionCheckpoint CHANGES WHAT THE MODEL SEES. `latest_checkpoint`
208
+ resolves by timestamp and feeds ConversationMemory._derived_view(), so
209
+ writing one is an edit to the live conversation.
210
+ * a ThreadSummary changes nothing. It is read by `/summary` to show a
211
+ person, and by `/ref` to inject into a DIFFERENT thread. The thread it
212
+ describes is untouched.
213
+
214
+ So a summary row in the checkpoint table would silently compact the
215
+ thread it was only meant to describe -- and it would win, because it
216
+ would be the newest row. That is the failure this table exists to make
217
+ impossible rather than to warn about.
218
+
219
+ `covers_up_to_message_id` is the last archive row the summary accounts
220
+ for, which is what makes staleness answerable: storage.advances() already
221
+ knows whether a thread has moved past a given position, so a `/ref` of an
222
+ unchanged thread costs no model call and a `/ref` of a grown one is
223
+ re-summarized. Same column name as the checkpoint's on purpose -- it means
224
+ the same thing there, and `advances()` serves both.
225
+ """
226
+
227
+ id: UUID = Field(default_factory=uuid4, primary_key=True)
228
+ thread_id: UUID = Field(foreign_key="conversationthread.id", index=True)
229
+ summary_text: str
230
+ covers_up_to_message_id: UUID
231
+ created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
232
+
233
+
234
+ class UserMemory(SQLModel, table=True):
235
+ """One durable fact, carried BETWEEN threads (ROADMAP_v2 §21b).
236
+
237
+ Unlike everything else in this file, a row here is not part of any one
238
+ conversation: `source_thread_id` records where it was learned, but the
239
+ row outlives that thread and is injected into later ones. That is the
240
+ whole point, and it is also why D26 gates `remember` and leaves `pin`
241
+ ungated -- a wrong pin costs some context budget inside a conversation
242
+ the user is watching, while a wrong memory silently shapes
243
+ conversations that have not started yet.
244
+
245
+ SCOPE IS TWO COLUMNS, not one (M12). §21's schema block carries only
246
+ `scope`, but D25 says a project-scoped memory is "keyed to the same
247
+ resolved project path the workspace-trust store uses" -- and with no
248
+ path recorded, `"project"` cannot tell two projects apart, which makes
249
+ §21 AC6 untestable rather than merely unenforced. `project_path` is the
250
+ realpath for `scope="project"` and None for `scope="global"`, taken
251
+ from config_loader.get_project_path() so "which project" means one
252
+ thing across trust, config and memory.
253
+
254
+ `category` is free text the model may set for its own grouping. It is
255
+ deliberately not an enum: the harness has no basis to decide what
256
+ kinds of fact are worth distinguishing, and a wrong enum would push
257
+ every real memory into "other".
258
+ """
259
+
260
+ id: UUID = Field(default_factory=uuid4, primary_key=True)
261
+ content: str
262
+ category: Optional[str] = None
263
+ # D25: "project" (default) or "global". Narrow by default because the
264
+ # two error directions are not symmetric -- a memory that should have
265
+ # been project-scoped but landed globally contaminates every later
266
+ # conversation with a stale fact about a codebase you may not be
267
+ # working in, and presents as the model simply being confident.
268
+ scope: str = Field(default="project", index=True)
269
+ project_path: Optional[str] = Field(default=None, index=True)
270
+ source_thread_id: UUID = Field(foreign_key="conversationthread.id")
271
+ created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
272
+
273
+
274
+ #: ROADMAP_v2 §27 (T1). The three things a thread can be. Not an Enum:
275
+ #: the column is a plain string so an unmigrated row reads back as data
276
+ #: rather than raising, and config.py's "plain values only" rule means a
277
+ #: vocabulary shared by storage and its callers belongs beside the table
278
+ #: that stores it.
279
+ THREAD_KIND_CHAT = "chat"
280
+ THREAD_KIND_RESEARCH_PASS = "research_pass"
281
+ THREAD_KIND_SUBAGENT = "subagent"
282
+
283
+
284
+ def create_thread(kind: str = THREAD_KIND_CHAT, *,
285
+ parent_thread_id: Optional[UUID] = None,
286
+ parent_call_id: Optional[str] = None,
287
+ agent_name: Optional[str] = None) -> UUID:
288
+ """Starts a brand-new conversation thread and returns its id.
289
+
290
+ `kind` defaults to "chat" (§27 AC1), so every existing caller keeps
291
+ creating conversations and only the paths that know they are creating
292
+ something else pass anything here.
293
+
294
+ The three lineage arguments (§47) are keyword-only and all default to
295
+ None, for the same reason: a caller that does not know it is creating
296
+ a child is not one, and should not be able to say so by accident in a
297
+ positional slot.
298
+ """
299
+ with Session(engine) as session:
300
+ thread = ConversationThread(
301
+ kind=kind, parent_thread_id=parent_thread_id,
302
+ parent_call_id=parent_call_id, agent_name=agent_name)
303
+ session.add(thread)
304
+ session.commit()
305
+ session.refresh(thread)
306
+ return thread.id
307
+
308
+
309
+ def get_thread(thread_id: UUID) -> Optional[dict]:
310
+ """The thread row as a plain column dict, or None if unknown (#31).
311
+
312
+ Used to confirm a thread_id is real before resuming it, and by
313
+ ConversationMemory to read extra_data and kind on resume. A PLAIN
314
+ DICT, like every other public read here: returning a detached ORM
315
+ instance worked only because nothing had expired it -- what a
316
+ detached instance hands back depends on what happened to be loaded,
317
+ which is exactly the question _ordered_rows' docstring says copying
318
+ columns removes.
319
+
320
+ THE `getattr` FALLBACKS COVER A NULL, NOT A MISSING COLUMN, and this
321
+ paragraph said the second thing until batch 75. It read: "a row read
322
+ from a database whose ALTER has not run yet has no attribute at all".
323
+ It cannot -- `session.get` SELECTs every mapped column, so a database
324
+ missing one raises OperationalError before any attribute is read, and
325
+ the class always has the attribute anyway. What these actually absorb
326
+ is a NULL on a row written before the column existed (and a hand-built
327
+ stand-in in a test), which is why each pairs the lookup with a default
328
+ the caller can use. The migration case belongs to
329
+ `database.ensure_columns()`, which runs at startup.
330
+
331
+ `kind` IS THE EXCEPTION, and the sentence above was wrong about it
332
+ until #257. ensure_columns has added `kind` with DEFAULT 'chat' since
333
+ §27 shipped it, and SQLite reads every row older than that ALTER as
334
+ the default -- so a migrated row arrives as 'chat' before any reader
335
+ runs (the lineage columns and `last_activity_at` have no SQL default
336
+ and really are NULL there). A NULL `kind` takes SQL written by hand,
337
+ on a database first migrated between §27 and #26 (2026-08-06 to
338
+ 08-22), where the column was added nullable. That is all the
339
+ `or THREAD_KIND_CHAT` here, in child_threads and in list_threads
340
+ absorbs; tests/test_storage_e2e.py builds that database and pins both
341
+ the default and the fallback. list_threads filters in SQL, so its
342
+ default picker never meets such a row at all.
343
+ """
344
+ with Session(engine) as session:
345
+ thread = session.get(ConversationThread, thread_id)
346
+ if thread is None:
347
+ return None
348
+ return {
349
+ "id": thread.id,
350
+ "created_at": thread.created_at,
351
+ "extra_data": dict(thread.extra_data or {}),
352
+ "kind": getattr(thread, "kind", None) or THREAD_KIND_CHAT,
353
+ "last_activity_at": getattr(thread, "last_activity_at", None),
354
+ # getattr for the kind fallback's reason, one column over --
355
+ # see the docstring for what that reason actually is. NULL on
356
+ # a row written before §47 means "no known parent", and a
357
+ # viewer showing that is better than one that raises.
358
+ "parent_thread_id": getattr(thread, "parent_thread_id", None),
359
+ "parent_call_id": getattr(thread, "parent_call_id", None),
360
+ "agent_name": getattr(thread, "agent_name", None),
361
+ }
362
+
363
+
364
+ def child_threads(parent_thread_id: UUID) -> list[dict]:
365
+ """The threads spawned by this one, oldest first (§47).
366
+
367
+ Each entry: ``{"id", "created_at", "kind", "parent_call_id",
368
+ "agent_name"}``. Empty for a thread that spawned nothing, which is
369
+ most of them.
370
+
371
+ ONE QUERY, not one per spawn line. A resumed conversation resolves
372
+ every `▸ spawn_subagent` line it draws, and asking per line would be
373
+ a query per row of a screen.
374
+
375
+ Oldest first because that is the order the calls happened in, so a
376
+ reader comparing this against the transcript walks both the same
377
+ way. `parent_call_id` is what pairs an entry to a specific line;
378
+ ordering is for the human, not for the lookup.
379
+
380
+ ORDERED BY `(created_at, id)`, which is `_ordered_rows`' rule and was
381
+ missing here (batch 75). That rule reads: timestamps have microsecond
382
+ resolution so ties are unlikely, but "unlikely" is not an ordering
383
+ guarantee. This is the query where it stopped being merely unlikely --
384
+ §47's NA9 makes three `create_thread` calls land from three threads at
385
+ once, which is the one place in this file that can actually tie.
386
+
387
+ NO `kind` FILTER, deliberately. Every child of a chat thread is
388
+ machinery of some sort, and which kinds exist is §27's question
389
+ rather than this one's -- a filter here would have to be widened
390
+ again the moment a sixth source appears, which is how §27's own
391
+ picker missed the compactor.
392
+ """
393
+ with Session(engine) as session:
394
+ statement = (
395
+ select(ConversationThread)
396
+ .where(ConversationThread.parent_thread_id == parent_thread_id)
397
+ .order_by(ConversationThread.created_at.asc(),
398
+ ConversationThread.id.asc())
399
+ )
400
+ return [
401
+ {
402
+ "id": t.id,
403
+ "created_at": t.created_at,
404
+ # A hand-written NULL only (#257); see get_thread.
405
+ "kind": getattr(t, "kind", None) or THREAD_KIND_CHAT,
406
+ "parent_call_id": getattr(t, "parent_call_id", None),
407
+ "agent_name": getattr(t, "agent_name", None),
408
+ }
409
+ for t in session.exec(statement).all()
410
+ ]
411
+
412
+
413
+ def get_thread_extra(thread_id: UUID) -> dict[str, Any]:
414
+ """Copy of the thread's extra_data JSON column (goal mode §18, todo
415
+ list §23). Raises ValueError for an unknown thread, same contract as
416
+ the other accessors."""
417
+ with Session(engine) as session:
418
+ thread = session.get(ConversationThread, thread_id)
419
+ if thread is None:
420
+ raise ValueError(f"No conversation thread found with id {thread_id}")
421
+ return dict(thread.extra_data or {})
422
+
423
+
424
+ def update_thread_extra(thread_id: UUID, key: str, value: Any) -> None:
425
+ """Set one key of the thread's extra_data; value=None deletes it.
426
+ Reassigns a fresh dict rather than mutating in place so SQLModel sees
427
+ the JSON column change."""
428
+ with Session(engine) as session:
429
+ thread = session.get(ConversationThread, thread_id)
430
+ if thread is None:
431
+ raise ValueError(f"No conversation thread found with id {thread_id}")
432
+ extra = dict(thread.extra_data or {})
433
+ if value is None:
434
+ extra.pop(key, None)
435
+ else:
436
+ extra[key] = value
437
+ thread.extra_data = extra
438
+ session.add(thread)
439
+ session.commit()
440
+
441
+
442
+ def save_message(
443
+ thread_id: UUID,
444
+ role: str,
445
+ content: Any,
446
+ name: Optional[str] = None,
447
+ tool_call_id: Optional[str] = None,
448
+ thinking: Optional[dict] = None,
449
+ harness: Optional[dict] = None,
450
+ images: Optional[list] = None,
451
+ ) -> None:
452
+ """Append one archived row and stamp the thread's activity (#32).
453
+
454
+ THE single production writer of MessageLog rows, which is what makes
455
+ it the right place to maintain `last_activity_at`: the stamp rides in
456
+ the same session and commit as the insert, so a message can never be
457
+ persisted without its thread moving to the top of the picker. Any
458
+ role counts (Q8b) -- a thread whose last event was tool output is
459
+ still the one you were just in. An unknown thread_id writes the
460
+ orphan message exactly as before; there is simply no row to stamp.
461
+ """
462
+ with Session(engine) as session:
463
+ new_message = MessageLog(
464
+ thread_id=thread_id,
465
+ role=role,
466
+ content=json.dumps(content),
467
+ name=name,
468
+ tool_call_id=tool_call_id,
469
+ # Encoded separately from `content` and stored as NULL when
470
+ # absent, so "this turn did no reasoning" costs no bytes and
471
+ # reads back as None rather than as an empty structure.
472
+ thinking=json.dumps(thinking) if thinking else None,
473
+ # ROADMAP_v3 §49 (SS5), on `thinking`'s terms: NULL for every
474
+ # row a person or the model wrote, which is nearly all of them.
475
+ harness=json.dumps(harness) if harness else None,
476
+ # ROADMAP_v3 §81 (IM2), on `thinking`'s terms again: NULL for
477
+ # every row that attached no image.
478
+ images=json.dumps(images) if images else None,
479
+ )
480
+ session.add(new_message)
481
+ thread = session.get(ConversationThread, thread_id)
482
+ if thread is not None:
483
+ thread.last_activity_at = datetime.now(UTC)
484
+ session.add(thread)
485
+ session.commit()
486
+
487
+
488
+ def list_threads(
489
+ kind: Optional[str] = THREAD_KIND_CHAT, limit: Optional[int] = None,
490
+ ) -> list[dict]:
491
+ """Conversation threads, most recently ACTIVE first (#32).
492
+
493
+ Each entry: ``{"id": UUID, "created_at": datetime, "kind": str,
494
+ "preview": str}``. Used by the CLI / TUI layer for thread browsing —
495
+ core/memory.py does NOT call this.
496
+
497
+ `limit` caps the result AFTER ordering (#30/#32 together). The default
498
+ of None is uncached and changes nothing for existing callers -- a
499
+ caller that wants everything still gets everything. The TUI picker
500
+ passes a cap because a modal listing thousands of conversations is its
501
+ own usability defect; with activity ordering the cap costs the least,
502
+ since the threads most likely to be wanted are exactly the ones that
503
+ survive it. Anything older stays reachable by id (/resume, --thread).
504
+
505
+ The order key is `last_activity_at`, falling back to `created_at`
506
+ when it is NULL -- never-received-a-message threads, and every row
507
+ that predates the column (the additive migration never backfills).
508
+ §27 made the picker correct by filtering and identifiable by preview;
509
+ ordering is the third axis: resume yesterday's long thread, work in
510
+ it an hour, and it stays pinned at the top rather than sinking below
511
+ three threads opened and abandoned this morning.
512
+
513
+ ROADMAP_v2 §27 AC2: CONVERSATIONS ONLY by default. One research run
514
+ creates ~10 pass threads plus up to 4 retry-round threads plus the
515
+ reviewer's, and every automatic compaction creates one more — so the
516
+ unfiltered list was mostly machinery, which is what made the picker
517
+ unusable. `kind=None` returns everything, for a caller that genuinely
518
+ wants the machinery (see §27's note on reaching a run's pass threads).
519
+
520
+ `preview` is the thread's first user message, truncated (§27's picker
521
+ decision). A row of bare uuid + timestamp is filterable but still not
522
+ recognisable, and identifying a conversation was the actual complaint.
523
+ It costs ONE extra query for the whole list rather than one per row --
524
+ and ONE ROW per thread, not one row per user message (#30).
525
+ """
526
+ with Session(engine) as session:
527
+ statement = select(ConversationThread)
528
+ if kind is not None:
529
+ statement = statement.where(ConversationThread.kind == kind)
530
+ statement = statement.order_by(
531
+ sa_func.coalesce(
532
+ ConversationThread.last_activity_at,
533
+ ConversationThread.created_at,
534
+ ).desc(),
535
+ ConversationThread.created_at.desc(),
536
+ )
537
+ if limit is not None:
538
+ statement = statement.limit(limit)
539
+ threads = session.exec(statement).all()
540
+ previews = _first_user_messages(session, [t.id for t in threads])
541
+ return [
542
+ {
543
+ "id": t.id,
544
+ "created_at": t.created_at,
545
+ # The fallback absorbs a hand-written NULL only: the
546
+ # migration fills a row older than the column with 'chat'
547
+ # (#257). get_thread's docstring has the whole of it. NOT a
548
+ # database whose ALTER has not run: the mapped class always
549
+ # has the attribute, and a database missing the column
550
+ # raises OperationalError at the SELECT above, before any
551
+ # attribute is read.
552
+ "kind": getattr(t, "kind", None) or THREAD_KIND_CHAT,
553
+ "preview": previews.get(t.id, ""),
554
+ }
555
+ for t in threads
556
+ ]
557
+
558
+
559
+ #: How much of a thread's first user message list_threads() carries. Long
560
+ #: enough to tell two conversations apart, short enough for one row.
561
+ _PREVIEW_CHARS = 70
562
+
563
+
564
+ def thread_preview(thread_id: UUID) -> str:
565
+ """The picker's preview for ONE thread (#171).
566
+
567
+ The same first-user-message text `list_threads()` computes -- read
568
+ from the ARCHIVE, so a compacted thread is labelled by what the user
569
+ actually said, never by §21a's synthesized summary (the reason T3
570
+ makes replay do the same) -- through the same one-row window query,
571
+ so a caller with a single id pays one row too.
572
+
573
+ A PUBLIC wrapper rather than a list_threads() scan because the CLI's
574
+ --ref takes ANY thread id (#171): the picker list is chat-kind-
575
+ filtered and capped at 200, and a pass thread or an older
576
+ conversation is exactly the thing worth referencing by name. Unknown
577
+ id -> "", which the reader renders as "(no preview)".
578
+ """
579
+ with Session(engine) as session:
580
+ return _first_user_messages(session, [thread_id]).get(thread_id, "")
581
+
582
+
583
+ def _first_user_messages(session, thread_ids: list[UUID]) -> dict[UUID, str]:
584
+ """thread id -> truncated first user message, for the given threads.
585
+
586
+ ONE query for the whole list, walked oldest-first so the first row seen
587
+ per thread is the earliest one. Reads the ARCHIVE (every MessageLog row
588
+ of the thread) rather than the derived view, for the same reason §27's
589
+ T3 makes replay do it: on a compacted thread the view begins with the
590
+ synthesized summary, and previewing THAT would label harness-generated
591
+ text as something the user said.
592
+
593
+ The "one query" claim is now true of ROWS too (#30): the first-user
594
+ message is picked by `ROW_NUMBER() OVER (PARTITION BY thread_id ...)`,
595
+ so SQLite materializes one user message per thread instead of loading
596
+ every user message of every thread into ORM objects and discarding all
597
+ but one in Python -- measured at 40x over-read on a realistic database
598
+ (~98% of the picker's cost). A plain LIMIT-per-thread loop would need
599
+ one query per thread; the window function keeps the count at one.
600
+ """
601
+ if not thread_ids:
602
+ return {}
603
+ ranked = (
604
+ select(
605
+ MessageLog.thread_id.label("thread_id"),
606
+ MessageLog.content.label("content"),
607
+ sa_func.row_number().over(
608
+ partition_by=MessageLog.thread_id,
609
+ order_by=(MessageLog.created_at.asc(), MessageLog.id.asc()),
610
+ ).label("rn"),
611
+ )
612
+ .where(MessageLog.role == "user")
613
+ .where(MessageLog.thread_id.in_(thread_ids))
614
+ .cte("first_user_messages_ranked")
615
+ )
616
+ rows = session.exec(
617
+ select(ranked.c.thread_id, ranked.c.content)
618
+ .where(ranked.c.rn == 1)
619
+ ).all()
620
+ out: dict[UUID, str] = {}
621
+ for thread_id, raw in rows:
622
+ try:
623
+ text = json.loads(raw)
624
+ except (TypeError, ValueError):
625
+ text = raw
626
+ text = " ".join(str(text).split())
627
+ out[thread_id] = (
628
+ text[:_PREVIEW_CHARS] + "…" if len(text) > _PREVIEW_CHARS else text
629
+ )
630
+ return out
631
+
632
+
633
+ def _decode_thinking(raw):
634
+ """A stored thinking record, or None for anything unusable.
635
+
636
+ Degrades rather than raising, like every other read on this path: the
637
+ column is a convenience for replay and for echoing blocks back to the
638
+ model that wrote them, and a row whose JSON cannot be parsed must cost
639
+ that convenience rather than the ability to resume the thread at all.
640
+ """
641
+ if not raw:
642
+ return None
643
+ try:
644
+ value = json.loads(raw)
645
+ except (TypeError, ValueError):
646
+ return None
647
+ return value if isinstance(value, dict) and value.get("blocks") else None
648
+
649
+
650
+ def _decode_harness(raw):
651
+ """A stored harness record (ROADMAP_v3 §49, SS5), or None for anything
652
+ unusable -- `_decode_thinking`'s rule for `_decode_thinking`'s reason: a
653
+ mark that cannot be read costs the mark, never the thread."""
654
+ if not raw:
655
+ return None
656
+ try:
657
+ value = json.loads(raw)
658
+ except (TypeError, ValueError):
659
+ return None
660
+ return value if isinstance(value, dict) and value.get("kind") else None
661
+
662
+
663
+ def _decode_images(raw):
664
+ """A tool row's stored images (ROADMAP_v3 §81, IM2), or None.
665
+
666
+ `_decode_thinking`'s rule: an entry that cannot be read costs that
667
+ entry, never the thread. An entry is usable when it names a media type
668
+ and carries data, both as non-empty strings -- what translation needs to
669
+ build a block, and nothing it would have to guess at.
670
+ """
671
+ if not raw:
672
+ return None
673
+ try:
674
+ value = json.loads(raw)
675
+ except (TypeError, ValueError):
676
+ return None
677
+ if not isinstance(value, list):
678
+ return None
679
+ usable = [entry for entry in value
680
+ if isinstance(entry, dict)
681
+ and isinstance(entry.get("media_type"), str)
682
+ and entry["media_type"]
683
+ and isinstance(entry.get("data"), str)
684
+ and entry["data"]]
685
+ return usable or None
686
+
687
+
688
+ def _to_neutral(msg: dict) -> dict:
689
+ """Reconstructs the exact neutral shape core/memory.py's add_* methods
690
+ write, per msg.role -- this is the counterpart to memory.py only ever
691
+ persisting the role-specific payload (see that file's docstring for
692
+ why there's no separate normalization step on the read side either).
693
+
694
+ ONE reconstruction point. Every read below is a filter over the same
695
+ rows fed through this function, so a change to the neutral shape
696
+ cannot reach one read and miss another -- which is the same reason
697
+ memory.py fixes shapes on the write side rather than per-role at each
698
+ reader. tests/conftest.py's FakeStorage mirrors this function; if it
699
+ changes, that changes identically.
700
+ """
701
+ decoded = json.loads(msg["content"])
702
+ role = msg["role"]
703
+
704
+ if role == "assistant":
705
+ # decoded is {"text": ..., "tool_calls": [...]} -- exactly
706
+ # what add_assistant_message persisted, nothing nested.
707
+ payload = {
708
+ "role": "assistant",
709
+ "text": decoded.get("text", ""),
710
+ "tool_calls": decoded.get("tool_calls", []),
711
+ }
712
+ # §44, from its own column rather than from `content`. Present
713
+ # ONLY when the row carried reasoning, so every message this
714
+ # project has ever written reconstructs to exactly the shape it
715
+ # did before -- which is what keeps _messages_for_provider,
716
+ # compactable_span and the fake's mirror unchanged for them.
717
+ thinking = _decode_thinking(msg.get("thinking"))
718
+ if thinking is not None:
719
+ payload["thinking"] = thinking
720
+ elif role == "tool":
721
+ # decoded is the plain result string add_tool_result persisted.
722
+ payload = {"role": "tool", "tool_call_id": msg["tool_call_id"], "content": decoded}
723
+ # ROADMAP_v3 §81 (IM2), on §44's terms for `thinking`: present ONLY
724
+ # when the row attached an image, so every tool row written before
725
+ # reconstructs to exactly the shape it did. Read by translation and
726
+ # the compactor's size proxy; never by anything that renders text.
727
+ images = _decode_images(msg.get("images"))
728
+ if images is not None:
729
+ payload["images"] = images
730
+ else:
731
+ # user (and any future plain-content role): decoded is
732
+ # already the right value for "content".
733
+ payload = {"role": role, "content": decoded}
734
+ if msg["tool_call_id"]:
735
+ payload["tool_call_id"] = msg["tool_call_id"]
736
+ # Batch 91, on §44's terms for `thinking`: present ONLY when the row
737
+ # records a failed run, so every message written before reconstructs
738
+ # to exactly the shape it did. Nothing that builds a request reads
739
+ # it -- _messages_for_provider takes a user row's `content` alone,
740
+ # and so does the compactor's _as_text.
741
+ if msg.get("error"):
742
+ payload["error"] = msg["error"]
743
+ # ROADMAP_v3 §49 (SS5), on the same terms: present ONLY when the HARNESS
744
+ # wrote the row, so every other message reconstructs exactly as it did.
745
+ # Read by replay and the compactor's labelling; never by translation.
746
+ harness = _decode_harness(msg.get("harness"))
747
+ if harness is not None:
748
+ payload["harness"] = harness
749
+
750
+ if msg["name"]:
751
+ payload["name"] = msg["name"]
752
+
753
+ return payload
754
+
755
+
756
+ def _ordered_rows(thread_id: UUID) -> list[dict]:
757
+ """Every row of a thread as plain column values, oldest first.
758
+
759
+ Ordered by (created_at, id) rather than created_at alone. Timestamps
760
+ have microsecond resolution so ties are unlikely, but "unlikely" is
761
+ not an ordering guarantee, and §21's watermark arithmetic slices this
762
+ list by position -- an unstable sort there would move the fold
763
+ boundary between two reads of the same thread.
764
+
765
+ Plain dicts rather than ORM instances because every caller below
766
+ reads these after the Session has closed, and what a detached
767
+ instance will still hand back depends on what happened to be loaded.
768
+ Copying six columns removes the question.
769
+
770
+ `pinned` is coerced to bool: a database migrated by
771
+ database.ensure_columns() gets the column with a DEFAULT, but a row
772
+ written by an older build through some other path could still read
773
+ back NULL, and None is not the same thing as "not pinned" to a
774
+ caller doing `if row["pinned"]`.
775
+ """
776
+ with Session(engine) as session:
777
+ statement = (
778
+ select(MessageLog)
779
+ .where(MessageLog.thread_id == thread_id)
780
+ .order_by(MessageLog.created_at.asc(), MessageLog.id.asc())
781
+ )
782
+ return [
783
+ {
784
+ "id": m.id, "role": m.role, "content": m.content,
785
+ "name": m.name, "tool_call_id": m.tool_call_id,
786
+ "pinned": bool(m.pinned),
787
+ # Not coerced, unlike `pinned`: NULL is the ordinary state
788
+ # of this column and _decode_thinking answers None for it.
789
+ "thinking": m.thinking,
790
+ # Batch 91, and NULL is ordinary here too: a run failed.
791
+ "error": m.error,
792
+ # ROADMAP_v3 §49 (SS5). NULL for every row a person or the
793
+ # model wrote.
794
+ "harness": m.harness,
795
+ # ROADMAP_v3 §81 (IM2). NULL for every row that attached no
796
+ # image; the compactor's size proxy reads it (IM8).
797
+ "images": m.images,
798
+ }
799
+ for m in session.exec(statement).all()
800
+ ]
801
+
802
+
803
+ def _split_at(rows: list[dict], message_id: Optional[UUID]) -> int:
804
+ """Index just past `message_id` in `rows`, or 0 when it isn't there.
805
+
806
+ Resolving a watermark by POSITION in the already-loaded, already-ordered
807
+ list rather than by a `(created_at, id) > (t, i)` SQL predicate. The
808
+ comparison a tuple-ordered watermark needs is awkward to express
809
+ portably over a UUID column, and a thread is at most a few thousand
810
+ rows that this function's callers are loading anyway. Simplicity wins
811
+ over a query that has to be right about UUID collation.
812
+
813
+ A watermark that names a row this thread does not have returns 0 --
814
+ i.e. nothing is treated as covered. That is the safe direction: the
815
+ caller shows the whole thread rather than silently hiding a prefix
816
+ on the strength of an id it could not find.
817
+ """
818
+ if message_id is None:
819
+ return 0
820
+ for index, row in enumerate(rows):
821
+ if row["id"] == message_id:
822
+ return index + 1
823
+ return 0
824
+
825
+
826
+ def get_session_history(
827
+ thread_id: UUID, after_message_id: Optional[UUID] = None,
828
+ ) -> list[dict]:
829
+ """The thread's messages in neutral shape, oldest first.
830
+
831
+ `after_message_id` (ROADMAP_v2 §21) loads only the UNCOMPACTED TAIL --
832
+ everything strictly after that row. The rows before it are not gone;
833
+ they are represented by a CompactionCheckpoint summary that
834
+ core/memory.py prepends, and they remain readable in full via
835
+ archive_history(). None (the default) returns the whole thread, which
836
+ is what every pre-§21 caller gets and why their behaviour is
837
+ unchanged.
838
+ """
839
+ rows = _ordered_rows(thread_id)
840
+ return [_to_neutral(m) for m in rows[_split_at(rows, after_message_id):]]
841
+
842
+
843
+ def archive_history(thread_id: UUID) -> list[dict]:
844
+ """Every message ever written to this thread, compaction or not.
845
+
846
+ This is ROADMAP_v2 §21 AC1 made greppable: compaction adds
847
+ CompactionCheckpoint rows and never edits or deletes a MessageLog
848
+ row, so this read is complete no matter how many times a thread has
849
+ been compacted. It is a distinct name rather than a comment on
850
+ get_session_history because the guarantee is what callers want to
851
+ find -- and because a future watermark default on that function
852
+ would silently take this one with it.
853
+ """
854
+ return get_session_history(thread_id)
855
+
856
+
857
+ def _turn_opener_index(rows: list[dict], span: list[dict]) -> list[int]:
858
+ """`[i]` for the user row that OPENS the turn a MID-TURN watermark cut
859
+ into, or `[]`. Batch 120, register item 40.
860
+
861
+ A sub-turn fold puts the watermark inside a turn, so the turn's own
862
+ request falls on the covered side and would be summarized away -- and
863
+ the model would then read a paraphrase of the question it is answering
864
+ rather than the question. That is M5's objection to folding the current
865
+ turn at all, and re-adding the request verbatim is what answers it.
866
+
867
+ The TEST IS THE FIRST KEPT ROW, not a flag on the checkpoint. When the
868
+ watermark is turn-aligned the tail opens with a `user` row and nothing
869
+ needs re-adding; when it is mid-turn the tail opens with an `assistant`
870
+ row, because that is the only boundary _sub_turn_span will place. So
871
+ the state is derivable from the rows, which means no column, no
872
+ migration, and no checkpoint written before this existed that lies
873
+ about its own shape.
874
+ """
875
+ if not span or len(span) >= len(rows):
876
+ return []
877
+ if rows[len(span)]["role"] == "user":
878
+ return []
879
+ for index in range(len(span) - 1, -1, -1):
880
+ if span[index]["role"] == "user":
881
+ return [index]
882
+ return []
883
+
884
+
885
+ def pinned_through(thread_id: UUID, message_id: Optional[UUID]) -> list[dict]:
886
+ """Rows at or before the watermark the derived view must carry
887
+ verbatim: the pinned set, made whole (§21 M9), plus the opening request
888
+ of a turn a mid-turn watermark cut into (batch 120).
889
+
890
+ A summary covers a contiguous span, but §21 AC2 says a pinned message is
891
+ never summarized -- so pinned rows INSIDE the covered span have to
892
+ come back verbatim, and a single `covers_up_to_message_id` cannot say
893
+ "everything through K except these". The derived view is therefore
894
+ summary + these rows + tail.
895
+
896
+ Empty when nothing in the span is pinned, which is every thread until
897
+ someone calls pin().
898
+
899
+ THE PINNED SPAN IS MADE WHOLE HERE, which is M4's rule applied to the
900
+ second boundary. The derived view has two of them and only the
901
+ checkpoint watermark is turn-aligned: `pin` is dispatched from inside a
902
+ turn, so the set it writes always ends mid-turn. D20 has already
903
+ persisted the assistant message carrying the `tool_use` for `pin`
904
+ itself, and the answering `tool` row is written after dispatch returns
905
+ -- so it is never pinned. While the pin is still in the uncompacted
906
+ tail this is invisible, because the tail carries both rows. The first
907
+ compaction whose watermark passes the pinned region activates it: the
908
+ assistant row comes back with an unanswered `tool_use`, its result
909
+ having gone into the summary text instead, and a `tool_result` with no
910
+ preceding `tool_use` is an HTTP 400 on Anthropic -- on a thread that
911
+ was working a moment ago, with no visible connection to a `pin` several
912
+ turns earlier. Durable, too: the checkpoint is a persisted row, so
913
+ every later call and every resume rebuilds the same unsendable view.
914
+
915
+ Fixed at the PRODUCER, where a pinned span becomes view content.
916
+ Stopping pin_last at the last complete turn would change what "pin the
917
+ last N turns" means and leave any other non-turn-aligned pinned set
918
+ open; dropping unanswered tool_calls in _derived_view is consumer-side
919
+ -- this project's canonical bug shape -- and would hide from the model
920
+ a call it actually made.
921
+ """
922
+ rows = _ordered_rows(thread_id)
923
+ span = rows[:_split_at(rows, message_id)]
924
+ neutral = [_to_neutral(m) for m in span]
925
+ return [neutral[i] for i in _pinned_indices(rows, span, neutral)]
926
+
927
+
928
+ def _pinned_indices(rows: list, span: list, neutral: list) -> list[int]:
929
+ """The positions in `span` that `pinned_through` returns, in archive
930
+ order: the pinned rows, the tool results that make them whole, and the
931
+ opener of a turn a mid-turn watermark cut into.
932
+
933
+ A PURE FUNCTION SINCE BATCH 216 (#231), split out of `pinned_through`
934
+ so `tests/test_fake_storage_mirror.py` can compare it against
935
+ `FakeStorage`'s copy the way it compares `_to_neutral` and `_split_at`
936
+ -- the fake transcribes this algorithm, and a transcription nothing
937
+ compares is a convention somebody has to remember. `rows` and `span`
938
+ need only `role` and `pinned`; `neutral` is `span` reconstructed.
939
+ """
940
+ keep = [i for i, m in enumerate(span) if m["pinned"]]
941
+ # `have`, the set() below and the role check are all BACKSTOPS, and the
942
+ # measurement says so: each was deleted separately against the full
943
+ # suite and each survived. What actually keeps the result duplicate-free
944
+ # is the `not m["pinned"]` filter -- a pinned tool row is already in
945
+ # `keep` and can never be re-added -- and reaching any of the three
946
+ # needs two rows sharing one tool_call_id, or a non-tool row carrying
947
+ # one, neither of which has a producer: add_tool_result writes exactly
948
+ # one row per call, and add_user_message passes no tool_call_id.
949
+ #
950
+ # Kept rather than trimmed, and the reasoning recorded rather than
951
+ # asserted: the failure they guard against is a duplicate tool_result,
952
+ # which is the same wire error from the other direction as the one this
953
+ # function exists to prevent. That is the wrong place to trade a free
954
+ # backstop for a reachability argument.
955
+ have = {neutral[i].get("tool_call_id") for i in keep}
956
+ need = {call["id"] for i in keep
957
+ for call in (neutral[i].get("tool_calls") or [])
958
+ if call["id"] not in have}
959
+ keep += [i for i, m in enumerate(span)
960
+ if not m["pinned"] and m["role"] == "tool"
961
+ and neutral[i].get("tool_call_id") in need]
962
+
963
+ # Batch 120. ONE INDEX, AND THE SORT BELOW PLACES IT -- which is the
964
+ # whole reason the turn opener lives here rather than in a function of
965
+ # its own beside this one. The view needs these rows in ARCHIVE ORDER,
966
+ # and `_derived_view` has no positions to merge two lists by: a pinned
967
+ # row from an older turn belongs before the request and a pinned row
968
+ # from inside this turn belongs after it, and either sibling-function
969
+ # arrangement gets one of those wrong. `set()` covers the opener
970
+ # already being pinned.
971
+ keep += _turn_opener_index(rows, span)
972
+
973
+ return sorted(set(keep))
974
+
975
+
976
+ def history_through(
977
+ thread_id: UUID, message_id: Optional[UUID],
978
+ after_message_id: Optional[UUID] = None,
979
+ ) -> list[dict]:
980
+ """The span a compaction is about to summarize, in neutral shape.
981
+
982
+ Pinned rows are EXCLUDED (§21 AC2: a pinned message is never included in
983
+ what the compactor is asked to summarize). `after_message_id` is what
984
+ makes chaining possible -- M2's chain strategy summarizes only what
985
+ followed the previous checkpoint, while rederive passes None and
986
+ summarizes the whole span from the top.
987
+ """
988
+ rows = _ordered_rows(thread_id)
989
+ start = _split_at(rows, after_message_id)
990
+ end = _split_at(rows, message_id)
991
+ return [_to_neutral(m) for m in rows[start:end] if not m["pinned"]]
992
+
993
+
994
+ def turn_start_ids(thread_id: UUID) -> list[UUID]:
995
+ """Ids of the rows that BEGIN each turn, oldest first.
996
+
997
+ A turn starts at a user message: the wrapper adds one, then the loop
998
+ appends assistant and tool rows until it stops. §21's M4 needs this
999
+ because a fold boundary that lands anywhere else can separate an
1000
+ assistant message from its tool results, and a tool_result with no
1001
+ preceding tool_use is an HTTP 400 on Anthropic -- a hard wire error,
1002
+ not a degraded answer. M5's "keep the last N turns" needs it too.
1003
+ """
1004
+ return [m["id"] for m in _ordered_rows(thread_id) if m["role"] == "user"]
1005
+
1006
+
1007
+ def message_ids_from(thread_id: UUID, message_id: UUID) -> list[UUID]:
1008
+ """Ids of `message_id` and everything after it. Resolves pin()'s
1009
+ ordinal ("the last N turns") into the real row ids the pinned flag is
1010
+ written against -- §21 keeps MessageLog.id an implementation detail
1011
+ the model never sees, so the ordinal-to-id mapping has to happen
1012
+ somewhere below the tool."""
1013
+ rows = _ordered_rows(thread_id)
1014
+ start = _split_at(rows, message_id)
1015
+ if not start:
1016
+ return []
1017
+ return [message_id] + [m["id"] for m in rows[start:]]
1018
+
1019
+
1020
+ def set_pinned(message_ids: list[UUID], pinned: bool = True) -> int:
1021
+ """Flag rows compaction-exempt. Returns how many rows changed."""
1022
+ if not message_ids:
1023
+ return 0
1024
+ changed = 0
1025
+ with Session(engine) as session:
1026
+ for message_id in message_ids:
1027
+ row = session.get(MessageLog, message_id)
1028
+ if row is None or row.pinned == pinned:
1029
+ continue
1030
+ row.pinned = pinned
1031
+ session.add(row)
1032
+ changed += 1
1033
+ session.commit()
1034
+ return changed
1035
+
1036
+
1037
+ def mark_turn_failed(thread_id: UUID, error: str) -> bool:
1038
+ """Record on the thread's NEWEST user row why the run it started failed.
1039
+
1040
+ Batch 91; see MessageLog.error for why this is a column. The newest user
1041
+ row is always the right one: every entry point writes the message that
1042
+ starts a run BEFORE the loop begins -- run_agent_conversation,
1043
+ continue_conversation, stream_deep_research_mode and the TUI turn -- so
1044
+ by the time a failure escapes the loop, the row that started it is the
1045
+ last user row there is. Ordered by (created_at, id), `_ordered_rows`'
1046
+ own order, so the two can never disagree about which row is newest.
1047
+
1048
+ Returns False for a thread with no user row. Does not stamp
1049
+ last_activity_at the way save_message does: a failure is not a message.
1050
+ """
1051
+ with Session(engine) as session:
1052
+ statement = (
1053
+ select(MessageLog)
1054
+ .where(MessageLog.thread_id == thread_id)
1055
+ .where(MessageLog.role == "user")
1056
+ .order_by(MessageLog.created_at.desc(), MessageLog.id.desc())
1057
+ )
1058
+ row = session.exec(statement).first()
1059
+ if row is None:
1060
+ return False
1061
+ row.error = error
1062
+ session.add(row)
1063
+ session.commit()
1064
+ return True
1065
+
1066
+
1067
+ def _current_watermark(thread_id: UUID) -> Optional[UUID]:
1068
+ """The live checkpoint's watermark, or None if never compacted."""
1069
+ checkpoint = latest_checkpoint(thread_id)
1070
+ return checkpoint["covers_up_to_message_id"] if checkpoint else None
1071
+
1072
+
1073
+ def advances(thread_id: UUID, current: Optional[UUID], proposed: UUID) -> bool:
1074
+ """Is `proposed` strictly later in this thread than `current`?
1075
+
1076
+ The ordering test behind §21's one real monotonic invariant: a
1077
+ compaction watermark only ever moves FORWARD, and -- #27's half --
1078
+ it must name a row THIS THREAD actually has. `current` of None
1079
+ (no checkpoint yet) skips the ordering comparison but not the
1080
+ containment one: a watermark we cannot place is one we must not
1081
+ write, and "first compaction" is exactly when nothing else would
1082
+ catch a foreign id.
1083
+
1084
+ A `proposed` this thread does not contain returns False -- the safe
1085
+ direction, since a watermark we cannot place is one we must not write.
1086
+ """
1087
+ rows = _ordered_rows(thread_id)
1088
+ if current is None:
1089
+ return _split_at(rows, proposed) > 0
1090
+ return _split_at(rows, proposed) > _split_at(rows, current) > 0
1091
+
1092
+
1093
+ def latest_checkpoint(thread_id: UUID) -> Optional[dict]:
1094
+ """The most recent compaction of this thread as a plain column dict,
1095
+ or None if it has never been compacted (which is every thread until
1096
+ the trigger first fires).
1097
+
1098
+ A dict rather than a detached ORM instance (#31): two of the three
1099
+ callers read it after their Session has closed, and one of them is
1100
+ the compaction path -- the one place in the project where a wrong
1101
+ read rewrites a live conversation. Copying the columns removes the
1102
+ question of what a detached instance still hands back.
1103
+ """
1104
+ with Session(engine) as session:
1105
+ statement = (
1106
+ select(CompactionCheckpoint)
1107
+ .where(CompactionCheckpoint.thread_id == thread_id)
1108
+ .order_by(CompactionCheckpoint.created_at.desc(),
1109
+ CompactionCheckpoint.id.desc())
1110
+ )
1111
+ row = next(iter(session.exec(statement).all()), None)
1112
+ if row is None:
1113
+ return None
1114
+ return {
1115
+ "id": row.id,
1116
+ "thread_id": row.thread_id,
1117
+ "summary_text": row.summary_text,
1118
+ "covers_up_to_message_id": row.covers_up_to_message_id,
1119
+ "strategy": row.strategy,
1120
+ "created_at": row.created_at,
1121
+ }
1122
+
1123
+
1124
+ def save_checkpoint(
1125
+ thread_id: UUID, summary_text: str, covers_up_to_message_id: UUID,
1126
+ strategy: str = "rederive",
1127
+ ) -> UUID:
1128
+ """Record one compaction. Adds a row; touches nothing else (§21 AC1).
1129
+
1130
+ REFUSES A WATERMARK THAT DOES NOT ADVANCE, returning the existing
1131
+ checkpoint's id instead. `latest_checkpoint` resolves ties by
1132
+ timestamp, so a backwards watermark written here does not merely fail
1133
+ to help -- it becomes the live one, and the thread UN-COMPACTS: the
1134
+ derived view jumps back to showing everything after the earlier
1135
+ boundary. That is what §21a shipped, and the caller-side guard in
1136
+ core/compaction.py is not enough on its own, because it only covers
1137
+ the one caller that exists today.
1138
+
1139
+ Enforced rather than documented, for the same reason D24's declared-
1140
+ permission check is: the invariant is cheap to state and its violation
1141
+ is silent and durable.
1142
+ """
1143
+ if not advances(thread_id, _current_watermark(thread_id),
1144
+ covers_up_to_message_id):
1145
+ existing = latest_checkpoint(thread_id)
1146
+ logger.warning(
1147
+ "Refused a compaction checkpoint for thread %s whose watermark "
1148
+ "does not advance on the existing one; keeping the existing "
1149
+ "checkpoint. This would have made the thread larger, not "
1150
+ "smaller.", thread_id)
1151
+ return existing["id"] if existing is not None else None
1152
+
1153
+ with Session(engine) as session:
1154
+ checkpoint = CompactionCheckpoint(
1155
+ thread_id=thread_id,
1156
+ summary_text=summary_text,
1157
+ covers_up_to_message_id=covers_up_to_message_id,
1158
+ strategy=strategy,
1159
+ )
1160
+ session.add(checkpoint)
1161
+ session.commit()
1162
+ session.refresh(checkpoint)
1163
+ return checkpoint.id
1164
+
1165
+
1166
+ # ---------------------------------------------------------------------------
1167
+ # ---- Thread summaries (ROADMAP_v2 §21c) -----------------------------------
1168
+ # ---------------------------------------------------------------------------
1169
+
1170
+ def last_message_id(thread_id: UUID) -> Optional[UUID]:
1171
+ """The id of this thread's most recent archive row, or None if empty.
1172
+
1173
+ §21c needs a position meaning "everything, as of now", to store as a
1174
+ summary's watermark and to compare against a stored one. Nothing existing
1175
+ answers it: `message_ids_from` needs a start id, `turn_start_ids` returns
1176
+ turn starts (and the last row is usually not one), and `_ordered_rows` is
1177
+ private for the reasons its docstring gives.
1178
+
1179
+ Reads the ARCHIVE, like everything else here -- the ordering is
1180
+ `(created_at, id)`, so this is the same last row `advances()` compares
1181
+ positions within.
1182
+ """
1183
+ rows = _ordered_rows(thread_id)
1184
+ return rows[-1]["id"] if rows else None
1185
+
1186
+
1187
+ def latest_thread_summary(thread_id: UUID) -> Optional[dict]:
1188
+ """The most recent summary of this thread as a plain column dict, or
1189
+ None if never summarized (#31 -- same reasoning as latest_checkpoint).
1190
+
1191
+ Newest by `(created_at, id)`, matching `latest_checkpoint` -- but note
1192
+ that the resemblance ends at the query. A stale row winning here shows
1193
+ someone an out-of-date description, which `advances()` then detects and
1194
+ corrects; a stale row winning in `latest_checkpoint` silently rewrites a
1195
+ live conversation. See ThreadSummary's docstring.
1196
+ """
1197
+ with Session(engine) as session:
1198
+ statement = (
1199
+ select(ThreadSummary)
1200
+ .where(ThreadSummary.thread_id == thread_id)
1201
+ .order_by(ThreadSummary.created_at.desc(), ThreadSummary.id.desc())
1202
+ )
1203
+ row = next(iter(session.exec(statement).all()), None)
1204
+ if row is None:
1205
+ return None
1206
+ return {
1207
+ "id": row.id,
1208
+ "thread_id": row.thread_id,
1209
+ "summary_text": row.summary_text,
1210
+ "covers_up_to_message_id": row.covers_up_to_message_id,
1211
+ "created_at": row.created_at,
1212
+ }
1213
+
1214
+
1215
+ def save_thread_summary(
1216
+ thread_id: UUID, summary_text: str, covers_up_to_message_id: UUID,
1217
+ ) -> UUID:
1218
+ """Record one summary of one thread. Adds a row; touches nothing else.
1219
+
1220
+ NO ADVANCE GUARD, deliberately, and this is the one place the difference
1221
+ from `save_checkpoint` above is load-bearing rather than decorative. That
1222
+ function REFUSES a watermark that does not advance, because a backwards
1223
+ compaction watermark becomes the live one and un-compacts the thread. A
1224
+ summary row changes what NO model sees: the worst a redundant or
1225
+ backwards one can do is describe the thread as it was a moment ago, which
1226
+ the next `advances()` check notices.
1227
+
1228
+ So the guard is absent because it would be cargo -- not because the
1229
+ monotonic property does not matter here. Do not add one, and do not route
1230
+ a summary through `save_checkpoint` to get one.
1231
+ """
1232
+ with Session(engine) as session:
1233
+ summary = ThreadSummary(
1234
+ thread_id=thread_id,
1235
+ summary_text=summary_text,
1236
+ covers_up_to_message_id=covers_up_to_message_id,
1237
+ )
1238
+ session.add(summary)
1239
+ session.commit()
1240
+ session.refresh(summary)
1241
+ return summary.id
1242
+
1243
+
1244
+ # ---------------------------------------------------------------------------
1245
+ # ---- Durable memory (ROADMAP_v2 §21b) -------------------------------------
1246
+ # ---------------------------------------------------------------------------
1247
+
1248
+ def save_memory(
1249
+ content: str, source_thread_id: UUID, scope: str = "project",
1250
+ category: Optional[str] = None, project_path: Optional[str] = None,
1251
+ ) -> UUID:
1252
+ """Record one durable fact. Returns its id.
1253
+
1254
+ `project_path` is required in substance for scope="project" and
1255
+ meaningless for "global" -- the caller resolves it, because this file
1256
+ knows nothing about config_loader or the trust store. memories/manager.py
1257
+ is the one place that decides.
1258
+ """
1259
+ with Session(engine) as session:
1260
+ row = UserMemory(
1261
+ content=content,
1262
+ category=category,
1263
+ scope=scope,
1264
+ project_path=project_path if scope == "project" else None,
1265
+ source_thread_id=source_thread_id,
1266
+ )
1267
+ session.add(row)
1268
+ session.commit()
1269
+ session.refresh(row)
1270
+ return row.id
1271
+
1272
+
1273
+ def list_memories(
1274
+ project_path: Optional[str] = None, scope: Optional[str] = None,
1275
+ ) -> list[dict]:
1276
+ """Memories visible from `project_path`, NEWEST FIRST.
1277
+
1278
+ Returns global memories plus the project ones recorded against exactly
1279
+ this path -- which is §21 AC6: a memory written in one project does not
1280
+ surface in a thread opened from another. `project_path=None` returns
1281
+ global memories only, which is the honest answer when no project has
1282
+ been resolved rather than a reason to show everything.
1283
+
1284
+ `scope` filters to one kind, for the management commands. Absent means
1285
+ both, which is what prompt assembly wants.
1286
+
1287
+ Both predicates are pushed INTO THE QUERY (#28): `UserMemory` declares
1288
+ indexes on `scope` and `project_path`, and this function used to load
1289
+ every row and filter in Python -- the exact pattern T1 rejected, run
1290
+ against two columns whose indexes had no reader. SQLite answers the
1291
+ visibility rule below from those indexes now.
1292
+
1293
+ Plain dicts rather than ORM instances, for the same reason
1294
+ _ordered_rows returns them: every caller reads these after the Session
1295
+ has closed.
1296
+ """
1297
+ with Session(engine) as session:
1298
+ statement = select(UserMemory)
1299
+ if scope is not None:
1300
+ statement = statement.where(UserMemory.scope == scope)
1301
+ # Visibility: anything not project-scoped travels; a project memory
1302
+ # only matches its own path. A NULL path therefore matches nothing
1303
+ # except when the caller has no resolved project either -- same as
1304
+ # the Python filter this replaced, edge included deliberately.
1305
+ statement = statement.where(
1306
+ (UserMemory.scope != "project")
1307
+ | (UserMemory.project_path == project_path))
1308
+ statement = statement.order_by(
1309
+ UserMemory.created_at.desc(), UserMemory.id.desc())
1310
+ return [
1311
+ {
1312
+ "id": row.id, "content": row.content,
1313
+ "category": row.category, "scope": row.scope,
1314
+ "project_path": row.project_path,
1315
+ "source_thread_id": row.source_thread_id,
1316
+ "created_at": row.created_at,
1317
+ }
1318
+ for row in session.exec(statement).all()
1319
+ ]
1320
+
1321
+
1322
+ def forget_memory(memory_id: UUID) -> bool:
1323
+ """Delete one memory. True if a row was removed.
1324
+
1325
+ A REAL DELETE, unlike everything else here. MessageLog is append-only
1326
+ because a conversation is a record of what happened; a UserMemory is a
1327
+ standing assertion the harness keeps repeating, and "this is no longer
1328
+ true" has no useful archived form. Returning False for an unknown id
1329
+ rather than raising: the id came from a human reading a list, and a
1330
+ typo deserves "no such memory", not a traceback.
1331
+ """
1332
+ with Session(engine) as session:
1333
+ row = session.get(UserMemory, memory_id)
1334
+ if row is None:
1335
+ return False
1336
+ session.delete(row)
1337
+ session.commit()
1338
+ return True