@yahyahammad/venastine 0.2.0 → 0.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/.env.example +19 -14
  2. package/CONFIG_ARCHITECTURE.md +2002 -0
  3. package/NOTICE +1 -1
  4. package/README.md +233 -94
  5. package/agents/builtin/build.md +73 -0
  6. package/agents/builtin/general.md +80 -0
  7. package/agents/builtin/plan.md +23 -5
  8. package/agents/builtin/test.md +65 -0
  9. package/agents/builtin/writer.md +67 -0
  10. package/agents/manager.py +261 -195
  11. package/agents/subagent_tool.py +277 -43
  12. package/agents/tui_commands.py +7 -12
  13. package/bin/venastine.mjs +528 -382
  14. package/bin/venastine.py +71 -0
  15. package/cli_commands.py +835 -0
  16. package/config.py +95 -1198
  17. package/config.yaml +1090 -0
  18. package/config_edit.py +1301 -0
  19. package/config_schema.py +966 -0
  20. package/config_update.py +749 -0
  21. package/core/agent_activity.py +158 -20
  22. package/core/ansi.py +452 -0
  23. package/core/approval.py +27 -5
  24. package/core/call_view.py +364 -0
  25. package/core/cancel.py +121 -0
  26. package/core/chatgpt_auth.py +747 -0
  27. package/core/claims_view.py +107 -0
  28. package/core/client.py +1238 -82
  29. package/core/commands.py +454 -0
  30. package/core/compaction.py +830 -66
  31. package/core/config_loader.py +516 -97
  32. package/core/connect.py +686 -0
  33. package/core/events.py +68 -6
  34. package/core/file_mentions.py +280 -0
  35. package/core/interaction.py +87 -4
  36. package/core/line_pattern.py +128 -0
  37. package/core/loop.py +1516 -440
  38. package/core/memory.py +120 -8
  39. package/core/pipeline_models.py +23 -9
  40. package/core/provider_errors.py +234 -0
  41. package/core/reasoning/confidence_scoring.py +70 -54
  42. package/core/reasoning/json_retry.py +19 -1
  43. package/core/reasoning/orchestrator.py +232 -53
  44. package/core/reasoning/output_writer.py +90 -31
  45. package/core/reasoning/payload_validation.py +133 -7
  46. package/core/reasoning/pipeline_storage.py +427 -361
  47. package/core/reasoning/review.py +659 -615
  48. package/core/reasoning/scholar.py +38 -14
  49. package/core/reasoning/source_corpus.py +110 -4
  50. package/core/replay.py +158 -26
  51. package/core/session.py +36 -12
  52. package/core/session_view.py +259 -0
  53. package/core/session_wake.py +259 -0
  54. package/core/shell_sessions.py +1553 -0
  55. package/core/workspace_trust.py +1 -1
  56. package/credentials.py +308 -180
  57. package/database.py +82 -1
  58. package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
  59. package/docs/SECURITY.md +108 -0
  60. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  61. package/env_secrets.py +118 -77
  62. package/json_store.py +236 -16
  63. package/logging_setup.py +121 -11
  64. package/main.py +1671 -199
  65. package/mcp_client/client.py +257 -23
  66. package/mcp_client/config.py +484 -379
  67. package/mcp_client/registration.py +163 -2
  68. package/memories/manager.py +15 -0
  69. package/memories/tui_commands.py +7 -12
  70. package/package.json +234 -4
  71. package/project_init/config_files.py +4 -0
  72. package/project_init/generator.py +201 -37
  73. package/project_init/manifest.py +418 -406
  74. package/project_init/tui_commands.py +7 -4
  75. package/prompts/revalidate.md +1 -1
  76. package/prompts/revise.md +17 -15
  77. package/prompts/system_prompts.py +64 -10
  78. package/providers.json.example +94 -92
  79. package/pyproject.toml +51 -2
  80. package/requirements.txt +252 -29
  81. package/safety/policy_enforcement.py +353 -17
  82. package/security/askpass.py +419 -0
  83. package/security/capability.py +37 -7
  84. package/security/confine.py +484 -0
  85. package/security/hosted.py +182 -0
  86. package/security/permissions.py +54 -7
  87. package/security/posture.py +124 -12
  88. package/security/protected_paths.py +198 -6
  89. package/security/sandbox.py +3262 -139
  90. package/security/secrets.py +855 -0
  91. package/skills/tui_commands.py +3 -4
  92. package/storage.py +368 -35
  93. package/tools/base.py +465 -305
  94. package/tools/builtin/_math_common.py +306 -16
  95. package/tools/builtin/_net_common.py +267 -61
  96. package/tools/builtin/arxiv.py +681 -223
  97. package/tools/builtin/ask_user.py +231 -213
  98. package/tools/builtin/discrete_math.py +35 -4
  99. package/tools/builtin/fetch_url.py +239 -21
  100. package/tools/builtin/file_ops.py +341 -33
  101. package/tools/builtin/geometry.py +184 -148
  102. package/tools/builtin/get_time.py +2 -2
  103. package/tools/builtin/linear_algebra.py +66 -23
  104. package/tools/builtin/load_skill.py +10 -0
  105. package/tools/builtin/logic.py +171 -12
  106. package/tools/builtin/pin.py +46 -12
  107. package/tools/builtin/probability_stats.py +224 -228
  108. package/tools/builtin/project_docs.py +80 -21
  109. package/tools/builtin/remember.py +21 -4
  110. package/tools/builtin/shell.py +586 -67
  111. package/tools/builtin/shell_sessions.py +613 -0
  112. package/tools/builtin/symbolic_math.py +80 -10
  113. package/tools/builtin/todo.py +5 -3
  114. package/tools/builtin/web_search.py +329 -151
  115. package/tools/context.py +23 -0
  116. package/tools/isolation.py +387 -314
  117. package/tools/registry.py +347 -30
  118. package/tui/app.py +4645 -438
  119. package/tui/app.tcss +185 -2
  120. package/tui/commands.py +10 -154
  121. package/tui/diffs.py +286 -265
  122. package/tui/hosted_driver.py +102 -0
  123. package/tui/markdown.py +60 -10
  124. package/tui/meters.py +103 -7
  125. package/tui/screens.py +346 -92
  126. package/tui/themes.py +59 -19
  127. package/tui/widgets.py +1575 -111
  128. package/SECURITY.md +0 -88
  129. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -17,7 +17,8 @@ manager, which holds no state of its own (K3).
17
17
 
18
18
  from agents.manager import manager as agents
19
19
  from skills.manager import manager as skills
20
- from tui.commands import SlashCommand, registry as commands
20
+ from tui.commands import bind
21
+ from tui.commands import registry as commands
21
22
 
22
23
 
23
24
  def _current_context(app):
@@ -106,6 +107,4 @@ def _cmd_skill(app, args: str) -> None:
106
107
  def register_skill_commands() -> None:
107
108
  """Idempotent -- registering by name overwrites, matching §16's
108
109
  register_builtin_commands() and §18's register_agent_commands()."""
109
- commands.register(SlashCommand(
110
- "skill", "activate a skill for this session", _cmd_skill,
111
- "[name|off <name>|clear]"))
110
+ commands.register(bind("skill", _cmd_skill))
package/storage.py CHANGED
@@ -1,11 +1,13 @@
1
1
  import json
2
2
  import logging
3
- from datetime import datetime, timezone
4
- from typing import List, Dict, Any, Optional
3
+ from datetime import UTC, datetime
4
+ from typing import Any, Optional
5
5
  from uuid import UUID, uuid4
6
6
 
7
- from sqlmodel import Field, SQLModel, JSON, Session, select
8
- from sqlalchemy import func as sa_func # real sqlalchemy; sqlmodel is faked in tests, this is not
7
+ from sqlalchemy import (
8
+ func as sa_func, # real sqlalchemy; sqlmodel is faked in tests, this is not
9
+ )
10
+ from sqlmodel import JSON, Field, Session, SQLModel, select
9
11
 
10
12
  from database import engine # your SQLAlchemy engine, assumed to exist here
11
13
 
@@ -14,8 +16,8 @@ logger = logging.getLogger(__name__)
14
16
 
15
17
  class ConversationThread(SQLModel, table=True):
16
18
  id: UUID = Field(default_factory=uuid4, primary_key=True)
17
- created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
18
- extra_data: Dict[str, Any] = Field(default_factory=dict, sa_type=JSON)
19
+ created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
20
+ extra_data: dict[str, Any] = Field(default_factory=dict, sa_type=JSON)
19
21
  # ROADMAP_v2 §27 (T1). What this thread IS: "chat" (a conversation a
20
22
  # human had), "research_pass" (one pass of the ten-pass pipeline) or
21
23
  # "subagent" (a spawned agent, the §20 reviewer, or the compactor).
@@ -48,6 +50,38 @@ class ConversationThread(SQLModel, table=True):
48
50
  # MessageLog rows in production and stamps this in the same session
49
51
  # it writes the message, so the two cannot drift.
50
52
  last_activity_at: Optional[datetime] = Field(default=None)
53
+ # ROADMAP_v2 §47. WHO spawned this thread, and at which call.
54
+ #
55
+ # Before these, the only trace of the edge was the child's uuid
56
+ # stringified inside the parent's tool-result row -- a repr inside a
57
+ # JSON blob, not a field, not queryable, and skipped by replay's T4
58
+ # anyway. A sidebar row and a `▸ spawn_subagent` line could describe
59
+ # a run nobody could then open.
60
+ #
61
+ # COLUMNS, not `extra_data`, for `kind`'s reason exactly (T1): the
62
+ # question asked of them is "which threads are children of this
63
+ # one", and answering that from an opaque JSON blob means loading
64
+ # every row and testing in Python. Not indexed, also for T1's
65
+ # reason -- ensure_columns() ALTERs a column in and builds no index,
66
+ # and create_all() adds none to a table that already exists, so
67
+ # index=True would mean an index on fresh databases and none on
68
+ # migrated ones.
69
+ #
70
+ # Nullable and additive, so database.ensure_columns() adds them to an
71
+ # existing app.db exactly as it added `pinned`, `kind`, `thinking`
72
+ # and `last_activity_at`. NULL is the honest value for every thread
73
+ # that predates them and for every one nothing spawned.
74
+ parent_thread_id: Optional[UUID] = Field(default=None)
75
+ # The parent's `tool_call_id` for the call that made this thread --
76
+ # the SAME id MessageLog stores on the parent's side, which is what
77
+ # lets one specific `▸ spawn_subagent` line find one specific child
78
+ # when a turn spawned three. None for a run no tool call made: the
79
+ # compactor, the reviewer, /init's initializer.
80
+ parent_call_id: Optional[str] = Field(default=None)
81
+ # Which agent ran, for a reader who has only the row. `kind` says
82
+ # what a thread IS; this says who it was. A viewer opening a stored
83
+ # thread cannot ask the span, because the span is long gone.
84
+ agent_name: Optional[str] = Field(default=None)
51
85
 
52
86
 
53
87
  class MessageLog(SQLModel, table=True):
@@ -57,7 +91,7 @@ class MessageLog(SQLModel, table=True):
57
91
  content: str # always JSON-encoded, regardless of the original type
58
92
  name: Optional[str] = None
59
93
  tool_call_id: Optional[str] = None
60
- created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
94
+ created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
61
95
  # ROADMAP_v2 §21. Compaction-exempt within this thread (D26's pin
62
96
  # tool). This is the first field this project has ever added to a
63
97
  # table that already exists on disk, which is why database.py grew
@@ -77,6 +111,57 @@ class MessageLog(SQLModel, table=True):
77
111
  # an existing database exactly as it added `pinned` -- see the note on
78
112
  # that field's migration in _ordered_rows.
79
113
  thinking: Optional[str] = None
114
+ # Batch 91. Why the run this row STARTED failed, or None -- user rows
115
+ # only, written by mark_turn_failed when an exception escapes the loop.
116
+ #
117
+ # A column on the user row rather than a row of its own, and that is
118
+ # the whole design. A failed run leaves no assistant turn behind (D20
119
+ # persists one only when a call returns), so without this a thread
120
+ # whose first call failed read as a request nobody answered. An `error`
121
+ # ROLE would have been a harness notice stored as conversation -- the
122
+ # compactor's `_as_text` and /summary render every role, so it would
123
+ # have been summarized into what the model later reads, which is M8's
124
+ # mistake. A column is read by nothing that measures the conversation:
125
+ # provider translation and the compactor take `content` alone. On the
126
+ # user row because that is what says WHICH turn failed, which a
127
+ # thread-level field could not for a chat. Nullable and additive, so
128
+ # database.ensure_columns() adds it like `thinking`.
129
+ error: Optional[str] = None
130
+ # ROADMAP_v3 §49 (SS5). What wrote this row when the HARNESS did, as a
131
+ # JSON object -- {"kind": "session_wake" | "session_killed_at_quit" |
132
+ # "session_held", "sessions": [{"id", "call_id", "shape"}, ...]} -- or
133
+ # None for everything a person or the model wrote.
134
+ #
135
+ # A column on a USER row, for batch 91's reason one field over. A
136
+ # finished background session has to reach the model as a message that
137
+ # starts a turn, and a tool result cannot carry it: every tool_use is
138
+ # answered in the step that made it (D20, NA11), long before the session
139
+ # ends. A user-role row is the one shape every provider accepts there,
140
+ # and a compacted thread already sends one the person never wrote (M8's
141
+ # summary). A `harness` ROLE would need translating and summarizing as
142
+ # something new; a column is read by nothing that builds a request, so
143
+ # the row goes out as the plain user message it is, while replay and
144
+ # the compactor can still tell it was not the person.
145
+ #
146
+ # NEVER beside a tool_call_id. `pinned_through` treats any kept row's
147
+ # tool_call_id as an answered call, so a wake row carrying its session's
148
+ # start call id would hide the real tool_result -- the HTTP 400 M4
149
+ # exists to prevent. The call id lives inside this JSON instead.
150
+ harness: Optional[str] = None
151
+ # ROADMAP_v3 §81 (IM2). The images a tool result carried, as a JSON list
152
+ # of {"media_type", "data"} with `data` base64 -- tool rows only, and only
153
+ # where `view_image` attached one. NULL everywhere else.
154
+ #
155
+ # A COLUMN rather than blocks inside `content`, for `thinking`'s reason:
156
+ # `content` is read as a string by every consumer of the conversation --
157
+ # replay, the transcript, /copy, the compactor, the summarizer -- and
158
+ # only two readers have any use for the bytes, provider translation and
159
+ # the compactor's size proxy. In the database rather than in a blob store
160
+ # or as a path to the workspace file: a path would make a resumed thread
161
+ # send whatever the file holds NOW, and a second store would be a second
162
+ # place docs/PRIVACY.md's "wipe everything" has to name. Nullable and
163
+ # additive, so database.ensure_columns() adds it like `thinking`.
164
+ images: Optional[str] = None
80
165
 
81
166
 
82
167
  class CompactionCheckpoint(SQLModel, table=True):
@@ -108,7 +193,7 @@ class CompactionCheckpoint(SQLModel, table=True):
108
193
  summary_text: str
109
194
  covers_up_to_message_id: UUID
110
195
  strategy: str = Field(default="rederive")
111
- created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
196
+ created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
112
197
 
113
198
 
114
199
  class ThreadSummary(SQLModel, table=True):
@@ -143,7 +228,7 @@ class ThreadSummary(SQLModel, table=True):
143
228
  thread_id: UUID = Field(foreign_key="conversationthread.id", index=True)
144
229
  summary_text: str
145
230
  covers_up_to_message_id: UUID
146
- created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
231
+ created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
147
232
 
148
233
 
149
234
  class UserMemory(SQLModel, table=True):
@@ -183,7 +268,7 @@ class UserMemory(SQLModel, table=True):
183
268
  scope: str = Field(default="project", index=True)
184
269
  project_path: Optional[str] = Field(default=None, index=True)
185
270
  source_thread_id: UUID = Field(foreign_key="conversationthread.id")
186
- created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
271
+ created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
187
272
 
188
273
 
189
274
  #: ROADMAP_v2 §27 (T1). The three things a thread can be. Not an Enum:
@@ -196,15 +281,25 @@ THREAD_KIND_RESEARCH_PASS = "research_pass"
196
281
  THREAD_KIND_SUBAGENT = "subagent"
197
282
 
198
283
 
199
- def create_thread(kind: str = THREAD_KIND_CHAT) -> UUID:
284
+ def create_thread(kind: str = THREAD_KIND_CHAT, *,
285
+ parent_thread_id: Optional[UUID] = None,
286
+ parent_call_id: Optional[str] = None,
287
+ agent_name: Optional[str] = None) -> UUID:
200
288
  """Starts a brand-new conversation thread and returns its id.
201
289
 
202
290
  `kind` defaults to "chat" (§27 AC1), so every existing caller keeps
203
291
  creating conversations and only the paths that know they are creating
204
292
  something else pass anything here.
293
+
294
+ The three lineage arguments (§47) are keyword-only and all default to
295
+ None, for the same reason: a caller that does not know it is creating
296
+ a child is not one, and should not be able to say so by accident in a
297
+ positional slot.
205
298
  """
206
299
  with Session(engine) as session:
207
- thread = ConversationThread(kind=kind)
300
+ thread = ConversationThread(
301
+ kind=kind, parent_thread_id=parent_thread_id,
302
+ parent_call_id=parent_call_id, agent_name=agent_name)
208
303
  session.add(thread)
209
304
  session.commit()
210
305
  session.refresh(thread)
@@ -222,9 +317,29 @@ def get_thread(thread_id: UUID) -> Optional[dict]:
222
317
  which is exactly the question _ordered_rows' docstring says copying
223
318
  columns removes.
224
319
 
225
- The kind fallback lives HERE rather than at the caller: a row read
226
- from a database whose ALTER has not run yet has no attribute at all,
227
- and the caller should not have to know that.
320
+ THE `getattr` FALLBACKS COVER A NULL, NOT A MISSING COLUMN, and this
321
+ paragraph said the second thing until batch 75. It read: "a row read
322
+ from a database whose ALTER has not run yet has no attribute at all".
323
+ It cannot -- `session.get` SELECTs every mapped column, so a database
324
+ missing one raises OperationalError before any attribute is read, and
325
+ the class always has the attribute anyway. What these actually absorb
326
+ is a NULL on a row written before the column existed (and a hand-built
327
+ stand-in in a test), which is why each pairs the lookup with a default
328
+ the caller can use. The migration case belongs to
329
+ `database.ensure_columns()`, which runs at startup.
330
+
331
+ `kind` IS THE EXCEPTION, and the sentence above was wrong about it
332
+ until #257. ensure_columns has added `kind` with DEFAULT 'chat' since
333
+ §27 shipped it, and SQLite reads every row older than that ALTER as
334
+ the default -- so a migrated row arrives as 'chat' before any reader
335
+ runs (the lineage columns and `last_activity_at` have no SQL default
336
+ and really are NULL there). A NULL `kind` takes SQL written by hand,
337
+ on a database first migrated between §27 and #26 (2026-08-06 to
338
+ 08-22), where the column was added nullable. That is all the
339
+ `or THREAD_KIND_CHAT` here, in child_threads and in list_threads
340
+ absorbs; tests/test_storage_e2e.py builds that database and pins both
341
+ the default and the fallback. list_threads filters in SQL, so its
342
+ default picker never meets such a row at all.
228
343
  """
229
344
  with Session(engine) as session:
230
345
  thread = session.get(ConversationThread, thread_id)
@@ -236,10 +351,66 @@ def get_thread(thread_id: UUID) -> Optional[dict]:
236
351
  "extra_data": dict(thread.extra_data or {}),
237
352
  "kind": getattr(thread, "kind", None) or THREAD_KIND_CHAT,
238
353
  "last_activity_at": getattr(thread, "last_activity_at", None),
354
+ # getattr for the kind fallback's reason, one column over --
355
+ # see the docstring for what that reason actually is. NULL on
356
+ # a row written before §47 means "no known parent", and a
357
+ # viewer showing that is better than one that raises.
358
+ "parent_thread_id": getattr(thread, "parent_thread_id", None),
359
+ "parent_call_id": getattr(thread, "parent_call_id", None),
360
+ "agent_name": getattr(thread, "agent_name", None),
239
361
  }
240
362
 
241
363
 
242
- def get_thread_extra(thread_id: UUID) -> Dict[str, Any]:
364
+ def child_threads(parent_thread_id: UUID) -> list[dict]:
365
+ """The threads spawned by this one, oldest first (§47).
366
+
367
+ Each entry: ``{"id", "created_at", "kind", "parent_call_id",
368
+ "agent_name"}``. Empty for a thread that spawned nothing, which is
369
+ most of them.
370
+
371
+ ONE QUERY, not one per spawn line. A resumed conversation resolves
372
+ every `▸ spawn_subagent` line it draws, and asking per line would be
373
+ a query per row of a screen.
374
+
375
+ Oldest first because that is the order the calls happened in, so a
376
+ reader comparing this against the transcript walks both the same
377
+ way. `parent_call_id` is what pairs an entry to a specific line;
378
+ ordering is for the human, not for the lookup.
379
+
380
+ ORDERED BY `(created_at, id)`, which is `_ordered_rows`' rule and was
381
+ missing here (batch 75). That rule reads: timestamps have microsecond
382
+ resolution so ties are unlikely, but "unlikely" is not an ordering
383
+ guarantee. This is the query where it stopped being merely unlikely --
384
+ §47's NA9 makes three `create_thread` calls land from three threads at
385
+ once, which is the one place in this file that can actually tie.
386
+
387
+ NO `kind` FILTER, deliberately. Every child of a chat thread is
388
+ machinery of some sort, and which kinds exist is §27's question
389
+ rather than this one's -- a filter here would have to be widened
390
+ again the moment a sixth source appears, which is how §27's own
391
+ picker missed the compactor.
392
+ """
393
+ with Session(engine) as session:
394
+ statement = (
395
+ select(ConversationThread)
396
+ .where(ConversationThread.parent_thread_id == parent_thread_id)
397
+ .order_by(ConversationThread.created_at.asc(),
398
+ ConversationThread.id.asc())
399
+ )
400
+ return [
401
+ {
402
+ "id": t.id,
403
+ "created_at": t.created_at,
404
+ # A hand-written NULL only (#257); see get_thread.
405
+ "kind": getattr(t, "kind", None) or THREAD_KIND_CHAT,
406
+ "parent_call_id": getattr(t, "parent_call_id", None),
407
+ "agent_name": getattr(t, "agent_name", None),
408
+ }
409
+ for t in session.exec(statement).all()
410
+ ]
411
+
412
+
413
+ def get_thread_extra(thread_id: UUID) -> dict[str, Any]:
243
414
  """Copy of the thread's extra_data JSON column (goal mode §18, todo
244
415
  list §23). Raises ValueError for an unknown thread, same contract as
245
416
  the other accessors."""
@@ -275,6 +446,8 @@ def save_message(
275
446
  name: Optional[str] = None,
276
447
  tool_call_id: Optional[str] = None,
277
448
  thinking: Optional[dict] = None,
449
+ harness: Optional[dict] = None,
450
+ images: Optional[list] = None,
278
451
  ) -> None:
279
452
  """Append one archived row and stamp the thread's activity (#32).
280
453
 
@@ -297,18 +470,24 @@ def save_message(
297
470
  # absent, so "this turn did no reasoning" costs no bytes and
298
471
  # reads back as None rather than as an empty structure.
299
472
  thinking=json.dumps(thinking) if thinking else None,
473
+ # ROADMAP_v3 §49 (SS5), on `thinking`'s terms: NULL for every
474
+ # row a person or the model wrote, which is nearly all of them.
475
+ harness=json.dumps(harness) if harness else None,
476
+ # ROADMAP_v3 §81 (IM2), on `thinking`'s terms again: NULL for
477
+ # every row that attached no image.
478
+ images=json.dumps(images) if images else None,
300
479
  )
301
480
  session.add(new_message)
302
481
  thread = session.get(ConversationThread, thread_id)
303
482
  if thread is not None:
304
- thread.last_activity_at = datetime.now(timezone.utc)
483
+ thread.last_activity_at = datetime.now(UTC)
305
484
  session.add(thread)
306
485
  session.commit()
307
486
 
308
487
 
309
488
  def list_threads(
310
489
  kind: Optional[str] = THREAD_KIND_CHAT, limit: Optional[int] = None,
311
- ) -> List[dict]:
490
+ ) -> list[dict]:
312
491
  """Conversation threads, most recently ACTIVE first (#32).
313
492
 
314
493
  Each entry: ``{"id": UUID, "created_at": datetime, "kind": str,
@@ -363,9 +542,13 @@ def list_threads(
363
542
  {
364
543
  "id": t.id,
365
544
  "created_at": t.created_at,
366
- # getattr, not t.kind: a row read back from a database
367
- # whose ALTER has not run yet has no attribute at all, and
368
- # a picker that raises is worse than one showing "chat".
545
+ # The fallback absorbs a hand-written NULL only: the
546
+ # migration fills a row older than the column with 'chat'
547
+ # (#257). get_thread's docstring has the whole of it. NOT a
548
+ # database whose ALTER has not run: the mapped class always
549
+ # has the attribute, and a database missing the column
550
+ # raises OperationalError at the SELECT above, before any
551
+ # attribute is read.
369
552
  "kind": getattr(t, "kind", None) or THREAD_KIND_CHAT,
370
553
  "preview": previews.get(t.id, ""),
371
554
  }
@@ -397,7 +580,7 @@ def thread_preview(thread_id: UUID) -> str:
397
580
  return _first_user_messages(session, [thread_id]).get(thread_id, "")
398
581
 
399
582
 
400
- def _first_user_messages(session, thread_ids: List[UUID]) -> Dict[UUID, str]:
583
+ def _first_user_messages(session, thread_ids: list[UUID]) -> dict[UUID, str]:
401
584
  """thread id -> truncated first user message, for the given threads.
402
585
 
403
586
  ONE query for the whole list, walked oldest-first so the first row seen
@@ -434,7 +617,7 @@ def _first_user_messages(session, thread_ids: List[UUID]) -> Dict[UUID, str]:
434
617
  select(ranked.c.thread_id, ranked.c.content)
435
618
  .where(ranked.c.rn == 1)
436
619
  ).all()
437
- out: Dict[UUID, str] = {}
620
+ out: dict[UUID, str] = {}
438
621
  for thread_id, raw in rows:
439
622
  try:
440
623
  text = json.loads(raw)
@@ -464,6 +647,44 @@ def _decode_thinking(raw):
464
647
  return value if isinstance(value, dict) and value.get("blocks") else None
465
648
 
466
649
 
650
+ def _decode_harness(raw):
651
+ """A stored harness record (ROADMAP_v3 §49, SS5), or None for anything
652
+ unusable -- `_decode_thinking`'s rule for `_decode_thinking`'s reason: a
653
+ mark that cannot be read costs the mark, never the thread."""
654
+ if not raw:
655
+ return None
656
+ try:
657
+ value = json.loads(raw)
658
+ except (TypeError, ValueError):
659
+ return None
660
+ return value if isinstance(value, dict) and value.get("kind") else None
661
+
662
+
663
+ def _decode_images(raw):
664
+ """A tool row's stored images (ROADMAP_v3 §81, IM2), or None.
665
+
666
+ `_decode_thinking`'s rule: an entry that cannot be read costs that
667
+ entry, never the thread. An entry is usable when it names a media type
668
+ and carries data, both as non-empty strings -- what translation needs to
669
+ build a block, and nothing it would have to guess at.
670
+ """
671
+ if not raw:
672
+ return None
673
+ try:
674
+ value = json.loads(raw)
675
+ except (TypeError, ValueError):
676
+ return None
677
+ if not isinstance(value, list):
678
+ return None
679
+ usable = [entry for entry in value
680
+ if isinstance(entry, dict)
681
+ and isinstance(entry.get("media_type"), str)
682
+ and entry["media_type"]
683
+ and isinstance(entry.get("data"), str)
684
+ and entry["data"]]
685
+ return usable or None
686
+
687
+
467
688
  def _to_neutral(msg: dict) -> dict:
468
689
  """Reconstructs the exact neutral shape core/memory.py's add_* methods
469
690
  write, per msg.role -- this is the counterpart to memory.py only ever
@@ -499,12 +720,32 @@ def _to_neutral(msg: dict) -> dict:
499
720
  elif role == "tool":
500
721
  # decoded is the plain result string add_tool_result persisted.
501
722
  payload = {"role": "tool", "tool_call_id": msg["tool_call_id"], "content": decoded}
723
+ # ROADMAP_v3 §81 (IM2), on §44's terms for `thinking`: present ONLY
724
+ # when the row attached an image, so every tool row written before
725
+ # reconstructs to exactly the shape it did. Read by translation and
726
+ # the compactor's size proxy; never by anything that renders text.
727
+ images = _decode_images(msg.get("images"))
728
+ if images is not None:
729
+ payload["images"] = images
502
730
  else:
503
731
  # user (and any future plain-content role): decoded is
504
732
  # already the right value for "content".
505
733
  payload = {"role": role, "content": decoded}
506
734
  if msg["tool_call_id"]:
507
735
  payload["tool_call_id"] = msg["tool_call_id"]
736
+ # Batch 91, on §44's terms for `thinking`: present ONLY when the row
737
+ # records a failed run, so every message written before reconstructs
738
+ # to exactly the shape it did. Nothing that builds a request reads
739
+ # it -- _messages_for_provider takes a user row's `content` alone,
740
+ # and so does the compactor's _as_text.
741
+ if msg.get("error"):
742
+ payload["error"] = msg["error"]
743
+ # ROADMAP_v3 §49 (SS5), on the same terms: present ONLY when the HARNESS
744
+ # wrote the row, so every other message reconstructs exactly as it did.
745
+ # Read by replay and the compactor's labelling; never by translation.
746
+ harness = _decode_harness(msg.get("harness"))
747
+ if harness is not None:
748
+ payload["harness"] = harness
508
749
 
509
750
  if msg["name"]:
510
751
  payload["name"] = msg["name"]
@@ -512,7 +753,7 @@ def _to_neutral(msg: dict) -> dict:
512
753
  return payload
513
754
 
514
755
 
515
- def _ordered_rows(thread_id: UUID) -> List[dict]:
756
+ def _ordered_rows(thread_id: UUID) -> list[dict]:
516
757
  """Every row of a thread as plain column values, oldest first.
517
758
 
518
759
  Ordered by (created_at, id) rather than created_at alone. Timestamps
@@ -546,12 +787,20 @@ def _ordered_rows(thread_id: UUID) -> List[dict]:
546
787
  # Not coerced, unlike `pinned`: NULL is the ordinary state
547
788
  # of this column and _decode_thinking answers None for it.
548
789
  "thinking": m.thinking,
790
+ # Batch 91, and NULL is ordinary here too: a run failed.
791
+ "error": m.error,
792
+ # ROADMAP_v3 §49 (SS5). NULL for every row a person or the
793
+ # model wrote.
794
+ "harness": m.harness,
795
+ # ROADMAP_v3 §81 (IM2). NULL for every row that attached no
796
+ # image; the compactor's size proxy reads it (IM8).
797
+ "images": m.images,
549
798
  }
550
799
  for m in session.exec(statement).all()
551
800
  ]
552
801
 
553
802
 
554
- def _split_at(rows: List[dict], message_id: Optional[UUID]) -> int:
803
+ def _split_at(rows: list[dict], message_id: Optional[UUID]) -> int:
555
804
  """Index just past `message_id` in `rows`, or 0 when it isn't there.
556
805
 
557
806
  Resolving a watermark by POSITION in the already-loaded, already-ordered
@@ -576,7 +825,7 @@ def _split_at(rows: List[dict], message_id: Optional[UUID]) -> int:
576
825
 
577
826
  def get_session_history(
578
827
  thread_id: UUID, after_message_id: Optional[UUID] = None,
579
- ) -> List[dict]:
828
+ ) -> list[dict]:
580
829
  """The thread's messages in neutral shape, oldest first.
581
830
 
582
831
  `after_message_id` (ROADMAP_v2 §21) loads only the UNCOMPACTED TAIL --
@@ -591,7 +840,7 @@ def get_session_history(
591
840
  return [_to_neutral(m) for m in rows[_split_at(rows, after_message_id):]]
592
841
 
593
842
 
594
- def archive_history(thread_id: UUID) -> List[dict]:
843
+ def archive_history(thread_id: UUID) -> list[dict]:
595
844
  """Every message ever written to this thread, compaction or not.
596
845
 
597
846
  This is ROADMAP_v2 §21 AC1 made greppable: compaction adds
@@ -605,8 +854,38 @@ def archive_history(thread_id: UUID) -> List[dict]:
605
854
  return get_session_history(thread_id)
606
855
 
607
856
 
608
- def pinned_through(thread_id: UUID, message_id: Optional[UUID]) -> List[dict]:
609
- """Pinned rows at or before the watermark, in neutral shape (§21 M9).
857
+ def _turn_opener_index(rows: list[dict], span: list[dict]) -> list[int]:
858
+ """`[i]` for the user row that OPENS the turn a MID-TURN watermark cut
859
+ into, or `[]`. Batch 120, register item 40.
860
+
861
+ A sub-turn fold puts the watermark inside a turn, so the turn's own
862
+ request falls on the covered side and would be summarized away -- and
863
+ the model would then read a paraphrase of the question it is answering
864
+ rather than the question. That is M5's objection to folding the current
865
+ turn at all, and re-adding the request verbatim is what answers it.
866
+
867
+ The TEST IS THE FIRST KEPT ROW, not a flag on the checkpoint. When the
868
+ watermark is turn-aligned the tail opens with a `user` row and nothing
869
+ needs re-adding; when it is mid-turn the tail opens with an `assistant`
870
+ row, because that is the only boundary _sub_turn_span will place. So
871
+ the state is derivable from the rows, which means no column, no
872
+ migration, and no checkpoint written before this existed that lies
873
+ about its own shape.
874
+ """
875
+ if not span or len(span) >= len(rows):
876
+ return []
877
+ if rows[len(span)]["role"] == "user":
878
+ return []
879
+ for index in range(len(span) - 1, -1, -1):
880
+ if span[index]["role"] == "user":
881
+ return [index]
882
+ return []
883
+
884
+
885
+ def pinned_through(thread_id: UUID, message_id: Optional[UUID]) -> list[dict]:
886
+ """Rows at or before the watermark the derived view must carry
887
+ verbatim: the pinned set, made whole (§21 M9), plus the opening request
888
+ of a turn a mid-turn watermark cut into (batch 120).
610
889
 
611
890
  A summary covers a contiguous span, but §21 AC2 says a pinned message is
612
891
  never summarized -- so pinned rows INSIDE the covered span have to
@@ -643,7 +922,21 @@ def pinned_through(thread_id: UUID, message_id: Optional[UUID]) -> List[dict]:
643
922
  rows = _ordered_rows(thread_id)
644
923
  span = rows[:_split_at(rows, message_id)]
645
924
  neutral = [_to_neutral(m) for m in span]
925
+ return [neutral[i] for i in _pinned_indices(rows, span, neutral)]
926
+
646
927
 
928
+ def _pinned_indices(rows: list, span: list, neutral: list) -> list[int]:
929
+ """The positions in `span` that `pinned_through` returns, in archive
930
+ order: the pinned rows, the tool results that make them whole, and the
931
+ opener of a turn a mid-turn watermark cut into.
932
+
933
+ A PURE FUNCTION SINCE BATCH 216 (#231), split out of `pinned_through`
934
+ so `tests/test_fake_storage_mirror.py` can compare it against
935
+ `FakeStorage`'s copy the way it compares `_to_neutral` and `_split_at`
936
+ -- the fake transcribes this algorithm, and a transcription nothing
937
+ compares is a convention somebody has to remember. `rows` and `span`
938
+ need only `role` and `pinned`; `neutral` is `span` reconstructed.
939
+ """
647
940
  keep = [i for i, m in enumerate(span) if m["pinned"]]
648
941
  # `have`, the set() below and the role check are all BACKSTOPS, and the
649
942
  # measurement says so: each was deleted separately against the full
@@ -667,13 +960,23 @@ def pinned_through(thread_id: UUID, message_id: Optional[UUID]) -> List[dict]:
667
960
  if not m["pinned"] and m["role"] == "tool"
668
961
  and neutral[i].get("tool_call_id") in need]
669
962
 
670
- return [neutral[i] for i in sorted(set(keep))]
963
+ # Batch 120. ONE INDEX, AND THE SORT BELOW PLACES IT -- which is the
964
+ # whole reason the turn opener lives here rather than in a function of
965
+ # its own beside this one. The view needs these rows in ARCHIVE ORDER,
966
+ # and `_derived_view` has no positions to merge two lists by: a pinned
967
+ # row from an older turn belongs before the request and a pinned row
968
+ # from inside this turn belongs after it, and either sibling-function
969
+ # arrangement gets one of those wrong. `set()` covers the opener
970
+ # already being pinned.
971
+ keep += _turn_opener_index(rows, span)
972
+
973
+ return sorted(set(keep))
671
974
 
672
975
 
673
976
  def history_through(
674
977
  thread_id: UUID, message_id: Optional[UUID],
675
978
  after_message_id: Optional[UUID] = None,
676
- ) -> List[dict]:
979
+ ) -> list[dict]:
677
980
  """The span a compaction is about to summarize, in neutral shape.
678
981
 
679
982
  Pinned rows are EXCLUDED (§21 AC2: a pinned message is never included in
@@ -688,7 +991,7 @@ def history_through(
688
991
  return [_to_neutral(m) for m in rows[start:end] if not m["pinned"]]
689
992
 
690
993
 
691
- def turn_start_ids(thread_id: UUID) -> List[UUID]:
994
+ def turn_start_ids(thread_id: UUID) -> list[UUID]:
692
995
  """Ids of the rows that BEGIN each turn, oldest first.
693
996
 
694
997
  A turn starts at a user message: the wrapper adds one, then the loop
@@ -701,7 +1004,7 @@ def turn_start_ids(thread_id: UUID) -> List[UUID]:
701
1004
  return [m["id"] for m in _ordered_rows(thread_id) if m["role"] == "user"]
702
1005
 
703
1006
 
704
- def message_ids_from(thread_id: UUID, message_id: UUID) -> List[UUID]:
1007
+ def message_ids_from(thread_id: UUID, message_id: UUID) -> list[UUID]:
705
1008
  """Ids of `message_id` and everything after it. Resolves pin()'s
706
1009
  ordinal ("the last N turns") into the real row ids the pinned flag is
707
1010
  written against -- §21 keeps MessageLog.id an implementation detail
@@ -714,7 +1017,7 @@ def message_ids_from(thread_id: UUID, message_id: UUID) -> List[UUID]:
714
1017
  return [message_id] + [m["id"] for m in rows[start:]]
715
1018
 
716
1019
 
717
- def set_pinned(message_ids: List[UUID], pinned: bool = True) -> int:
1020
+ def set_pinned(message_ids: list[UUID], pinned: bool = True) -> int:
718
1021
  """Flag rows compaction-exempt. Returns how many rows changed."""
719
1022
  if not message_ids:
720
1023
  return 0
@@ -731,6 +1034,36 @@ def set_pinned(message_ids: List[UUID], pinned: bool = True) -> int:
731
1034
  return changed
732
1035
 
733
1036
 
1037
+ def mark_turn_failed(thread_id: UUID, error: str) -> bool:
1038
+ """Record on the thread's NEWEST user row why the run it started failed.
1039
+
1040
+ Batch 91; see MessageLog.error for why this is a column. The newest user
1041
+ row is always the right one: every entry point writes the message that
1042
+ starts a run BEFORE the loop begins -- run_agent_conversation,
1043
+ continue_conversation, stream_deep_research_mode and the TUI turn -- so
1044
+ by the time a failure escapes the loop, the row that started it is the
1045
+ last user row there is. Ordered by (created_at, id), `_ordered_rows`'
1046
+ own order, so the two can never disagree about which row is newest.
1047
+
1048
+ Returns False for a thread with no user row. Does not stamp
1049
+ last_activity_at the way save_message does: a failure is not a message.
1050
+ """
1051
+ with Session(engine) as session:
1052
+ statement = (
1053
+ select(MessageLog)
1054
+ .where(MessageLog.thread_id == thread_id)
1055
+ .where(MessageLog.role == "user")
1056
+ .order_by(MessageLog.created_at.desc(), MessageLog.id.desc())
1057
+ )
1058
+ row = session.exec(statement).first()
1059
+ if row is None:
1060
+ return False
1061
+ row.error = error
1062
+ session.add(row)
1063
+ session.commit()
1064
+ return True
1065
+
1066
+
734
1067
  def _current_watermark(thread_id: UUID) -> Optional[UUID]:
735
1068
  """The live checkpoint's watermark, or None if never compacted."""
736
1069
  checkpoint = latest_checkpoint(thread_id)
@@ -939,7 +1272,7 @@ def save_memory(
939
1272
 
940
1273
  def list_memories(
941
1274
  project_path: Optional[str] = None, scope: Optional[str] = None,
942
- ) -> List[dict]:
1275
+ ) -> list[dict]:
943
1276
  """Memories visible from `project_path`, NEWEST FIRST.
944
1277
 
945
1278
  Returns global memories plus the project ones recorded against exactly