@yahyahammad/venastine 0.2.0 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -14
- package/CONFIG_ARCHITECTURE.md +2002 -0
- package/NOTICE +1 -1
- package/README.md +233 -94
- package/agents/builtin/build.md +73 -0
- package/agents/builtin/general.md +80 -0
- package/agents/builtin/plan.md +23 -5
- package/agents/builtin/test.md +65 -0
- package/agents/builtin/writer.md +67 -0
- package/agents/manager.py +261 -195
- package/agents/subagent_tool.py +277 -43
- package/agents/tui_commands.py +7 -12
- package/bin/venastine.mjs +528 -382
- package/bin/venastine.py +71 -0
- package/cli_commands.py +835 -0
- package/config.py +95 -1198
- package/config.yaml +1090 -0
- package/config_edit.py +1301 -0
- package/config_schema.py +966 -0
- package/config_update.py +749 -0
- package/core/agent_activity.py +158 -20
- package/core/ansi.py +452 -0
- package/core/approval.py +27 -5
- package/core/call_view.py +364 -0
- package/core/cancel.py +121 -0
- package/core/chatgpt_auth.py +747 -0
- package/core/claims_view.py +107 -0
- package/core/client.py +1238 -82
- package/core/commands.py +454 -0
- package/core/compaction.py +830 -66
- package/core/config_loader.py +516 -97
- package/core/connect.py +686 -0
- package/core/events.py +68 -6
- package/core/file_mentions.py +280 -0
- package/core/interaction.py +87 -4
- package/core/line_pattern.py +128 -0
- package/core/loop.py +1516 -440
- package/core/memory.py +120 -8
- package/core/pipeline_models.py +23 -9
- package/core/provider_errors.py +234 -0
- package/core/reasoning/confidence_scoring.py +70 -54
- package/core/reasoning/json_retry.py +19 -1
- package/core/reasoning/orchestrator.py +232 -53
- package/core/reasoning/output_writer.py +90 -31
- package/core/reasoning/payload_validation.py +133 -7
- package/core/reasoning/pipeline_storage.py +427 -361
- package/core/reasoning/review.py +659 -615
- package/core/reasoning/scholar.py +38 -14
- package/core/reasoning/source_corpus.py +110 -4
- package/core/replay.py +158 -26
- package/core/session.py +36 -12
- package/core/session_view.py +259 -0
- package/core/session_wake.py +259 -0
- package/core/shell_sessions.py +1553 -0
- package/core/workspace_trust.py +1 -1
- package/credentials.py +308 -180
- package/database.py +82 -1
- package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
- package/docs/SECURITY.md +108 -0
- package/docs/THIRD_PARTY_NOTICES.md +148 -0
- package/env_secrets.py +118 -77
- package/json_store.py +236 -16
- package/logging_setup.py +121 -11
- package/main.py +1671 -199
- package/mcp_client/client.py +257 -23
- package/mcp_client/config.py +484 -379
- package/mcp_client/registration.py +163 -2
- package/memories/manager.py +15 -0
- package/memories/tui_commands.py +7 -12
- package/package.json +234 -4
- package/project_init/config_files.py +4 -0
- package/project_init/generator.py +201 -37
- package/project_init/manifest.py +418 -406
- package/project_init/tui_commands.py +7 -4
- package/prompts/revalidate.md +1 -1
- package/prompts/revise.md +17 -15
- package/prompts/system_prompts.py +64 -10
- package/providers.json.example +94 -92
- package/pyproject.toml +51 -2
- package/requirements.txt +252 -29
- package/safety/policy_enforcement.py +353 -17
- package/security/askpass.py +419 -0
- package/security/capability.py +37 -7
- package/security/confine.py +484 -0
- package/security/hosted.py +182 -0
- package/security/permissions.py +54 -7
- package/security/posture.py +124 -12
- package/security/protected_paths.py +198 -6
- package/security/sandbox.py +3262 -139
- package/security/secrets.py +855 -0
- package/skills/tui_commands.py +3 -4
- package/storage.py +368 -35
- package/tools/base.py +465 -305
- package/tools/builtin/_math_common.py +306 -16
- package/tools/builtin/_net_common.py +267 -61
- package/tools/builtin/arxiv.py +681 -223
- package/tools/builtin/ask_user.py +231 -213
- package/tools/builtin/discrete_math.py +35 -4
- package/tools/builtin/fetch_url.py +239 -21
- package/tools/builtin/file_ops.py +341 -33
- package/tools/builtin/geometry.py +184 -148
- package/tools/builtin/get_time.py +2 -2
- package/tools/builtin/linear_algebra.py +66 -23
- package/tools/builtin/load_skill.py +10 -0
- package/tools/builtin/logic.py +171 -12
- package/tools/builtin/pin.py +46 -12
- package/tools/builtin/probability_stats.py +224 -228
- package/tools/builtin/project_docs.py +80 -21
- package/tools/builtin/remember.py +21 -4
- package/tools/builtin/shell.py +586 -67
- package/tools/builtin/shell_sessions.py +613 -0
- package/tools/builtin/symbolic_math.py +80 -10
- package/tools/builtin/todo.py +5 -3
- package/tools/builtin/web_search.py +329 -151
- package/tools/context.py +23 -0
- package/tools/isolation.py +387 -314
- package/tools/registry.py +347 -30
- package/tui/app.py +4645 -438
- package/tui/app.tcss +185 -2
- package/tui/commands.py +10 -154
- package/tui/diffs.py +286 -265
- package/tui/hosted_driver.py +102 -0
- package/tui/markdown.py +60 -10
- package/tui/meters.py +103 -7
- package/tui/screens.py +346 -92
- package/tui/themes.py +59 -19
- package/tui/widgets.py +1575 -111
- package/SECURITY.md +0 -88
- package/THIRD_PARTY_NOTICES.md +0 -100
package/skills/tui_commands.py
CHANGED
|
@@ -17,7 +17,8 @@ manager, which holds no state of its own (K3).
|
|
|
17
17
|
|
|
18
18
|
from agents.manager import manager as agents
|
|
19
19
|
from skills.manager import manager as skills
|
|
20
|
-
from tui.commands import
|
|
20
|
+
from tui.commands import bind
|
|
21
|
+
from tui.commands import registry as commands
|
|
21
22
|
|
|
22
23
|
|
|
23
24
|
def _current_context(app):
|
|
@@ -106,6 +107,4 @@ def _cmd_skill(app, args: str) -> None:
|
|
|
106
107
|
def register_skill_commands() -> None:
|
|
107
108
|
"""Idempotent -- registering by name overwrites, matching §16's
|
|
108
109
|
register_builtin_commands() and §18's register_agent_commands()."""
|
|
109
|
-
commands.register(
|
|
110
|
-
"skill", "activate a skill for this session", _cmd_skill,
|
|
111
|
-
"[name|off <name>|clear]"))
|
|
110
|
+
commands.register(bind("skill", _cmd_skill))
|
package/storage.py
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import logging
|
|
3
|
-
from datetime import
|
|
4
|
-
from typing import
|
|
3
|
+
from datetime import UTC, datetime
|
|
4
|
+
from typing import Any, Optional
|
|
5
5
|
from uuid import UUID, uuid4
|
|
6
6
|
|
|
7
|
-
from
|
|
8
|
-
|
|
7
|
+
from sqlalchemy import (
|
|
8
|
+
func as sa_func, # real sqlalchemy; sqlmodel is faked in tests, this is not
|
|
9
|
+
)
|
|
10
|
+
from sqlmodel import JSON, Field, Session, SQLModel, select
|
|
9
11
|
|
|
10
12
|
from database import engine # your SQLAlchemy engine, assumed to exist here
|
|
11
13
|
|
|
@@ -14,8 +16,8 @@ logger = logging.getLogger(__name__)
|
|
|
14
16
|
|
|
15
17
|
class ConversationThread(SQLModel, table=True):
|
|
16
18
|
id: UUID = Field(default_factory=uuid4, primary_key=True)
|
|
17
|
-
created_at: datetime = Field(default_factory=lambda: datetime.now(
|
|
18
|
-
extra_data:
|
|
19
|
+
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
|
20
|
+
extra_data: dict[str, Any] = Field(default_factory=dict, sa_type=JSON)
|
|
19
21
|
# ROADMAP_v2 §27 (T1). What this thread IS: "chat" (a conversation a
|
|
20
22
|
# human had), "research_pass" (one pass of the ten-pass pipeline) or
|
|
21
23
|
# "subagent" (a spawned agent, the §20 reviewer, or the compactor).
|
|
@@ -48,6 +50,38 @@ class ConversationThread(SQLModel, table=True):
|
|
|
48
50
|
# MessageLog rows in production and stamps this in the same session
|
|
49
51
|
# it writes the message, so the two cannot drift.
|
|
50
52
|
last_activity_at: Optional[datetime] = Field(default=None)
|
|
53
|
+
# ROADMAP_v2 §47. WHO spawned this thread, and at which call.
|
|
54
|
+
#
|
|
55
|
+
# Before these, the only trace of the edge was the child's uuid
|
|
56
|
+
# stringified inside the parent's tool-result row -- a repr inside a
|
|
57
|
+
# JSON blob, not a field, not queryable, and skipped by replay's T4
|
|
58
|
+
# anyway. A sidebar row and a `▸ spawn_subagent` line could describe
|
|
59
|
+
# a run nobody could then open.
|
|
60
|
+
#
|
|
61
|
+
# COLUMNS, not `extra_data`, for `kind`'s reason exactly (T1): the
|
|
62
|
+
# question asked of them is "which threads are children of this
|
|
63
|
+
# one", and answering that from an opaque JSON blob means loading
|
|
64
|
+
# every row and testing in Python. Not indexed, also for T1's
|
|
65
|
+
# reason -- ensure_columns() ALTERs a column in and builds no index,
|
|
66
|
+
# and create_all() adds none to a table that already exists, so
|
|
67
|
+
# index=True would mean an index on fresh databases and none on
|
|
68
|
+
# migrated ones.
|
|
69
|
+
#
|
|
70
|
+
# Nullable and additive, so database.ensure_columns() adds them to an
|
|
71
|
+
# existing app.db exactly as it added `pinned`, `kind`, `thinking`
|
|
72
|
+
# and `last_activity_at`. NULL is the honest value for every thread
|
|
73
|
+
# that predates them and for every one nothing spawned.
|
|
74
|
+
parent_thread_id: Optional[UUID] = Field(default=None)
|
|
75
|
+
# The parent's `tool_call_id` for the call that made this thread --
|
|
76
|
+
# the SAME id MessageLog stores on the parent's side, which is what
|
|
77
|
+
# lets one specific `▸ spawn_subagent` line find one specific child
|
|
78
|
+
# when a turn spawned three. None for a run no tool call made: the
|
|
79
|
+
# compactor, the reviewer, /init's initializer.
|
|
80
|
+
parent_call_id: Optional[str] = Field(default=None)
|
|
81
|
+
# Which agent ran, for a reader who has only the row. `kind` says
|
|
82
|
+
# what a thread IS; this says who it was. A viewer opening a stored
|
|
83
|
+
# thread cannot ask the span, because the span is long gone.
|
|
84
|
+
agent_name: Optional[str] = Field(default=None)
|
|
51
85
|
|
|
52
86
|
|
|
53
87
|
class MessageLog(SQLModel, table=True):
|
|
@@ -57,7 +91,7 @@ class MessageLog(SQLModel, table=True):
|
|
|
57
91
|
content: str # always JSON-encoded, regardless of the original type
|
|
58
92
|
name: Optional[str] = None
|
|
59
93
|
tool_call_id: Optional[str] = None
|
|
60
|
-
created_at: datetime = Field(default_factory=lambda: datetime.now(
|
|
94
|
+
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
|
61
95
|
# ROADMAP_v2 §21. Compaction-exempt within this thread (D26's pin
|
|
62
96
|
# tool). This is the first field this project has ever added to a
|
|
63
97
|
# table that already exists on disk, which is why database.py grew
|
|
@@ -77,6 +111,57 @@ class MessageLog(SQLModel, table=True):
|
|
|
77
111
|
# an existing database exactly as it added `pinned` -- see the note on
|
|
78
112
|
# that field's migration in _ordered_rows.
|
|
79
113
|
thinking: Optional[str] = None
|
|
114
|
+
# Batch 91. Why the run this row STARTED failed, or None -- user rows
|
|
115
|
+
# only, written by mark_turn_failed when an exception escapes the loop.
|
|
116
|
+
#
|
|
117
|
+
# A column on the user row rather than a row of its own, and that is
|
|
118
|
+
# the whole design. A failed run leaves no assistant turn behind (D20
|
|
119
|
+
# persists one only when a call returns), so without this a thread
|
|
120
|
+
# whose first call failed read as a request nobody answered. An `error`
|
|
121
|
+
# ROLE would have been a harness notice stored as conversation -- the
|
|
122
|
+
# compactor's `_as_text` and /summary render every role, so it would
|
|
123
|
+
# have been summarized into what the model later reads, which is M8's
|
|
124
|
+
# mistake. A column is read by nothing that measures the conversation:
|
|
125
|
+
# provider translation and the compactor take `content` alone. On the
|
|
126
|
+
# user row because that is what says WHICH turn failed, which a
|
|
127
|
+
# thread-level field could not for a chat. Nullable and additive, so
|
|
128
|
+
# database.ensure_columns() adds it like `thinking`.
|
|
129
|
+
error: Optional[str] = None
|
|
130
|
+
# ROADMAP_v3 §49 (SS5). What wrote this row when the HARNESS did, as a
|
|
131
|
+
# JSON object -- {"kind": "session_wake" | "session_killed_at_quit" |
|
|
132
|
+
# "session_held", "sessions": [{"id", "call_id", "shape"}, ...]} -- or
|
|
133
|
+
# None for everything a person or the model wrote.
|
|
134
|
+
#
|
|
135
|
+
# A column on a USER row, for batch 91's reason one field over. A
|
|
136
|
+
# finished background session has to reach the model as a message that
|
|
137
|
+
# starts a turn, and a tool result cannot carry it: every tool_use is
|
|
138
|
+
# answered in the step that made it (D20, NA11), long before the session
|
|
139
|
+
# ends. A user-role row is the one shape every provider accepts there,
|
|
140
|
+
# and a compacted thread already sends one the person never wrote (M8's
|
|
141
|
+
# summary). A `harness` ROLE would need translating and summarizing as
|
|
142
|
+
# something new; a column is read by nothing that builds a request, so
|
|
143
|
+
# the row goes out as the plain user message it is, while replay and
|
|
144
|
+
# the compactor can still tell it was not the person.
|
|
145
|
+
#
|
|
146
|
+
# NEVER beside a tool_call_id. `pinned_through` treats any kept row's
|
|
147
|
+
# tool_call_id as an answered call, so a wake row carrying its session's
|
|
148
|
+
# start call id would hide the real tool_result -- the HTTP 400 M4
|
|
149
|
+
# exists to prevent. The call id lives inside this JSON instead.
|
|
150
|
+
harness: Optional[str] = None
|
|
151
|
+
# ROADMAP_v3 §81 (IM2). The images a tool result carried, as a JSON list
|
|
152
|
+
# of {"media_type", "data"} with `data` base64 -- tool rows only, and only
|
|
153
|
+
# where `view_image` attached one. NULL everywhere else.
|
|
154
|
+
#
|
|
155
|
+
# A COLUMN rather than blocks inside `content`, for `thinking`'s reason:
|
|
156
|
+
# `content` is read as a string by every consumer of the conversation --
|
|
157
|
+
# replay, the transcript, /copy, the compactor, the summarizer -- and
|
|
158
|
+
# only two readers have any use for the bytes, provider translation and
|
|
159
|
+
# the compactor's size proxy. In the database rather than in a blob store
|
|
160
|
+
# or as a path to the workspace file: a path would make a resumed thread
|
|
161
|
+
# send whatever the file holds NOW, and a second store would be a second
|
|
162
|
+
# place docs/PRIVACY.md's "wipe everything" has to name. Nullable and
|
|
163
|
+
# additive, so database.ensure_columns() adds it like `thinking`.
|
|
164
|
+
images: Optional[str] = None
|
|
80
165
|
|
|
81
166
|
|
|
82
167
|
class CompactionCheckpoint(SQLModel, table=True):
|
|
@@ -108,7 +193,7 @@ class CompactionCheckpoint(SQLModel, table=True):
|
|
|
108
193
|
summary_text: str
|
|
109
194
|
covers_up_to_message_id: UUID
|
|
110
195
|
strategy: str = Field(default="rederive")
|
|
111
|
-
created_at: datetime = Field(default_factory=lambda: datetime.now(
|
|
196
|
+
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
|
112
197
|
|
|
113
198
|
|
|
114
199
|
class ThreadSummary(SQLModel, table=True):
|
|
@@ -143,7 +228,7 @@ class ThreadSummary(SQLModel, table=True):
|
|
|
143
228
|
thread_id: UUID = Field(foreign_key="conversationthread.id", index=True)
|
|
144
229
|
summary_text: str
|
|
145
230
|
covers_up_to_message_id: UUID
|
|
146
|
-
created_at: datetime = Field(default_factory=lambda: datetime.now(
|
|
231
|
+
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
|
147
232
|
|
|
148
233
|
|
|
149
234
|
class UserMemory(SQLModel, table=True):
|
|
@@ -183,7 +268,7 @@ class UserMemory(SQLModel, table=True):
|
|
|
183
268
|
scope: str = Field(default="project", index=True)
|
|
184
269
|
project_path: Optional[str] = Field(default=None, index=True)
|
|
185
270
|
source_thread_id: UUID = Field(foreign_key="conversationthread.id")
|
|
186
|
-
created_at: datetime = Field(default_factory=lambda: datetime.now(
|
|
271
|
+
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
|
187
272
|
|
|
188
273
|
|
|
189
274
|
#: ROADMAP_v2 §27 (T1). The three things a thread can be. Not an Enum:
|
|
@@ -196,15 +281,25 @@ THREAD_KIND_RESEARCH_PASS = "research_pass"
|
|
|
196
281
|
THREAD_KIND_SUBAGENT = "subagent"
|
|
197
282
|
|
|
198
283
|
|
|
199
|
-
def create_thread(kind: str = THREAD_KIND_CHAT
|
|
284
|
+
def create_thread(kind: str = THREAD_KIND_CHAT, *,
|
|
285
|
+
parent_thread_id: Optional[UUID] = None,
|
|
286
|
+
parent_call_id: Optional[str] = None,
|
|
287
|
+
agent_name: Optional[str] = None) -> UUID:
|
|
200
288
|
"""Starts a brand-new conversation thread and returns its id.
|
|
201
289
|
|
|
202
290
|
`kind` defaults to "chat" (§27 AC1), so every existing caller keeps
|
|
203
291
|
creating conversations and only the paths that know they are creating
|
|
204
292
|
something else pass anything here.
|
|
293
|
+
|
|
294
|
+
The three lineage arguments (§47) are keyword-only and all default to
|
|
295
|
+
None, for the same reason: a caller that does not know it is creating
|
|
296
|
+
a child is not one, and should not be able to say so by accident in a
|
|
297
|
+
positional slot.
|
|
205
298
|
"""
|
|
206
299
|
with Session(engine) as session:
|
|
207
|
-
thread = ConversationThread(
|
|
300
|
+
thread = ConversationThread(
|
|
301
|
+
kind=kind, parent_thread_id=parent_thread_id,
|
|
302
|
+
parent_call_id=parent_call_id, agent_name=agent_name)
|
|
208
303
|
session.add(thread)
|
|
209
304
|
session.commit()
|
|
210
305
|
session.refresh(thread)
|
|
@@ -222,9 +317,29 @@ def get_thread(thread_id: UUID) -> Optional[dict]:
|
|
|
222
317
|
which is exactly the question _ordered_rows' docstring says copying
|
|
223
318
|
columns removes.
|
|
224
319
|
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
320
|
+
THE `getattr` FALLBACKS COVER A NULL, NOT A MISSING COLUMN, and this
|
|
321
|
+
paragraph said the second thing until batch 75. It read: "a row read
|
|
322
|
+
from a database whose ALTER has not run yet has no attribute at all".
|
|
323
|
+
It cannot -- `session.get` SELECTs every mapped column, so a database
|
|
324
|
+
missing one raises OperationalError before any attribute is read, and
|
|
325
|
+
the class always has the attribute anyway. What these actually absorb
|
|
326
|
+
is a NULL on a row written before the column existed (and a hand-built
|
|
327
|
+
stand-in in a test), which is why each pairs the lookup with a default
|
|
328
|
+
the caller can use. The migration case belongs to
|
|
329
|
+
`database.ensure_columns()`, which runs at startup.
|
|
330
|
+
|
|
331
|
+
`kind` IS THE EXCEPTION, and the sentence above was wrong about it
|
|
332
|
+
until #257. ensure_columns has added `kind` with DEFAULT 'chat' since
|
|
333
|
+
§27 shipped it, and SQLite reads every row older than that ALTER as
|
|
334
|
+
the default -- so a migrated row arrives as 'chat' before any reader
|
|
335
|
+
runs (the lineage columns and `last_activity_at` have no SQL default
|
|
336
|
+
and really are NULL there). A NULL `kind` takes SQL written by hand,
|
|
337
|
+
on a database first migrated between §27 and #26 (2026-08-06 to
|
|
338
|
+
08-22), where the column was added nullable. That is all the
|
|
339
|
+
`or THREAD_KIND_CHAT` here, in child_threads and in list_threads
|
|
340
|
+
absorbs; tests/test_storage_e2e.py builds that database and pins both
|
|
341
|
+
the default and the fallback. list_threads filters in SQL, so its
|
|
342
|
+
default picker never meets such a row at all.
|
|
228
343
|
"""
|
|
229
344
|
with Session(engine) as session:
|
|
230
345
|
thread = session.get(ConversationThread, thread_id)
|
|
@@ -236,10 +351,66 @@ def get_thread(thread_id: UUID) -> Optional[dict]:
|
|
|
236
351
|
"extra_data": dict(thread.extra_data or {}),
|
|
237
352
|
"kind": getattr(thread, "kind", None) or THREAD_KIND_CHAT,
|
|
238
353
|
"last_activity_at": getattr(thread, "last_activity_at", None),
|
|
354
|
+
# getattr for the kind fallback's reason, one column over --
|
|
355
|
+
# see the docstring for what that reason actually is. NULL on
|
|
356
|
+
# a row written before §47 means "no known parent", and a
|
|
357
|
+
# viewer showing that is better than one that raises.
|
|
358
|
+
"parent_thread_id": getattr(thread, "parent_thread_id", None),
|
|
359
|
+
"parent_call_id": getattr(thread, "parent_call_id", None),
|
|
360
|
+
"agent_name": getattr(thread, "agent_name", None),
|
|
239
361
|
}
|
|
240
362
|
|
|
241
363
|
|
|
242
|
-
def
|
|
364
|
+
def child_threads(parent_thread_id: UUID) -> list[dict]:
|
|
365
|
+
"""The threads spawned by this one, oldest first (§47).
|
|
366
|
+
|
|
367
|
+
Each entry: ``{"id", "created_at", "kind", "parent_call_id",
|
|
368
|
+
"agent_name"}``. Empty for a thread that spawned nothing, which is
|
|
369
|
+
most of them.
|
|
370
|
+
|
|
371
|
+
ONE QUERY, not one per spawn line. A resumed conversation resolves
|
|
372
|
+
every `▸ spawn_subagent` line it draws, and asking per line would be
|
|
373
|
+
a query per row of a screen.
|
|
374
|
+
|
|
375
|
+
Oldest first because that is the order the calls happened in, so a
|
|
376
|
+
reader comparing this against the transcript walks both the same
|
|
377
|
+
way. `parent_call_id` is what pairs an entry to a specific line;
|
|
378
|
+
ordering is for the human, not for the lookup.
|
|
379
|
+
|
|
380
|
+
ORDERED BY `(created_at, id)`, which is `_ordered_rows`' rule and was
|
|
381
|
+
missing here (batch 75). That rule reads: timestamps have microsecond
|
|
382
|
+
resolution so ties are unlikely, but "unlikely" is not an ordering
|
|
383
|
+
guarantee. This is the query where it stopped being merely unlikely --
|
|
384
|
+
§47's NA9 makes three `create_thread` calls land from three threads at
|
|
385
|
+
once, which is the one place in this file that can actually tie.
|
|
386
|
+
|
|
387
|
+
NO `kind` FILTER, deliberately. Every child of a chat thread is
|
|
388
|
+
machinery of some sort, and which kinds exist is §27's question
|
|
389
|
+
rather than this one's -- a filter here would have to be widened
|
|
390
|
+
again the moment a sixth source appears, which is how §27's own
|
|
391
|
+
picker missed the compactor.
|
|
392
|
+
"""
|
|
393
|
+
with Session(engine) as session:
|
|
394
|
+
statement = (
|
|
395
|
+
select(ConversationThread)
|
|
396
|
+
.where(ConversationThread.parent_thread_id == parent_thread_id)
|
|
397
|
+
.order_by(ConversationThread.created_at.asc(),
|
|
398
|
+
ConversationThread.id.asc())
|
|
399
|
+
)
|
|
400
|
+
return [
|
|
401
|
+
{
|
|
402
|
+
"id": t.id,
|
|
403
|
+
"created_at": t.created_at,
|
|
404
|
+
# A hand-written NULL only (#257); see get_thread.
|
|
405
|
+
"kind": getattr(t, "kind", None) or THREAD_KIND_CHAT,
|
|
406
|
+
"parent_call_id": getattr(t, "parent_call_id", None),
|
|
407
|
+
"agent_name": getattr(t, "agent_name", None),
|
|
408
|
+
}
|
|
409
|
+
for t in session.exec(statement).all()
|
|
410
|
+
]
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def get_thread_extra(thread_id: UUID) -> dict[str, Any]:
|
|
243
414
|
"""Copy of the thread's extra_data JSON column (goal mode §18, todo
|
|
244
415
|
list §23). Raises ValueError for an unknown thread, same contract as
|
|
245
416
|
the other accessors."""
|
|
@@ -275,6 +446,8 @@ def save_message(
|
|
|
275
446
|
name: Optional[str] = None,
|
|
276
447
|
tool_call_id: Optional[str] = None,
|
|
277
448
|
thinking: Optional[dict] = None,
|
|
449
|
+
harness: Optional[dict] = None,
|
|
450
|
+
images: Optional[list] = None,
|
|
278
451
|
) -> None:
|
|
279
452
|
"""Append one archived row and stamp the thread's activity (#32).
|
|
280
453
|
|
|
@@ -297,18 +470,24 @@ def save_message(
|
|
|
297
470
|
# absent, so "this turn did no reasoning" costs no bytes and
|
|
298
471
|
# reads back as None rather than as an empty structure.
|
|
299
472
|
thinking=json.dumps(thinking) if thinking else None,
|
|
473
|
+
# ROADMAP_v3 §49 (SS5), on `thinking`'s terms: NULL for every
|
|
474
|
+
# row a person or the model wrote, which is nearly all of them.
|
|
475
|
+
harness=json.dumps(harness) if harness else None,
|
|
476
|
+
# ROADMAP_v3 §81 (IM2), on `thinking`'s terms again: NULL for
|
|
477
|
+
# every row that attached no image.
|
|
478
|
+
images=json.dumps(images) if images else None,
|
|
300
479
|
)
|
|
301
480
|
session.add(new_message)
|
|
302
481
|
thread = session.get(ConversationThread, thread_id)
|
|
303
482
|
if thread is not None:
|
|
304
|
-
thread.last_activity_at = datetime.now(
|
|
483
|
+
thread.last_activity_at = datetime.now(UTC)
|
|
305
484
|
session.add(thread)
|
|
306
485
|
session.commit()
|
|
307
486
|
|
|
308
487
|
|
|
309
488
|
def list_threads(
|
|
310
489
|
kind: Optional[str] = THREAD_KIND_CHAT, limit: Optional[int] = None,
|
|
311
|
-
) ->
|
|
490
|
+
) -> list[dict]:
|
|
312
491
|
"""Conversation threads, most recently ACTIVE first (#32).
|
|
313
492
|
|
|
314
493
|
Each entry: ``{"id": UUID, "created_at": datetime, "kind": str,
|
|
@@ -363,9 +542,13 @@ def list_threads(
|
|
|
363
542
|
{
|
|
364
543
|
"id": t.id,
|
|
365
544
|
"created_at": t.created_at,
|
|
366
|
-
#
|
|
367
|
-
#
|
|
368
|
-
#
|
|
545
|
+
# The fallback absorbs a hand-written NULL only: the
|
|
546
|
+
# migration fills a row older than the column with 'chat'
|
|
547
|
+
# (#257). get_thread's docstring has the whole of it. NOT a
|
|
548
|
+
# database whose ALTER has not run: the mapped class always
|
|
549
|
+
# has the attribute, and a database missing the column
|
|
550
|
+
# raises OperationalError at the SELECT above, before any
|
|
551
|
+
# attribute is read.
|
|
369
552
|
"kind": getattr(t, "kind", None) or THREAD_KIND_CHAT,
|
|
370
553
|
"preview": previews.get(t.id, ""),
|
|
371
554
|
}
|
|
@@ -397,7 +580,7 @@ def thread_preview(thread_id: UUID) -> str:
|
|
|
397
580
|
return _first_user_messages(session, [thread_id]).get(thread_id, "")
|
|
398
581
|
|
|
399
582
|
|
|
400
|
-
def _first_user_messages(session, thread_ids:
|
|
583
|
+
def _first_user_messages(session, thread_ids: list[UUID]) -> dict[UUID, str]:
|
|
401
584
|
"""thread id -> truncated first user message, for the given threads.
|
|
402
585
|
|
|
403
586
|
ONE query for the whole list, walked oldest-first so the first row seen
|
|
@@ -434,7 +617,7 @@ def _first_user_messages(session, thread_ids: List[UUID]) -> Dict[UUID, str]:
|
|
|
434
617
|
select(ranked.c.thread_id, ranked.c.content)
|
|
435
618
|
.where(ranked.c.rn == 1)
|
|
436
619
|
).all()
|
|
437
|
-
out:
|
|
620
|
+
out: dict[UUID, str] = {}
|
|
438
621
|
for thread_id, raw in rows:
|
|
439
622
|
try:
|
|
440
623
|
text = json.loads(raw)
|
|
@@ -464,6 +647,44 @@ def _decode_thinking(raw):
|
|
|
464
647
|
return value if isinstance(value, dict) and value.get("blocks") else None
|
|
465
648
|
|
|
466
649
|
|
|
650
|
+
def _decode_harness(raw):
|
|
651
|
+
"""A stored harness record (ROADMAP_v3 §49, SS5), or None for anything
|
|
652
|
+
unusable -- `_decode_thinking`'s rule for `_decode_thinking`'s reason: a
|
|
653
|
+
mark that cannot be read costs the mark, never the thread."""
|
|
654
|
+
if not raw:
|
|
655
|
+
return None
|
|
656
|
+
try:
|
|
657
|
+
value = json.loads(raw)
|
|
658
|
+
except (TypeError, ValueError):
|
|
659
|
+
return None
|
|
660
|
+
return value if isinstance(value, dict) and value.get("kind") else None
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
def _decode_images(raw):
|
|
664
|
+
"""A tool row's stored images (ROADMAP_v3 §81, IM2), or None.
|
|
665
|
+
|
|
666
|
+
`_decode_thinking`'s rule: an entry that cannot be read costs that
|
|
667
|
+
entry, never the thread. An entry is usable when it names a media type
|
|
668
|
+
and carries data, both as non-empty strings -- what translation needs to
|
|
669
|
+
build a block, and nothing it would have to guess at.
|
|
670
|
+
"""
|
|
671
|
+
if not raw:
|
|
672
|
+
return None
|
|
673
|
+
try:
|
|
674
|
+
value = json.loads(raw)
|
|
675
|
+
except (TypeError, ValueError):
|
|
676
|
+
return None
|
|
677
|
+
if not isinstance(value, list):
|
|
678
|
+
return None
|
|
679
|
+
usable = [entry for entry in value
|
|
680
|
+
if isinstance(entry, dict)
|
|
681
|
+
and isinstance(entry.get("media_type"), str)
|
|
682
|
+
and entry["media_type"]
|
|
683
|
+
and isinstance(entry.get("data"), str)
|
|
684
|
+
and entry["data"]]
|
|
685
|
+
return usable or None
|
|
686
|
+
|
|
687
|
+
|
|
467
688
|
def _to_neutral(msg: dict) -> dict:
|
|
468
689
|
"""Reconstructs the exact neutral shape core/memory.py's add_* methods
|
|
469
690
|
write, per msg.role -- this is the counterpart to memory.py only ever
|
|
@@ -499,12 +720,32 @@ def _to_neutral(msg: dict) -> dict:
|
|
|
499
720
|
elif role == "tool":
|
|
500
721
|
# decoded is the plain result string add_tool_result persisted.
|
|
501
722
|
payload = {"role": "tool", "tool_call_id": msg["tool_call_id"], "content": decoded}
|
|
723
|
+
# ROADMAP_v3 §81 (IM2), on §44's terms for `thinking`: present ONLY
|
|
724
|
+
# when the row attached an image, so every tool row written before
|
|
725
|
+
# reconstructs to exactly the shape it did. Read by translation and
|
|
726
|
+
# the compactor's size proxy; never by anything that renders text.
|
|
727
|
+
images = _decode_images(msg.get("images"))
|
|
728
|
+
if images is not None:
|
|
729
|
+
payload["images"] = images
|
|
502
730
|
else:
|
|
503
731
|
# user (and any future plain-content role): decoded is
|
|
504
732
|
# already the right value for "content".
|
|
505
733
|
payload = {"role": role, "content": decoded}
|
|
506
734
|
if msg["tool_call_id"]:
|
|
507
735
|
payload["tool_call_id"] = msg["tool_call_id"]
|
|
736
|
+
# Batch 91, on §44's terms for `thinking`: present ONLY when the row
|
|
737
|
+
# records a failed run, so every message written before reconstructs
|
|
738
|
+
# to exactly the shape it did. Nothing that builds a request reads
|
|
739
|
+
# it -- _messages_for_provider takes a user row's `content` alone,
|
|
740
|
+
# and so does the compactor's _as_text.
|
|
741
|
+
if msg.get("error"):
|
|
742
|
+
payload["error"] = msg["error"]
|
|
743
|
+
# ROADMAP_v3 §49 (SS5), on the same terms: present ONLY when the HARNESS
|
|
744
|
+
# wrote the row, so every other message reconstructs exactly as it did.
|
|
745
|
+
# Read by replay and the compactor's labelling; never by translation.
|
|
746
|
+
harness = _decode_harness(msg.get("harness"))
|
|
747
|
+
if harness is not None:
|
|
748
|
+
payload["harness"] = harness
|
|
508
749
|
|
|
509
750
|
if msg["name"]:
|
|
510
751
|
payload["name"] = msg["name"]
|
|
@@ -512,7 +753,7 @@ def _to_neutral(msg: dict) -> dict:
|
|
|
512
753
|
return payload
|
|
513
754
|
|
|
514
755
|
|
|
515
|
-
def _ordered_rows(thread_id: UUID) ->
|
|
756
|
+
def _ordered_rows(thread_id: UUID) -> list[dict]:
|
|
516
757
|
"""Every row of a thread as plain column values, oldest first.
|
|
517
758
|
|
|
518
759
|
Ordered by (created_at, id) rather than created_at alone. Timestamps
|
|
@@ -546,12 +787,20 @@ def _ordered_rows(thread_id: UUID) -> List[dict]:
|
|
|
546
787
|
# Not coerced, unlike `pinned`: NULL is the ordinary state
|
|
547
788
|
# of this column and _decode_thinking answers None for it.
|
|
548
789
|
"thinking": m.thinking,
|
|
790
|
+
# Batch 91, and NULL is ordinary here too: a run failed.
|
|
791
|
+
"error": m.error,
|
|
792
|
+
# ROADMAP_v3 §49 (SS5). NULL for every row a person or the
|
|
793
|
+
# model wrote.
|
|
794
|
+
"harness": m.harness,
|
|
795
|
+
# ROADMAP_v3 §81 (IM2). NULL for every row that attached no
|
|
796
|
+
# image; the compactor's size proxy reads it (IM8).
|
|
797
|
+
"images": m.images,
|
|
549
798
|
}
|
|
550
799
|
for m in session.exec(statement).all()
|
|
551
800
|
]
|
|
552
801
|
|
|
553
802
|
|
|
554
|
-
def _split_at(rows:
|
|
803
|
+
def _split_at(rows: list[dict], message_id: Optional[UUID]) -> int:
|
|
555
804
|
"""Index just past `message_id` in `rows`, or 0 when it isn't there.
|
|
556
805
|
|
|
557
806
|
Resolving a watermark by POSITION in the already-loaded, already-ordered
|
|
@@ -576,7 +825,7 @@ def _split_at(rows: List[dict], message_id: Optional[UUID]) -> int:
|
|
|
576
825
|
|
|
577
826
|
def get_session_history(
|
|
578
827
|
thread_id: UUID, after_message_id: Optional[UUID] = None,
|
|
579
|
-
) ->
|
|
828
|
+
) -> list[dict]:
|
|
580
829
|
"""The thread's messages in neutral shape, oldest first.
|
|
581
830
|
|
|
582
831
|
`after_message_id` (ROADMAP_v2 §21) loads only the UNCOMPACTED TAIL --
|
|
@@ -591,7 +840,7 @@ def get_session_history(
|
|
|
591
840
|
return [_to_neutral(m) for m in rows[_split_at(rows, after_message_id):]]
|
|
592
841
|
|
|
593
842
|
|
|
594
|
-
def archive_history(thread_id: UUID) ->
|
|
843
|
+
def archive_history(thread_id: UUID) -> list[dict]:
|
|
595
844
|
"""Every message ever written to this thread, compaction or not.
|
|
596
845
|
|
|
597
846
|
This is ROADMAP_v2 §21 AC1 made greppable: compaction adds
|
|
@@ -605,8 +854,38 @@ def archive_history(thread_id: UUID) -> List[dict]:
|
|
|
605
854
|
return get_session_history(thread_id)
|
|
606
855
|
|
|
607
856
|
|
|
608
|
-
def
|
|
609
|
-
"""
|
|
857
|
+
def _turn_opener_index(rows: list[dict], span: list[dict]) -> list[int]:
|
|
858
|
+
"""`[i]` for the user row that OPENS the turn a MID-TURN watermark cut
|
|
859
|
+
into, or `[]`. Batch 120, register item 40.
|
|
860
|
+
|
|
861
|
+
A sub-turn fold puts the watermark inside a turn, so the turn's own
|
|
862
|
+
request falls on the covered side and would be summarized away -- and
|
|
863
|
+
the model would then read a paraphrase of the question it is answering
|
|
864
|
+
rather than the question. That is M5's objection to folding the current
|
|
865
|
+
turn at all, and re-adding the request verbatim is what answers it.
|
|
866
|
+
|
|
867
|
+
The TEST IS THE FIRST KEPT ROW, not a flag on the checkpoint. When the
|
|
868
|
+
watermark is turn-aligned the tail opens with a `user` row and nothing
|
|
869
|
+
needs re-adding; when it is mid-turn the tail opens with an `assistant`
|
|
870
|
+
row, because that is the only boundary _sub_turn_span will place. So
|
|
871
|
+
the state is derivable from the rows, which means no column, no
|
|
872
|
+
migration, and no checkpoint written before this existed that lies
|
|
873
|
+
about its own shape.
|
|
874
|
+
"""
|
|
875
|
+
if not span or len(span) >= len(rows):
|
|
876
|
+
return []
|
|
877
|
+
if rows[len(span)]["role"] == "user":
|
|
878
|
+
return []
|
|
879
|
+
for index in range(len(span) - 1, -1, -1):
|
|
880
|
+
if span[index]["role"] == "user":
|
|
881
|
+
return [index]
|
|
882
|
+
return []
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
def pinned_through(thread_id: UUID, message_id: Optional[UUID]) -> list[dict]:
|
|
886
|
+
"""Rows at or before the watermark the derived view must carry
|
|
887
|
+
verbatim: the pinned set, made whole (§21 M9), plus the opening request
|
|
888
|
+
of a turn a mid-turn watermark cut into (batch 120).
|
|
610
889
|
|
|
611
890
|
A summary covers a contiguous span, but §21 AC2 says a pinned message is
|
|
612
891
|
never summarized -- so pinned rows INSIDE the covered span have to
|
|
@@ -643,7 +922,21 @@ def pinned_through(thread_id: UUID, message_id: Optional[UUID]) -> List[dict]:
|
|
|
643
922
|
rows = _ordered_rows(thread_id)
|
|
644
923
|
span = rows[:_split_at(rows, message_id)]
|
|
645
924
|
neutral = [_to_neutral(m) for m in span]
|
|
925
|
+
return [neutral[i] for i in _pinned_indices(rows, span, neutral)]
|
|
926
|
+
|
|
646
927
|
|
|
928
|
+
def _pinned_indices(rows: list, span: list, neutral: list) -> list[int]:
|
|
929
|
+
"""The positions in `span` that `pinned_through` returns, in archive
|
|
930
|
+
order: the pinned rows, the tool results that make them whole, and the
|
|
931
|
+
opener of a turn a mid-turn watermark cut into.
|
|
932
|
+
|
|
933
|
+
A PURE FUNCTION SINCE BATCH 216 (#231), split out of `pinned_through`
|
|
934
|
+
so `tests/test_fake_storage_mirror.py` can compare it against
|
|
935
|
+
`FakeStorage`'s copy the way it compares `_to_neutral` and `_split_at`
|
|
936
|
+
-- the fake transcribes this algorithm, and a transcription nothing
|
|
937
|
+
compares is a convention somebody has to remember. `rows` and `span`
|
|
938
|
+
need only `role` and `pinned`; `neutral` is `span` reconstructed.
|
|
939
|
+
"""
|
|
647
940
|
keep = [i for i, m in enumerate(span) if m["pinned"]]
|
|
648
941
|
# `have`, the set() below and the role check are all BACKSTOPS, and the
|
|
649
942
|
# measurement says so: each was deleted separately against the full
|
|
@@ -667,13 +960,23 @@ def pinned_through(thread_id: UUID, message_id: Optional[UUID]) -> List[dict]:
|
|
|
667
960
|
if not m["pinned"] and m["role"] == "tool"
|
|
668
961
|
and neutral[i].get("tool_call_id") in need]
|
|
669
962
|
|
|
670
|
-
|
|
963
|
+
# Batch 120. ONE INDEX, AND THE SORT BELOW PLACES IT -- which is the
|
|
964
|
+
# whole reason the turn opener lives here rather than in a function of
|
|
965
|
+
# its own beside this one. The view needs these rows in ARCHIVE ORDER,
|
|
966
|
+
# and `_derived_view` has no positions to merge two lists by: a pinned
|
|
967
|
+
# row from an older turn belongs before the request and a pinned row
|
|
968
|
+
# from inside this turn belongs after it, and either sibling-function
|
|
969
|
+
# arrangement gets one of those wrong. `set()` covers the opener
|
|
970
|
+
# already being pinned.
|
|
971
|
+
keep += _turn_opener_index(rows, span)
|
|
972
|
+
|
|
973
|
+
return sorted(set(keep))
|
|
671
974
|
|
|
672
975
|
|
|
673
976
|
def history_through(
|
|
674
977
|
thread_id: UUID, message_id: Optional[UUID],
|
|
675
978
|
after_message_id: Optional[UUID] = None,
|
|
676
|
-
) ->
|
|
979
|
+
) -> list[dict]:
|
|
677
980
|
"""The span a compaction is about to summarize, in neutral shape.
|
|
678
981
|
|
|
679
982
|
Pinned rows are EXCLUDED (§21 AC2: a pinned message is never included in
|
|
@@ -688,7 +991,7 @@ def history_through(
|
|
|
688
991
|
return [_to_neutral(m) for m in rows[start:end] if not m["pinned"]]
|
|
689
992
|
|
|
690
993
|
|
|
691
|
-
def turn_start_ids(thread_id: UUID) ->
|
|
994
|
+
def turn_start_ids(thread_id: UUID) -> list[UUID]:
|
|
692
995
|
"""Ids of the rows that BEGIN each turn, oldest first.
|
|
693
996
|
|
|
694
997
|
A turn starts at a user message: the wrapper adds one, then the loop
|
|
@@ -701,7 +1004,7 @@ def turn_start_ids(thread_id: UUID) -> List[UUID]:
|
|
|
701
1004
|
return [m["id"] for m in _ordered_rows(thread_id) if m["role"] == "user"]
|
|
702
1005
|
|
|
703
1006
|
|
|
704
|
-
def message_ids_from(thread_id: UUID, message_id: UUID) ->
|
|
1007
|
+
def message_ids_from(thread_id: UUID, message_id: UUID) -> list[UUID]:
|
|
705
1008
|
"""Ids of `message_id` and everything after it. Resolves pin()'s
|
|
706
1009
|
ordinal ("the last N turns") into the real row ids the pinned flag is
|
|
707
1010
|
written against -- §21 keeps MessageLog.id an implementation detail
|
|
@@ -714,7 +1017,7 @@ def message_ids_from(thread_id: UUID, message_id: UUID) -> List[UUID]:
|
|
|
714
1017
|
return [message_id] + [m["id"] for m in rows[start:]]
|
|
715
1018
|
|
|
716
1019
|
|
|
717
|
-
def set_pinned(message_ids:
|
|
1020
|
+
def set_pinned(message_ids: list[UUID], pinned: bool = True) -> int:
|
|
718
1021
|
"""Flag rows compaction-exempt. Returns how many rows changed."""
|
|
719
1022
|
if not message_ids:
|
|
720
1023
|
return 0
|
|
@@ -731,6 +1034,36 @@ def set_pinned(message_ids: List[UUID], pinned: bool = True) -> int:
|
|
|
731
1034
|
return changed
|
|
732
1035
|
|
|
733
1036
|
|
|
1037
|
+
def mark_turn_failed(thread_id: UUID, error: str) -> bool:
|
|
1038
|
+
"""Record on the thread's NEWEST user row why the run it started failed.
|
|
1039
|
+
|
|
1040
|
+
Batch 91; see MessageLog.error for why this is a column. The newest user
|
|
1041
|
+
row is always the right one: every entry point writes the message that
|
|
1042
|
+
starts a run BEFORE the loop begins -- run_agent_conversation,
|
|
1043
|
+
continue_conversation, stream_deep_research_mode and the TUI turn -- so
|
|
1044
|
+
by the time a failure escapes the loop, the row that started it is the
|
|
1045
|
+
last user row there is. Ordered by (created_at, id), `_ordered_rows`'
|
|
1046
|
+
own order, so the two can never disagree about which row is newest.
|
|
1047
|
+
|
|
1048
|
+
Returns False for a thread with no user row. Does not stamp
|
|
1049
|
+
last_activity_at the way save_message does: a failure is not a message.
|
|
1050
|
+
"""
|
|
1051
|
+
with Session(engine) as session:
|
|
1052
|
+
statement = (
|
|
1053
|
+
select(MessageLog)
|
|
1054
|
+
.where(MessageLog.thread_id == thread_id)
|
|
1055
|
+
.where(MessageLog.role == "user")
|
|
1056
|
+
.order_by(MessageLog.created_at.desc(), MessageLog.id.desc())
|
|
1057
|
+
)
|
|
1058
|
+
row = session.exec(statement).first()
|
|
1059
|
+
if row is None:
|
|
1060
|
+
return False
|
|
1061
|
+
row.error = error
|
|
1062
|
+
session.add(row)
|
|
1063
|
+
session.commit()
|
|
1064
|
+
return True
|
|
1065
|
+
|
|
1066
|
+
|
|
734
1067
|
def _current_watermark(thread_id: UUID) -> Optional[UUID]:
|
|
735
1068
|
"""The live checkpoint's watermark, or None if never compacted."""
|
|
736
1069
|
checkpoint = latest_checkpoint(thread_id)
|
|
@@ -939,7 +1272,7 @@ def save_memory(
|
|
|
939
1272
|
|
|
940
1273
|
def list_memories(
|
|
941
1274
|
project_path: Optional[str] = None, scope: Optional[str] = None,
|
|
942
|
-
) ->
|
|
1275
|
+
) -> list[dict]:
|
|
943
1276
|
"""Memories visible from `project_path`, NEWEST FIRST.
|
|
944
1277
|
|
|
945
1278
|
Returns global memories plus the project ones recorded against exactly
|