@yahyahammad/venastine 0.3.8 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -19
- package/CONFIG_ARCHITECTURE.md +2030 -2002
- package/LICENSE +201 -201
- package/README.md +1034 -1034
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -106
- package/agents/builtin/review.md +74 -74
- package/agents/manager.py +261 -261
- package/agents/subagent_tool.py +472 -472
- package/agents/tui_commands.py +238 -146
- package/bin/venastine.mjs +528 -528
- package/bin/venastine.py +71 -71
- package/cli_commands.py +838 -835
- package/config.yaml +13 -0
- package/config_edit.py +1301 -1301
- package/config_schema.py +971 -966
- package/config_update.py +749 -749
- package/core/agent_activity.py +256 -256
- package/core/ansi.py +452 -452
- package/core/approval.py +137 -137
- package/core/call_view.py +364 -364
- package/core/cancel.py +121 -121
- package/core/chatgpt_auth.py +747 -747
- package/core/claims_view.py +107 -107
- package/core/client.py +2715 -2715
- package/core/commands.py +460 -454
- package/core/compaction.py +1735 -1735
- package/core/config_loader.py +1982 -1982
- package/core/connect.py +686 -686
- package/core/events.py +123 -123
- package/core/file_mentions.py +280 -280
- package/core/interaction.py +411 -411
- package/core/loop.py +2566 -2566
- package/core/memory.py +482 -482
- package/core/pipeline_models.py +190 -190
- package/core/provider_errors.py +234 -234
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -484
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/orchestrator.py +1561 -1561
- package/core/reasoning/output_writer.py +345 -345
- package/core/reasoning/payload_validation.py +524 -524
- package/core/reasoning/pipeline_storage.py +427 -427
- package/core/reasoning/scholar.py +458 -458
- package/core/reasoning/source_corpus.py +433 -433
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -315
- package/core/shell_sessions.py +1553 -1553
- package/credentials.py +307 -307
- package/database.py +333 -333
- package/docs/PRIVACY.md +78 -78
- package/docs/SECURITY.md +108 -108
- package/docs/THIRD_PARTY_NOTICES.md +148 -148
- package/env_secrets.py +118 -118
- package/json_store.py +343 -343
- package/logging_setup.py +295 -295
- package/main.py +3433 -3433
- package/mcp_client/client.py +733 -733
- package/mcp_client/config.py +484 -484
- package/mcp_client/registration.py +343 -343
- package/memories/manager.py +147 -147
- package/memories/tui_commands.py +319 -239
- package/package.json +303 -303
- package/project_init/config_files.py +282 -282
- package/project_init/generator.py +728 -728
- package/project_init/manifest.py +418 -418
- package/project_init/tui_commands.py +147 -147
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +17 -17
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -316
- package/providers.json.example +94 -94
- package/pyproject.toml +149 -149
- package/requirements.txt +295 -295
- package/safety/policy_enforcement.py +1082 -1082
- package/security/askpass.py +419 -419
- package/security/capability.py +207 -207
- package/security/confine.py +484 -484
- package/security/hosted.py +182 -182
- package/security/permissions.py +237 -237
- package/security/protected_paths.py +501 -501
- package/security/sandbox.py +4108 -4108
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -110
- package/storage.py +1338 -1338
- package/tools/base.py +465 -465
- package/tools/builtin/_math_common.py +667 -667
- package/tools/builtin/_net_common.py +267 -267
- package/tools/builtin/arxiv.py +681 -681
- package/tools/builtin/fetch_url.py +371 -371
- package/tools/builtin/file_ops.py +720 -720
- package/tools/builtin/geometry.py +184 -184
- package/tools/builtin/load_skill.py +76 -76
- package/tools/builtin/pin.py +227 -227
- package/tools/builtin/probability_stats.py +224 -224
- package/tools/builtin/project_docs.py +464 -464
- package/tools/builtin/remember.py +175 -175
- package/tools/builtin/shell.py +980 -980
- package/tools/builtin/todo.py +149 -149
- package/tools/builtin/web_search.py +328 -328
- package/tools/context.py +159 -159
- package/tools/isolation.py +387 -387
- package/tools/registry.py +1242 -1242
- package/tui/app.py +8318 -8307
- package/tui/app.tcss +747 -739
- package/tui/commands.py +26 -26
- package/tui/diffs.py +286 -286
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -102
- package/tui/markdown.py +1329 -1329
- package/tui/meters.py +364 -364
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -1197
- package/tui/themes.py +599 -599
- package/tui/widgets.py +3813 -3796
|
@@ -1,427 +1,427 @@
|
|
|
1
|
-
"""
|
|
2
|
-
core/reasoning/pipeline_storage.py
|
|
3
|
-
|
|
4
|
-
Persistence layer for deep-research pipeline runs (ROADMAP §5). A
|
|
5
|
-
PipelineRunRecord row is created at the start of
|
|
6
|
-
stream_deep_research_pipeline(), checkpointed with a data-only
|
|
7
|
-
update_pipeline_run() call after every pass's trace-log line, and
|
|
8
|
-
flipped to a terminal status (complete / failed) at the end. The
|
|
9
|
-
load_pipeline_run() read API returns the stored state as a plain dict
|
|
10
|
-
for inspection and debugging.
|
|
11
|
-
|
|
12
|
-
Schema deliberately mirrors the codebase convention established by
|
|
13
|
-
storage.py's MessageLog: semantically distinct data lives in separate
|
|
14
|
-
typed columns (claims_json / trace_json / coverage_gaps_json /
|
|
15
|
-
pass_threads_json / candidates_json / final_report), NOT one catch-all
|
|
16
|
-
blob. The *_json columns hold json.dumps'd strings (same pattern as
|
|
17
|
-
MessageLog.content); final_report is plain text. This keeps a serialization problem in one field from
|
|
18
|
-
corrupting the others -- a persistence layer built for crash-resilience
|
|
19
|
-
shouldn't itself be a single point of failure.
|
|
20
|
-
|
|
21
|
-
Tables created here are registered in SQLModel.metadata and built by
|
|
22
|
-
database.create_db_and_tables(), the same startup hook that creates
|
|
23
|
-
ConversationThread / MessageLog.
|
|
24
|
-
"""
|
|
25
|
-
|
|
26
|
-
from __future__ import annotations
|
|
27
|
-
|
|
28
|
-
import json
|
|
29
|
-
import logging
|
|
30
|
-
from datetime import UTC, datetime
|
|
31
|
-
from typing import Optional
|
|
32
|
-
from uuid import UUID, uuid4
|
|
33
|
-
|
|
34
|
-
from sqlmodel import Field, Session, SQLModel
|
|
35
|
-
|
|
36
|
-
from database import engine
|
|
37
|
-
|
|
38
|
-
# §27: the thread-kind vocabulary lives beside the table that stores it.
|
|
39
|
-
# This import direction is the only one that exists -- storage.py knows
|
|
40
|
-
# nothing about pipeline runs, and classify_legacy_pass_threads() below is
|
|
41
|
-
# where the two meet.
|
|
42
|
-
from storage import THREAD_KIND_CHAT, THREAD_KIND_RESEARCH_PASS
|
|
43
|
-
|
|
44
|
-
logger = logging.getLogger(__name__)
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
class PipelineRunRecord(SQLModel, table=True):
|
|
48
|
-
"""One row per stream_deep_research_pipeline() invocation. `status` is
|
|
49
|
-
'running' from creation until the wrapping try/except in the
|
|
50
|
-
orchestrator flips it to 'complete' or 'failed'. The SIX data columns
|
|
51
|
-
carry the run's structured output (or the partial state available at
|
|
52
|
-
failure time) so an operator can inspect what happened without a
|
|
53
|
-
separate trace log.
|
|
54
|
-
|
|
55
|
-
SIX, not the four this paragraph said until batch 154: T2 added
|
|
56
|
-
pass_threads_json and E13 added candidates_json, and neither batch
|
|
57
|
-
found the enumerations written about them. The count is pinned by
|
|
58
|
-
test_the_documented_record_shape_matches_what_load_returns, which
|
|
59
|
-
reads it out of the prose rather than out of a second list here."""
|
|
60
|
-
|
|
61
|
-
id: UUID = Field(default_factory=uuid4, primary_key=True)
|
|
62
|
-
user_query: str = Field(index=True)
|
|
63
|
-
status: str = Field(default="running") # "running" | "complete" | "failed"
|
|
64
|
-
claims_json: str = "[]"
|
|
65
|
-
trace_json: str = "[]"
|
|
66
|
-
coverage_gaps_json: str = "[]"
|
|
67
|
-
# §27 (T2): [{"pass": ..., "thread_id": ...}] for this run's pass
|
|
68
|
-
# threads. Persisted rather than left to the output directory alone
|
|
69
|
-
# (where granted_calls and subagent_reviews live) because those are
|
|
70
|
-
# written once, at the end, by a run that finished -- and a run whose
|
|
71
|
-
# passes are worth chasing through their threads is frequently one
|
|
72
|
-
# that did not. Checkpointed like the other data columns, so an
|
|
73
|
-
# abandoned run keeps the ids it had reached.
|
|
74
|
-
pass_threads_json: str = "[]"
|
|
75
|
-
# E13/#77: [{"candidate", "provider_name", "model", "chars"}] per
|
|
76
|
-
# ensemble-Pass-1 SURVIVOR, so a stored claim's
|
|
77
|
-
# asserted_by_candidates names models this row can still identify.
|
|
78
|
-
# Metadata only: the full texts are per-candidate artifacts and pass
|
|
79
|
-
# threads, and update_pipeline_run strips the in-memory `text` key
|
|
80
|
-
# here rather than duplicating bodies into the database.
|
|
81
|
-
# Checkpointed like the other data columns, so an abandoned run keeps
|
|
82
|
-
# what it had reached.
|
|
83
|
-
candidates_json: str = "[]"
|
|
84
|
-
final_report: str = ""
|
|
85
|
-
started_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
|
86
|
-
finished_at: Optional[datetime] = None
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
# ---------------------------------------------------------------------------
|
|
90
|
-
# ---- Public write API ------------------------------------------------------
|
|
91
|
-
# ---------------------------------------------------------------------------
|
|
92
|
-
|
|
93
|
-
def create_pipeline_run(user_query: str) -> UUID:
|
|
94
|
-
"""Inserts a 'running' PipelineRunRecord row and returns its id.
|
|
95
|
-
The orchestrator assigns this id to PipelineRun.run_id so later
|
|
96
|
-
updates can find the row."""
|
|
97
|
-
with Session(engine) as session:
|
|
98
|
-
record = PipelineRunRecord(user_query=user_query, status="running")
|
|
99
|
-
session.add(record)
|
|
100
|
-
session.commit()
|
|
101
|
-
session.refresh(record)
|
|
102
|
-
return record.id
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
def update_pipeline_run(
|
|
106
|
-
run_id: UUID,
|
|
107
|
-
run,
|
|
108
|
-
status: Optional[str] = None,
|
|
109
|
-
) -> None:
|
|
110
|
-
"""Updates an existing row's data columns (claims / trace /
|
|
111
|
-
coverage_gaps / pass_threads / candidates / final_report) from the
|
|
112
|
-
live PipelineRun, and -- only when `status` is given -- flips status
|
|
113
|
-
and sets finished_at on a terminal transition. Raises ValueError if run_id doesn't exist (the
|
|
114
|
-
record was never created or was deleted), which is a real bug, not an
|
|
115
|
-
expected path.
|
|
116
|
-
|
|
117
|
-
`status` is Optional by design: the minimal core only ever passes a
|
|
118
|
-
terminal status, but §5 proper's deferred per-pass checkpointing will
|
|
119
|
-
call this for plain data snapshots WITHOUT a status. Defaulting status
|
|
120
|
-
to a value here would silently reset a 'complete'/'failed' record back
|
|
121
|
-
to that default on a data-only update; Optional=None means "don't touch
|
|
122
|
-
status unless told to," which makes that future extension safe by
|
|
123
|
-
construction.
|
|
124
|
-
|
|
125
|
-
Claims are serialized with vars(c), matching every other
|
|
126
|
-
claim-serialization call site in orchestrator.py (3a/3b/5/6a/6c/final
|
|
127
|
-
synthesis). If a nested-dataclass field is ever added to Claim, ALL of
|
|
128
|
-
those sites migrate to asdict(c) together -- see the note on Claim in
|
|
129
|
-
core/reasoning/base.py.
|
|
130
|
-
"""
|
|
131
|
-
with Session(engine) as session:
|
|
132
|
-
record = session.get(PipelineRunRecord, run_id)
|
|
133
|
-
if record is None:
|
|
134
|
-
raise ValueError(f"No pipeline run found with id {run_id}")
|
|
135
|
-
if status is not None:
|
|
136
|
-
record.status = status
|
|
137
|
-
if status in ("complete", "failed"):
|
|
138
|
-
record.finished_at = datetime.now(UTC)
|
|
139
|
-
record.claims_json = json.dumps([vars(c) for c in run.claims])
|
|
140
|
-
record.trace_json = json.dumps(run.trace)
|
|
141
|
-
record.coverage_gaps_json = json.dumps(run.coverage_gaps)
|
|
142
|
-
record.pass_threads_json = json.dumps(run.pass_threads)
|
|
143
|
-
# getattr, not run.candidates: this function is called with test
|
|
144
|
-
# doubles and with runs reconstructed elsewhere, and a checkpoint
|
|
145
|
-
# that raises on a missing optional field would take down the pass
|
|
146
|
-
# that was only trying to record its progress.
|
|
147
|
-
# `text` rides the in-memory entry (it feeds the artifacts and
|
|
148
|
-
# Pass 3b) and is stripped HERE -- the database carries metadata
|
|
149
|
-
# only; see PipelineRunRecord's column comment.
|
|
150
|
-
record.candidates_json = json.dumps([
|
|
151
|
-
{k: v for k, v in entry.items() if k != "text"}
|
|
152
|
-
for entry in getattr(run, "candidates", []) or []
|
|
153
|
-
])
|
|
154
|
-
record.final_report = run.final_report
|
|
155
|
-
session.add(record)
|
|
156
|
-
session.commit()
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
# ---------------------------------------------------------------------------
|
|
160
|
-
# ---- Public read API -------------------------------------------------------
|
|
161
|
-
# ---------------------------------------------------------------------------
|
|
162
|
-
|
|
163
|
-
def load_pipeline_run(run_id: UUID) -> dict:
|
|
164
|
-
"""Loads a PipelineRunRecord by id and returns its state as a plain
|
|
165
|
-
dict. Raises ValueError if run_id doesn't exist (matching
|
|
166
|
-
update_pipeline_run's convention -- a missing record is a real bug,
|
|
167
|
-
not an expected path).
|
|
168
|
-
|
|
169
|
-
Claims are returned as raw dicts (the shape vars(c) produced at write
|
|
170
|
-
time), not reconstructed Claim dataclass instances. If/when
|
|
171
|
-
programmatic pipeline resume is built, that function reconstructs
|
|
172
|
-
Claim objects itself, using the same field-filtering pattern
|
|
173
|
-
_claim_from_json already established in orchestrator.py.
|
|
174
|
-
"""
|
|
175
|
-
with Session(engine) as session:
|
|
176
|
-
record = session.get(PipelineRunRecord, run_id)
|
|
177
|
-
if record is None:
|
|
178
|
-
raise ValueError(f"No pipeline run found with id {run_id}")
|
|
179
|
-
return {
|
|
180
|
-
"id": record.id,
|
|
181
|
-
"user_query": record.user_query,
|
|
182
|
-
"status": record.status,
|
|
183
|
-
"started_at": record.started_at,
|
|
184
|
-
"finished_at": record.finished_at,
|
|
185
|
-
"claims": json.loads(record.claims_json),
|
|
186
|
-
"trace": json.loads(record.trace_json),
|
|
187
|
-
"coverage_gaps": json.loads(record.coverage_gaps_json),
|
|
188
|
-
# §27 AC6. The `or "[]"` is for a NULL written BY HAND, and
|
|
189
|
-
# until #258 this said a migrated row was one. It is not:
|
|
190
|
-
# ensure_columns adds the column with DEFAULT '[]', and SQLite
|
|
191
|
-
# reads every row older than the ALTER as that default. A NULL
|
|
192
|
-
# takes a database first migrated before #26 carried NOT NULL
|
|
193
|
-
# (2026-08-06 to 08-22), where the column is nullable, and SQL
|
|
194
|
-
# nobody here wrote -- on which json.loads(None) would raise.
|
|
195
|
-
"pass_threads": json.loads(record.pass_threads_json or "[]"),
|
|
196
|
-
# E13 arrived after #26, so this column is NOT NULL on every
|
|
197
|
-
# database this project has built, and SQLite refuses the NULL
|
|
198
|
-
# this would read past. Kept as the reader's own floor beside
|
|
199
|
-
# its sibling's; tests/test_storage_e2e.py pins both facts.
|
|
200
|
-
"candidates": json.loads(record.candidates_json or "[]"),
|
|
201
|
-
"final_report": record.final_report,
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
# ---------------------------------------------------------------------------
|
|
206
|
-
# ---- One-time classification of pre-§27 threads ---------------------------
|
|
207
|
-
# ---------------------------------------------------------------------------
|
|
208
|
-
|
|
209
|
-
def _as_uuid_key(value) -> Optional[str]:
|
|
210
|
-
"""One id's single spelling, or None for anything that is not a UUID.
|
|
211
|
-
|
|
212
|
-
THE TWO SIDES OF TIER 1 ARE SPELT DIFFERENTLY, and until batch 154
|
|
213
|
-
they were compared literally. `pass_threads_json` carries
|
|
214
|
-
`str(thread_id)` -- the dashed 36-character form, which is also what
|
|
215
|
-
output_writer.py ships as an artifact and what an operator reads --
|
|
216
|
-
while `conversationthread.id` comes back from the raw DBAPI as the
|
|
217
|
-
32-character hex sqlmodel stores. `'a-b' in {'ab'}` is False for
|
|
218
|
-
every id there has ever been, so the tier #82 calls authoritative
|
|
219
|
-
matched nothing, ever. Measured end to end on real SQLite (the
|
|
220
|
-
suite's fake sqlmodel cannot show a storage format): recorded
|
|
221
|
-
'acb451c5-9c50-...', stored 'acb451c59c50...', relabelled 0.
|
|
222
|
-
|
|
223
|
-
NONE RATHER THAN THE STRING ITSELF for a value that does not parse. A
|
|
224
|
-
thread id IS a UUID (`ConversationThread.id`), so a record holding
|
|
225
|
-
something else is not an id this function can speak about, and
|
|
226
|
-
deciding that two unparseable strings denote one thread is the
|
|
227
|
-
over-classifying direction -- the one that hides a real conversation.
|
|
228
|
-
The caller treats None as "no tier-1 answer", which is how an entry
|
|
229
|
-
with no usable thread_id has always been treated.
|
|
230
|
-
|
|
231
|
-
ONE SIDE, NOT TWO, AND THE OTHER SIDE IS PINNED. This normalises the
|
|
232
|
-
RECORDED ids only; the row ids are compared as the database hands
|
|
233
|
-
them over. Normalising both was written first, and the control run
|
|
234
|
-
as mutation C1 found it unearned -- removing the row-side call left
|
|
235
|
-
every test green, because `conversationthread.id` is hex and so is
|
|
236
|
-
this. The remaining assumption is that the column holds hex, which
|
|
237
|
-
is a fact about sqlmodel's UUID type rather than about this project,
|
|
238
|
-
so it is ASSERTED where it can be:
|
|
239
|
-
`test_a_thread_id_is_stored_in_a_spelling_str_does_not_produce` in
|
|
240
|
-
tests/test_storage_e2e.py fails naming this function if the library
|
|
241
|
-
ever starts storing the dashed form. A pinned dependency beats a
|
|
242
|
-
branch for a spelling nothing here has ever written.
|
|
243
|
-
"""
|
|
244
|
-
try:
|
|
245
|
-
return UUID(value).hex
|
|
246
|
-
except (ValueError, AttributeError, TypeError):
|
|
247
|
-
return None
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
def classify_legacy_pass_threads(connection) -> int:
|
|
251
|
-
"""Label pass threads that predate ConversationThread.kind. Returns the
|
|
252
|
-
number of rows relabelled.
|
|
253
|
-
|
|
254
|
-
WHY THIS IS NOT PART OF ensure_columns(). That function is additive
|
|
255
|
-
only and explicitly never backfills (#21 M7) -- so every row already on
|
|
256
|
-
disk takes `kind='chat'` from the column default, and the hundreds of
|
|
257
|
-
pass threads in an existing database would still fill the picker #27
|
|
258
|
-
exists to clear. Widening ensure_columns' contract to backfill would
|
|
259
|
-
put data policy inside a schema migrator; if that function ever grows a
|
|
260
|
-
DROP or an UPDATE, the project has outgrown it.
|
|
261
|
-
|
|
262
|
-
WHY IT LIVES HERE rather than in storage.py. The signal is a pipeline
|
|
263
|
-
fact -- which threads belong to which run -- and storage.py owns schema
|
|
264
|
-
and CRUD without knowing what a run is. The dependency runs this way
|
|
265
|
-
round only; storage.py does not import this module.
|
|
266
|
-
|
|
267
|
-
TWO TIERS, EXACT FIRST (#82). Since T2, every run records the exact
|
|
268
|
-
thread ids of its passes in `pass_threads_json`. That record is
|
|
269
|
-
authoritative over any clock comparison -- so:
|
|
270
|
-
|
|
271
|
-
* TIER 1 (exact): every thread id named in a populated
|
|
272
|
-
`pass_threads_json` is a pass thread, whatever its timestamp says
|
|
273
|
-
and whether or not the run ever finished. Membership was recorded
|
|
274
|
-
at run time; there is nothing to infer. COMPARED AS UUIDS, NOT AS
|
|
275
|
-
STRINGS -- the two sides are spelt differently and `_as_uuid_key`
|
|
276
|
-
below is where they are made one thing (batch 154).
|
|
277
|
-
* TIER 2 (window): only runs whose `pass_threads_json` is absent,
|
|
278
|
-
NULL, empty, or unusable fall back to the time-window heuristic --
|
|
279
|
-
"created between started_at and finished_at". That set is the
|
|
280
|
-
pre-T2 population, WHICH CANNOT GROW: a run recorded after T2
|
|
281
|
-
always names its own passes, so its window is never consulted and
|
|
282
|
-
a chat thread another session creates during it is safe (#82's
|
|
283
|
-
headline hazard).
|
|
284
|
-
|
|
285
|
-
CORRUPTED JSON FALLS BACK, LOUDLY. An unparseable `pass_threads_json`
|
|
286
|
-
puts THAT run in the window tier rather than crashing the launch --
|
|
287
|
-
the under-classification direction, which this docstring already
|
|
288
|
-
prefers -- and warns naming the run, because silently ignoring data
|
|
289
|
-
the writer did write is how the corruption spreads unnoticed.
|
|
290
|
-
|
|
291
|
-
A RUN WITH finished_at IS NULL GETS ONLY TIER 1. The window needs a
|
|
292
|
-
closing bound: #22's abandoned-generator case deliberately leaves
|
|
293
|
-
status='running', so an open window would swallow every thread created
|
|
294
|
-
since -- including real conversations. Tier 1 needs no bound (exact
|
|
295
|
-
ids), so an abandoned run's recorded passes ARE relabelled.
|
|
296
|
-
Under-classifying leaves a straggler in the picker; over-classifying
|
|
297
|
-
HIDES A REAL CONVERSATION, and only one of those is recoverable by the
|
|
298
|
-
user.
|
|
299
|
-
|
|
300
|
-
IDEMPOTENT BY CONSTRUCTION. It only ever moves rows still 'chat', so
|
|
301
|
-
running it at every launch is a no-op after the first -- and a database
|
|
302
|
-
last opened by an older build still gets classified, both for its
|
|
303
|
-
pre-T2 rows (window tier) and its post-T2 rows (whose ids were recorded
|
|
304
|
-
all along).
|
|
305
|
-
|
|
306
|
-
Raw SQL over a DBAPI connection (`engine.raw_connection()` in
|
|
307
|
-
production, stdlib `sqlite3.connect(...)` in tests), the same seam
|
|
308
|
-
database.ensure_columns() uses and for the same reason: the suite's
|
|
309
|
-
fake `sqlmodel` cannot execute DDL, and a seam that only works in
|
|
310
|
-
production is not a seam. Does NOT close the connection -- the caller
|
|
311
|
-
owns its lifetime. Computation happens in Python (fetch runs once,
|
|
312
|
-
fetch chat rows once, one bounded UPDATE per chunk) so the two tiers'
|
|
313
|
-
conditions are ordinary code a mutation can target, not a correlated
|
|
314
|
-
subquery whose bounds the suite could only probe end-to-end.
|
|
315
|
-
|
|
316
|
-
Threads a legacy compactor or subagent created are relabelled
|
|
317
|
-
'research_pass' here if they happen to fall inside a PRE-T2 run's
|
|
318
|
-
window, and left as 'chat' otherwise. Both acceptable: the first is
|
|
319
|
-
hidden either way, and the second is under-classification.
|
|
320
|
-
"""
|
|
321
|
-
cursor = connection.cursor()
|
|
322
|
-
cursor.execute("PRAGMA table_info(conversationthread)")
|
|
323
|
-
if not any(row[1] == "kind" for row in cursor.fetchall()):
|
|
324
|
-
# No column yet, so nothing to classify. create_all/ensure_columns
|
|
325
|
-
# run before this in production; a caller that skipped them gets a
|
|
326
|
-
# no-op rather than "no such column".
|
|
327
|
-
return 0
|
|
328
|
-
cursor.execute("PRAGMA table_info(pipelinerunrecord)")
|
|
329
|
-
run_columns = {row[1] for row in cursor.fetchall()}
|
|
330
|
-
if not run_columns:
|
|
331
|
-
# A database that has never run research. Nothing to classify, and
|
|
332
|
-
# neither query below would find its table.
|
|
333
|
-
return 0
|
|
334
|
-
has_recorded = "pass_threads_json" in run_columns
|
|
335
|
-
|
|
336
|
-
# ---- Fetch the runs once, and sort each into a tier -------------------
|
|
337
|
-
cursor.execute(
|
|
338
|
-
"SELECT id, started_at, finished_at"
|
|
339
|
-
+ (", pass_threads_json" if has_recorded else "")
|
|
340
|
-
+ " FROM pipelinerunrecord")
|
|
341
|
-
run_rows = cursor.fetchall()
|
|
342
|
-
|
|
343
|
-
exact_ids = set() # tier 1: named in some run's pass_threads_json
|
|
344
|
-
window_runs = [] # tier 2: (started_at, finished_at) pairs
|
|
345
|
-
for row in run_rows:
|
|
346
|
-
run_id, started_at, finished_at = row[0], row[1], row[2]
|
|
347
|
-
raw = row[3] if has_recorded else None
|
|
348
|
-
usable, corrupted = False, False
|
|
349
|
-
if raw:
|
|
350
|
-
try:
|
|
351
|
-
entries = json.loads(raw)
|
|
352
|
-
except ValueError:
|
|
353
|
-
corrupted = True
|
|
354
|
-
else:
|
|
355
|
-
if not isinstance(entries, list):
|
|
356
|
-
corrupted = True
|
|
357
|
-
else:
|
|
358
|
-
# T2's shape: [{"pass": ..., "thread_id": ...}]. Entries
|
|
359
|
-
# without a usable thread_id contribute nothing either
|
|
360
|
-
# way; an entry list that is merely EMPTY is the
|
|
361
|
-
# legitimate pre-T2 shape, i.e. window tier.
|
|
362
|
-
for entry in entries:
|
|
363
|
-
if (isinstance(entry, dict)
|
|
364
|
-
and isinstance(entry.get("thread_id"), str)):
|
|
365
|
-
# POPULATED is what puts this run in tier 1,
|
|
366
|
-
# whatever its ids turn out to parse as. An id
|
|
367
|
-
# this function cannot read contributes no
|
|
368
|
-
# membership, but it must not drop the run to
|
|
369
|
-
# the window either: that would consult a
|
|
370
|
-
# modern run's clock and hide the concurrent
|
|
371
|
-
# chat thread #82's headline is about.
|
|
372
|
-
usable = True
|
|
373
|
-
key = _as_uuid_key(entry["thread_id"])
|
|
374
|
-
if key is not None:
|
|
375
|
-
exact_ids.add(key)
|
|
376
|
-
# raw None/"" (or a missing column) falls through with usable=False:
|
|
377
|
-
# the legitimate pre-T2 shape, i.e. window tier -- not corruption.
|
|
378
|
-
if corrupted:
|
|
379
|
-
# #82, owner decision B3: fall back for THIS run, and say so --
|
|
380
|
-
# data the writer did write must not vanish without a trace.
|
|
381
|
-
logger.warning(
|
|
382
|
-
"Classify pass threads: run %s has unparseable "
|
|
383
|
-
"pass_threads_json (%r); using its time window instead.",
|
|
384
|
-
run_id, raw[:80] if isinstance(raw, str) else raw)
|
|
385
|
-
if not usable:
|
|
386
|
-
# Window tier -- but only a CLOSED window. The intent guard
|
|
387
|
-
# against a future edit "kindly" treating NULL finished_at as
|
|
388
|
-
# open-ended lives here, in ordinary code, where removing it
|
|
389
|
-
# turns a test red (it never could under the old correlated
|
|
390
|
-
# subquery: SQL's x <= NULL was already not true).
|
|
391
|
-
if started_at is not None and finished_at is not None:
|
|
392
|
-
window_runs.append((started_at, finished_at))
|
|
393
|
-
|
|
394
|
-
# ---- Fetch the still-chat rows once ------------------------------------
|
|
395
|
-
cursor.execute(
|
|
396
|
-
"SELECT id, created_at FROM conversationthread WHERE kind = ?",
|
|
397
|
-
(THREAD_KIND_CHAT,))
|
|
398
|
-
chat_rows = cursor.fetchall()
|
|
399
|
-
|
|
400
|
-
to_relabel = []
|
|
401
|
-
for thread_id, created_at in chat_rows:
|
|
402
|
-
# `exact_ids` holds hex, which is the spelling this column is in.
|
|
403
|
-
# ONE normalisation, on the recorded side, and the assumption on
|
|
404
|
-
# this side is PINNED rather than branched on -- see _as_uuid_key.
|
|
405
|
-
if thread_id in exact_ids:
|
|
406
|
-
to_relabel.append(thread_id)
|
|
407
|
-
elif created_at is not None and any(
|
|
408
|
-
start <= created_at <= finish for start, finish in window_runs):
|
|
409
|
-
to_relabel.append(thread_id)
|
|
410
|
-
|
|
411
|
-
relabelled = 0
|
|
412
|
-
# Chunked: SQLite's host-parameter ceiling is finite, and a long-lived
|
|
413
|
-
# database can carry thousands of pass threads.
|
|
414
|
-
for offset in range(0, len(to_relabel), 500):
|
|
415
|
-
chunk = to_relabel[offset:offset + 500]
|
|
416
|
-
placeholders = ",".join("?" * len(chunk))
|
|
417
|
-
cursor.execute(
|
|
418
|
-
f"UPDATE conversationthread SET kind = ? "
|
|
419
|
-
f" WHERE kind = ? AND id IN ({placeholders})",
|
|
420
|
-
(THREAD_KIND_RESEARCH_PASS, THREAD_KIND_CHAT, *chunk))
|
|
421
|
-
relabelled += cursor.rowcount if cursor.rowcount > 0 else 0
|
|
422
|
-
connection.commit()
|
|
423
|
-
if relabelled:
|
|
424
|
-
logger.info(
|
|
425
|
-
"Classified %d pre-\u00a727 thread(s) as research passes; they no "
|
|
426
|
-
"longer appear in the thread picker.", relabelled)
|
|
427
|
-
return relabelled
|
|
1
|
+
"""
|
|
2
|
+
core/reasoning/pipeline_storage.py
|
|
3
|
+
|
|
4
|
+
Persistence layer for deep-research pipeline runs (ROADMAP §5). A
|
|
5
|
+
PipelineRunRecord row is created at the start of
|
|
6
|
+
stream_deep_research_pipeline(), checkpointed with a data-only
|
|
7
|
+
update_pipeline_run() call after every pass's trace-log line, and
|
|
8
|
+
flipped to a terminal status (complete / failed) at the end. The
|
|
9
|
+
load_pipeline_run() read API returns the stored state as a plain dict
|
|
10
|
+
for inspection and debugging.
|
|
11
|
+
|
|
12
|
+
Schema deliberately mirrors the codebase convention established by
|
|
13
|
+
storage.py's MessageLog: semantically distinct data lives in separate
|
|
14
|
+
typed columns (claims_json / trace_json / coverage_gaps_json /
|
|
15
|
+
pass_threads_json / candidates_json / final_report), NOT one catch-all
|
|
16
|
+
blob. The *_json columns hold json.dumps'd strings (same pattern as
|
|
17
|
+
MessageLog.content); final_report is plain text. This keeps a serialization problem in one field from
|
|
18
|
+
corrupting the others -- a persistence layer built for crash-resilience
|
|
19
|
+
shouldn't itself be a single point of failure.
|
|
20
|
+
|
|
21
|
+
Tables created here are registered in SQLModel.metadata and built by
|
|
22
|
+
database.create_db_and_tables(), the same startup hook that creates
|
|
23
|
+
ConversationThread / MessageLog.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import json
|
|
29
|
+
import logging
|
|
30
|
+
from datetime import UTC, datetime
|
|
31
|
+
from typing import Optional
|
|
32
|
+
from uuid import UUID, uuid4
|
|
33
|
+
|
|
34
|
+
from sqlmodel import Field, Session, SQLModel
|
|
35
|
+
|
|
36
|
+
from database import engine
|
|
37
|
+
|
|
38
|
+
# §27: the thread-kind vocabulary lives beside the table that stores it.
|
|
39
|
+
# This import direction is the only one that exists -- storage.py knows
|
|
40
|
+
# nothing about pipeline runs, and classify_legacy_pass_threads() below is
|
|
41
|
+
# where the two meet.
|
|
42
|
+
from storage import THREAD_KIND_CHAT, THREAD_KIND_RESEARCH_PASS
|
|
43
|
+
|
|
44
|
+
logger = logging.getLogger(__name__)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class PipelineRunRecord(SQLModel, table=True):
|
|
48
|
+
"""One row per stream_deep_research_pipeline() invocation. `status` is
|
|
49
|
+
'running' from creation until the wrapping try/except in the
|
|
50
|
+
orchestrator flips it to 'complete' or 'failed'. The SIX data columns
|
|
51
|
+
carry the run's structured output (or the partial state available at
|
|
52
|
+
failure time) so an operator can inspect what happened without a
|
|
53
|
+
separate trace log.
|
|
54
|
+
|
|
55
|
+
SIX, not the four this paragraph said until batch 154: T2 added
|
|
56
|
+
pass_threads_json and E13 added candidates_json, and neither batch
|
|
57
|
+
found the enumerations written about them. The count is pinned by
|
|
58
|
+
test_the_documented_record_shape_matches_what_load_returns, which
|
|
59
|
+
reads it out of the prose rather than out of a second list here."""
|
|
60
|
+
|
|
61
|
+
id: UUID = Field(default_factory=uuid4, primary_key=True)
|
|
62
|
+
user_query: str = Field(index=True)
|
|
63
|
+
status: str = Field(default="running") # "running" | "complete" | "failed"
|
|
64
|
+
claims_json: str = "[]"
|
|
65
|
+
trace_json: str = "[]"
|
|
66
|
+
coverage_gaps_json: str = "[]"
|
|
67
|
+
# §27 (T2): [{"pass": ..., "thread_id": ...}] for this run's pass
|
|
68
|
+
# threads. Persisted rather than left to the output directory alone
|
|
69
|
+
# (where granted_calls and subagent_reviews live) because those are
|
|
70
|
+
# written once, at the end, by a run that finished -- and a run whose
|
|
71
|
+
# passes are worth chasing through their threads is frequently one
|
|
72
|
+
# that did not. Checkpointed like the other data columns, so an
|
|
73
|
+
# abandoned run keeps the ids it had reached.
|
|
74
|
+
pass_threads_json: str = "[]"
|
|
75
|
+
# E13/#77: [{"candidate", "provider_name", "model", "chars"}] per
|
|
76
|
+
# ensemble-Pass-1 SURVIVOR, so a stored claim's
|
|
77
|
+
# asserted_by_candidates names models this row can still identify.
|
|
78
|
+
# Metadata only: the full texts are per-candidate artifacts and pass
|
|
79
|
+
# threads, and update_pipeline_run strips the in-memory `text` key
|
|
80
|
+
# here rather than duplicating bodies into the database.
|
|
81
|
+
# Checkpointed like the other data columns, so an abandoned run keeps
|
|
82
|
+
# what it had reached.
|
|
83
|
+
candidates_json: str = "[]"
|
|
84
|
+
final_report: str = ""
|
|
85
|
+
started_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
|
86
|
+
finished_at: Optional[datetime] = None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# ---------------------------------------------------------------------------
|
|
90
|
+
# ---- Public write API ------------------------------------------------------
|
|
91
|
+
# ---------------------------------------------------------------------------
|
|
92
|
+
|
|
93
|
+
def create_pipeline_run(user_query: str) -> UUID:
|
|
94
|
+
"""Inserts a 'running' PipelineRunRecord row and returns its id.
|
|
95
|
+
The orchestrator assigns this id to PipelineRun.run_id so later
|
|
96
|
+
updates can find the row."""
|
|
97
|
+
with Session(engine) as session:
|
|
98
|
+
record = PipelineRunRecord(user_query=user_query, status="running")
|
|
99
|
+
session.add(record)
|
|
100
|
+
session.commit()
|
|
101
|
+
session.refresh(record)
|
|
102
|
+
return record.id
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def update_pipeline_run(
|
|
106
|
+
run_id: UUID,
|
|
107
|
+
run,
|
|
108
|
+
status: Optional[str] = None,
|
|
109
|
+
) -> None:
|
|
110
|
+
"""Updates an existing row's data columns (claims / trace /
|
|
111
|
+
coverage_gaps / pass_threads / candidates / final_report) from the
|
|
112
|
+
live PipelineRun, and -- only when `status` is given -- flips status
|
|
113
|
+
and sets finished_at on a terminal transition. Raises ValueError if run_id doesn't exist (the
|
|
114
|
+
record was never created or was deleted), which is a real bug, not an
|
|
115
|
+
expected path.
|
|
116
|
+
|
|
117
|
+
`status` is Optional by design: the minimal core only ever passes a
|
|
118
|
+
terminal status, but §5 proper's deferred per-pass checkpointing will
|
|
119
|
+
call this for plain data snapshots WITHOUT a status. Defaulting status
|
|
120
|
+
to a value here would silently reset a 'complete'/'failed' record back
|
|
121
|
+
to that default on a data-only update; Optional=None means "don't touch
|
|
122
|
+
status unless told to," which makes that future extension safe by
|
|
123
|
+
construction.
|
|
124
|
+
|
|
125
|
+
Claims are serialized with vars(c), matching every other
|
|
126
|
+
claim-serialization call site in orchestrator.py (3a/3b/5/6a/6c/final
|
|
127
|
+
synthesis). If a nested-dataclass field is ever added to Claim, ALL of
|
|
128
|
+
those sites migrate to asdict(c) together -- see the note on Claim in
|
|
129
|
+
core/reasoning/base.py.
|
|
130
|
+
"""
|
|
131
|
+
with Session(engine) as session:
|
|
132
|
+
record = session.get(PipelineRunRecord, run_id)
|
|
133
|
+
if record is None:
|
|
134
|
+
raise ValueError(f"No pipeline run found with id {run_id}")
|
|
135
|
+
if status is not None:
|
|
136
|
+
record.status = status
|
|
137
|
+
if status in ("complete", "failed"):
|
|
138
|
+
record.finished_at = datetime.now(UTC)
|
|
139
|
+
record.claims_json = json.dumps([vars(c) for c in run.claims])
|
|
140
|
+
record.trace_json = json.dumps(run.trace)
|
|
141
|
+
record.coverage_gaps_json = json.dumps(run.coverage_gaps)
|
|
142
|
+
record.pass_threads_json = json.dumps(run.pass_threads)
|
|
143
|
+
# getattr, not run.candidates: this function is called with test
|
|
144
|
+
# doubles and with runs reconstructed elsewhere, and a checkpoint
|
|
145
|
+
# that raises on a missing optional field would take down the pass
|
|
146
|
+
# that was only trying to record its progress.
|
|
147
|
+
# `text` rides the in-memory entry (it feeds the artifacts and
|
|
148
|
+
# Pass 3b) and is stripped HERE -- the database carries metadata
|
|
149
|
+
# only; see PipelineRunRecord's column comment.
|
|
150
|
+
record.candidates_json = json.dumps([
|
|
151
|
+
{k: v for k, v in entry.items() if k != "text"}
|
|
152
|
+
for entry in getattr(run, "candidates", []) or []
|
|
153
|
+
])
|
|
154
|
+
record.final_report = run.final_report
|
|
155
|
+
session.add(record)
|
|
156
|
+
session.commit()
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
# ---------------------------------------------------------------------------
|
|
160
|
+
# ---- Public read API -------------------------------------------------------
|
|
161
|
+
# ---------------------------------------------------------------------------
|
|
162
|
+
|
|
163
|
+
def load_pipeline_run(run_id: UUID) -> dict:
|
|
164
|
+
"""Loads a PipelineRunRecord by id and returns its state as a plain
|
|
165
|
+
dict. Raises ValueError if run_id doesn't exist (matching
|
|
166
|
+
update_pipeline_run's convention -- a missing record is a real bug,
|
|
167
|
+
not an expected path).
|
|
168
|
+
|
|
169
|
+
Claims are returned as raw dicts (the shape vars(c) produced at write
|
|
170
|
+
time), not reconstructed Claim dataclass instances. If/when
|
|
171
|
+
programmatic pipeline resume is built, that function reconstructs
|
|
172
|
+
Claim objects itself, using the same field-filtering pattern
|
|
173
|
+
_claim_from_json already established in orchestrator.py.
|
|
174
|
+
"""
|
|
175
|
+
with Session(engine) as session:
|
|
176
|
+
record = session.get(PipelineRunRecord, run_id)
|
|
177
|
+
if record is None:
|
|
178
|
+
raise ValueError(f"No pipeline run found with id {run_id}")
|
|
179
|
+
return {
|
|
180
|
+
"id": record.id,
|
|
181
|
+
"user_query": record.user_query,
|
|
182
|
+
"status": record.status,
|
|
183
|
+
"started_at": record.started_at,
|
|
184
|
+
"finished_at": record.finished_at,
|
|
185
|
+
"claims": json.loads(record.claims_json),
|
|
186
|
+
"trace": json.loads(record.trace_json),
|
|
187
|
+
"coverage_gaps": json.loads(record.coverage_gaps_json),
|
|
188
|
+
# §27 AC6. The `or "[]"` is for a NULL written BY HAND, and
|
|
189
|
+
# until #258 this said a migrated row was one. It is not:
|
|
190
|
+
# ensure_columns adds the column with DEFAULT '[]', and SQLite
|
|
191
|
+
# reads every row older than the ALTER as that default. A NULL
|
|
192
|
+
# takes a database first migrated before #26 carried NOT NULL
|
|
193
|
+
# (2026-08-06 to 08-22), where the column is nullable, and SQL
|
|
194
|
+
# nobody here wrote -- on which json.loads(None) would raise.
|
|
195
|
+
"pass_threads": json.loads(record.pass_threads_json or "[]"),
|
|
196
|
+
# E13 arrived after #26, so this column is NOT NULL on every
|
|
197
|
+
# database this project has built, and SQLite refuses the NULL
|
|
198
|
+
# this would read past. Kept as the reader's own floor beside
|
|
199
|
+
# its sibling's; tests/test_storage_e2e.py pins both facts.
|
|
200
|
+
"candidates": json.loads(record.candidates_json or "[]"),
|
|
201
|
+
"final_report": record.final_report,
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
# ---------------------------------------------------------------------------
|
|
206
|
+
# ---- One-time classification of pre-§27 threads ---------------------------
|
|
207
|
+
# ---------------------------------------------------------------------------
|
|
208
|
+
|
|
209
|
+
def _as_uuid_key(value) -> Optional[str]:
|
|
210
|
+
"""One id's single spelling, or None for anything that is not a UUID.
|
|
211
|
+
|
|
212
|
+
THE TWO SIDES OF TIER 1 ARE SPELT DIFFERENTLY, and until batch 154
|
|
213
|
+
they were compared literally. `pass_threads_json` carries
|
|
214
|
+
`str(thread_id)` -- the dashed 36-character form, which is also what
|
|
215
|
+
output_writer.py ships as an artifact and what an operator reads --
|
|
216
|
+
while `conversationthread.id` comes back from the raw DBAPI as the
|
|
217
|
+
32-character hex sqlmodel stores. `'a-b' in {'ab'}` is False for
|
|
218
|
+
every id there has ever been, so the tier #82 calls authoritative
|
|
219
|
+
matched nothing, ever. Measured end to end on real SQLite (the
|
|
220
|
+
suite's fake sqlmodel cannot show a storage format): recorded
|
|
221
|
+
'acb451c5-9c50-...', stored 'acb451c59c50...', relabelled 0.
|
|
222
|
+
|
|
223
|
+
NONE RATHER THAN THE STRING ITSELF for a value that does not parse. A
|
|
224
|
+
thread id IS a UUID (`ConversationThread.id`), so a record holding
|
|
225
|
+
something else is not an id this function can speak about, and
|
|
226
|
+
deciding that two unparseable strings denote one thread is the
|
|
227
|
+
over-classifying direction -- the one that hides a real conversation.
|
|
228
|
+
The caller treats None as "no tier-1 answer", which is how an entry
|
|
229
|
+
with no usable thread_id has always been treated.
|
|
230
|
+
|
|
231
|
+
ONE SIDE, NOT TWO, AND THE OTHER SIDE IS PINNED. This normalises the
|
|
232
|
+
RECORDED ids only; the row ids are compared as the database hands
|
|
233
|
+
them over. Normalising both was written first, and the control run
|
|
234
|
+
as mutation C1 found it unearned -- removing the row-side call left
|
|
235
|
+
every test green, because `conversationthread.id` is hex and so is
|
|
236
|
+
this. The remaining assumption is that the column holds hex, which
|
|
237
|
+
is a fact about sqlmodel's UUID type rather than about this project,
|
|
238
|
+
so it is ASSERTED where it can be:
|
|
239
|
+
`test_a_thread_id_is_stored_in_a_spelling_str_does_not_produce` in
|
|
240
|
+
tests/test_storage_e2e.py fails naming this function if the library
|
|
241
|
+
ever starts storing the dashed form. A pinned dependency beats a
|
|
242
|
+
branch for a spelling nothing here has ever written.
|
|
243
|
+
"""
|
|
244
|
+
try:
|
|
245
|
+
return UUID(value).hex
|
|
246
|
+
except (ValueError, AttributeError, TypeError):
|
|
247
|
+
return None
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def classify_legacy_pass_threads(connection) -> int:
|
|
251
|
+
"""Label pass threads that predate ConversationThread.kind. Returns the
|
|
252
|
+
number of rows relabelled.
|
|
253
|
+
|
|
254
|
+
WHY THIS IS NOT PART OF ensure_columns(). That function is additive
|
|
255
|
+
only and explicitly never backfills (#21 M7) -- so every row already on
|
|
256
|
+
disk takes `kind='chat'` from the column default, and the hundreds of
|
|
257
|
+
pass threads in an existing database would still fill the picker #27
|
|
258
|
+
exists to clear. Widening ensure_columns' contract to backfill would
|
|
259
|
+
put data policy inside a schema migrator; if that function ever grows a
|
|
260
|
+
DROP or an UPDATE, the project has outgrown it.
|
|
261
|
+
|
|
262
|
+
WHY IT LIVES HERE rather than in storage.py. The signal is a pipeline
|
|
263
|
+
fact -- which threads belong to which run -- and storage.py owns schema
|
|
264
|
+
and CRUD without knowing what a run is. The dependency runs this way
|
|
265
|
+
round only; storage.py does not import this module.
|
|
266
|
+
|
|
267
|
+
TWO TIERS, EXACT FIRST (#82). Since T2, every run records the exact
|
|
268
|
+
thread ids of its passes in `pass_threads_json`. That record is
|
|
269
|
+
authoritative over any clock comparison -- so:
|
|
270
|
+
|
|
271
|
+
* TIER 1 (exact): every thread id named in a populated
|
|
272
|
+
`pass_threads_json` is a pass thread, whatever its timestamp says
|
|
273
|
+
and whether or not the run ever finished. Membership was recorded
|
|
274
|
+
at run time; there is nothing to infer. COMPARED AS UUIDS, NOT AS
|
|
275
|
+
STRINGS -- the two sides are spelt differently and `_as_uuid_key`
|
|
276
|
+
below is where they are made one thing (batch 154).
|
|
277
|
+
* TIER 2 (window): only runs whose `pass_threads_json` is absent,
|
|
278
|
+
NULL, empty, or unusable fall back to the time-window heuristic --
|
|
279
|
+
"created between started_at and finished_at". That set is the
|
|
280
|
+
pre-T2 population, WHICH CANNOT GROW: a run recorded after T2
|
|
281
|
+
always names its own passes, so its window is never consulted and
|
|
282
|
+
a chat thread another session creates during it is safe (#82's
|
|
283
|
+
headline hazard).
|
|
284
|
+
|
|
285
|
+
CORRUPTED JSON FALLS BACK, LOUDLY. An unparseable `pass_threads_json`
|
|
286
|
+
puts THAT run in the window tier rather than crashing the launch --
|
|
287
|
+
the under-classification direction, which this docstring already
|
|
288
|
+
prefers -- and warns naming the run, because silently ignoring data
|
|
289
|
+
the writer did write is how the corruption spreads unnoticed.
|
|
290
|
+
|
|
291
|
+
A RUN WITH finished_at IS NULL GETS ONLY TIER 1. The window needs a
|
|
292
|
+
closing bound: #22's abandoned-generator case deliberately leaves
|
|
293
|
+
status='running', so an open window would swallow every thread created
|
|
294
|
+
since -- including real conversations. Tier 1 needs no bound (exact
|
|
295
|
+
ids), so an abandoned run's recorded passes ARE relabelled.
|
|
296
|
+
Under-classifying leaves a straggler in the picker; over-classifying
|
|
297
|
+
HIDES A REAL CONVERSATION, and only one of those is recoverable by the
|
|
298
|
+
user.
|
|
299
|
+
|
|
300
|
+
IDEMPOTENT BY CONSTRUCTION. It only ever moves rows still 'chat', so
|
|
301
|
+
running it at every launch is a no-op after the first -- and a database
|
|
302
|
+
last opened by an older build still gets classified, both for its
|
|
303
|
+
pre-T2 rows (window tier) and its post-T2 rows (whose ids were recorded
|
|
304
|
+
all along).
|
|
305
|
+
|
|
306
|
+
Raw SQL over a DBAPI connection (`engine.raw_connection()` in
|
|
307
|
+
production, stdlib `sqlite3.connect(...)` in tests), the same seam
|
|
308
|
+
database.ensure_columns() uses and for the same reason: the suite's
|
|
309
|
+
fake `sqlmodel` cannot execute DDL, and a seam that only works in
|
|
310
|
+
production is not a seam. Does NOT close the connection -- the caller
|
|
311
|
+
owns its lifetime. Computation happens in Python (fetch runs once,
|
|
312
|
+
fetch chat rows once, one bounded UPDATE per chunk) so the two tiers'
|
|
313
|
+
conditions are ordinary code a mutation can target, not a correlated
|
|
314
|
+
subquery whose bounds the suite could only probe end-to-end.
|
|
315
|
+
|
|
316
|
+
Threads a legacy compactor or subagent created are relabelled
|
|
317
|
+
'research_pass' here if they happen to fall inside a PRE-T2 run's
|
|
318
|
+
window, and left as 'chat' otherwise. Both acceptable: the first is
|
|
319
|
+
hidden either way, and the second is under-classification.
|
|
320
|
+
"""
|
|
321
|
+
cursor = connection.cursor()
|
|
322
|
+
cursor.execute("PRAGMA table_info(conversationthread)")
|
|
323
|
+
if not any(row[1] == "kind" for row in cursor.fetchall()):
|
|
324
|
+
# No column yet, so nothing to classify. create_all/ensure_columns
|
|
325
|
+
# run before this in production; a caller that skipped them gets a
|
|
326
|
+
# no-op rather than "no such column".
|
|
327
|
+
return 0
|
|
328
|
+
cursor.execute("PRAGMA table_info(pipelinerunrecord)")
|
|
329
|
+
run_columns = {row[1] for row in cursor.fetchall()}
|
|
330
|
+
if not run_columns:
|
|
331
|
+
# A database that has never run research. Nothing to classify, and
|
|
332
|
+
# neither query below would find its table.
|
|
333
|
+
return 0
|
|
334
|
+
has_recorded = "pass_threads_json" in run_columns
|
|
335
|
+
|
|
336
|
+
# ---- Fetch the runs once, and sort each into a tier -------------------
|
|
337
|
+
cursor.execute(
|
|
338
|
+
"SELECT id, started_at, finished_at"
|
|
339
|
+
+ (", pass_threads_json" if has_recorded else "")
|
|
340
|
+
+ " FROM pipelinerunrecord")
|
|
341
|
+
run_rows = cursor.fetchall()
|
|
342
|
+
|
|
343
|
+
exact_ids = set() # tier 1: named in some run's pass_threads_json
|
|
344
|
+
window_runs = [] # tier 2: (started_at, finished_at) pairs
|
|
345
|
+
for row in run_rows:
|
|
346
|
+
run_id, started_at, finished_at = row[0], row[1], row[2]
|
|
347
|
+
raw = row[3] if has_recorded else None
|
|
348
|
+
usable, corrupted = False, False
|
|
349
|
+
if raw:
|
|
350
|
+
try:
|
|
351
|
+
entries = json.loads(raw)
|
|
352
|
+
except ValueError:
|
|
353
|
+
corrupted = True
|
|
354
|
+
else:
|
|
355
|
+
if not isinstance(entries, list):
|
|
356
|
+
corrupted = True
|
|
357
|
+
else:
|
|
358
|
+
# T2's shape: [{"pass": ..., "thread_id": ...}]. Entries
|
|
359
|
+
# without a usable thread_id contribute nothing either
|
|
360
|
+
# way; an entry list that is merely EMPTY is the
|
|
361
|
+
# legitimate pre-T2 shape, i.e. window tier.
|
|
362
|
+
for entry in entries:
|
|
363
|
+
if (isinstance(entry, dict)
|
|
364
|
+
and isinstance(entry.get("thread_id"), str)):
|
|
365
|
+
# POPULATED is what puts this run in tier 1,
|
|
366
|
+
# whatever its ids turn out to parse as. An id
|
|
367
|
+
# this function cannot read contributes no
|
|
368
|
+
# membership, but it must not drop the run to
|
|
369
|
+
# the window either: that would consult a
|
|
370
|
+
# modern run's clock and hide the concurrent
|
|
371
|
+
# chat thread #82's headline is about.
|
|
372
|
+
usable = True
|
|
373
|
+
key = _as_uuid_key(entry["thread_id"])
|
|
374
|
+
if key is not None:
|
|
375
|
+
exact_ids.add(key)
|
|
376
|
+
# raw None/"" (or a missing column) falls through with usable=False:
|
|
377
|
+
# the legitimate pre-T2 shape, i.e. window tier -- not corruption.
|
|
378
|
+
if corrupted:
|
|
379
|
+
# #82, owner decision B3: fall back for THIS run, and say so --
|
|
380
|
+
# data the writer did write must not vanish without a trace.
|
|
381
|
+
logger.warning(
|
|
382
|
+
"Classify pass threads: run %s has unparseable "
|
|
383
|
+
"pass_threads_json (%r); using its time window instead.",
|
|
384
|
+
run_id, raw[:80] if isinstance(raw, str) else raw)
|
|
385
|
+
if not usable:
|
|
386
|
+
# Window tier -- but only a CLOSED window. The intent guard
|
|
387
|
+
# against a future edit "kindly" treating NULL finished_at as
|
|
388
|
+
# open-ended lives here, in ordinary code, where removing it
|
|
389
|
+
# turns a test red (it never could under the old correlated
|
|
390
|
+
# subquery: SQL's x <= NULL was already not true).
|
|
391
|
+
if started_at is not None and finished_at is not None:
|
|
392
|
+
window_runs.append((started_at, finished_at))
|
|
393
|
+
|
|
394
|
+
# ---- Fetch the still-chat rows once ------------------------------------
|
|
395
|
+
cursor.execute(
|
|
396
|
+
"SELECT id, created_at FROM conversationthread WHERE kind = ?",
|
|
397
|
+
(THREAD_KIND_CHAT,))
|
|
398
|
+
chat_rows = cursor.fetchall()
|
|
399
|
+
|
|
400
|
+
to_relabel = []
|
|
401
|
+
for thread_id, created_at in chat_rows:
|
|
402
|
+
# `exact_ids` holds hex, which is the spelling this column is in.
|
|
403
|
+
# ONE normalisation, on the recorded side, and the assumption on
|
|
404
|
+
# this side is PINNED rather than branched on -- see _as_uuid_key.
|
|
405
|
+
if thread_id in exact_ids:
|
|
406
|
+
to_relabel.append(thread_id)
|
|
407
|
+
elif created_at is not None and any(
|
|
408
|
+
start <= created_at <= finish for start, finish in window_runs):
|
|
409
|
+
to_relabel.append(thread_id)
|
|
410
|
+
|
|
411
|
+
relabelled = 0
|
|
412
|
+
# Chunked: SQLite's host-parameter ceiling is finite, and a long-lived
|
|
413
|
+
# database can carry thousands of pass threads.
|
|
414
|
+
for offset in range(0, len(to_relabel), 500):
|
|
415
|
+
chunk = to_relabel[offset:offset + 500]
|
|
416
|
+
placeholders = ",".join("?" * len(chunk))
|
|
417
|
+
cursor.execute(
|
|
418
|
+
f"UPDATE conversationthread SET kind = ? "
|
|
419
|
+
f" WHERE kind = ? AND id IN ({placeholders})",
|
|
420
|
+
(THREAD_KIND_RESEARCH_PASS, THREAD_KIND_CHAT, *chunk))
|
|
421
|
+
relabelled += cursor.rowcount if cursor.rowcount > 0 else 0
|
|
422
|
+
connection.commit()
|
|
423
|
+
if relabelled:
|
|
424
|
+
logger.info(
|
|
425
|
+
"Classified %d pre-\u00a727 thread(s) as research passes; they no "
|
|
426
|
+
"longer appear in the thread picker.", relabelled)
|
|
427
|
+
return relabelled
|