@yahyahammad/venastine 0.3.8 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/.env.example +19 -19
  2. package/CONFIG_ARCHITECTURE.md +2030 -2002
  3. package/LICENSE +201 -201
  4. package/README.md +1034 -1034
  5. package/agents/builtin/compactor.md +75 -75
  6. package/agents/builtin/explore.md +77 -77
  7. package/agents/builtin/pipeline-reviewer.md +100 -100
  8. package/agents/builtin/plan.md +106 -106
  9. package/agents/builtin/review.md +74 -74
  10. package/agents/manager.py +261 -261
  11. package/agents/subagent_tool.py +472 -472
  12. package/agents/tui_commands.py +238 -146
  13. package/bin/venastine.mjs +528 -528
  14. package/bin/venastine.py +71 -71
  15. package/cli_commands.py +838 -835
  16. package/config.yaml +13 -0
  17. package/config_edit.py +1301 -1301
  18. package/config_schema.py +971 -966
  19. package/config_update.py +749 -749
  20. package/core/agent_activity.py +256 -256
  21. package/core/ansi.py +452 -452
  22. package/core/approval.py +137 -137
  23. package/core/call_view.py +364 -364
  24. package/core/cancel.py +121 -121
  25. package/core/chatgpt_auth.py +747 -747
  26. package/core/claims_view.py +107 -107
  27. package/core/client.py +2715 -2715
  28. package/core/commands.py +460 -454
  29. package/core/compaction.py +1735 -1735
  30. package/core/config_loader.py +1982 -1982
  31. package/core/connect.py +686 -686
  32. package/core/events.py +123 -123
  33. package/core/file_mentions.py +280 -280
  34. package/core/interaction.py +411 -411
  35. package/core/loop.py +2566 -2566
  36. package/core/memory.py +482 -482
  37. package/core/pipeline_models.py +190 -190
  38. package/core/provider_errors.py +234 -234
  39. package/core/reasoning/authorization.py +130 -130
  40. package/core/reasoning/base.py +219 -219
  41. package/core/reasoning/confidence_scoring.py +484 -484
  42. package/core/reasoning/events.py +149 -149
  43. package/core/reasoning/orchestrator.py +1561 -1561
  44. package/core/reasoning/output_writer.py +345 -345
  45. package/core/reasoning/payload_validation.py +524 -524
  46. package/core/reasoning/pipeline_storage.py +427 -427
  47. package/core/reasoning/scholar.py +458 -458
  48. package/core/reasoning/source_corpus.py +433 -433
  49. package/core/reasoning/source_scoring.py +855 -855
  50. package/core/replay.py +315 -315
  51. package/core/shell_sessions.py +1553 -1553
  52. package/credentials.py +307 -307
  53. package/database.py +333 -333
  54. package/docs/PRIVACY.md +78 -78
  55. package/docs/SECURITY.md +108 -108
  56. package/docs/THIRD_PARTY_NOTICES.md +148 -148
  57. package/env_secrets.py +118 -118
  58. package/json_store.py +343 -343
  59. package/logging_setup.py +295 -295
  60. package/main.py +3433 -3433
  61. package/mcp_client/client.py +733 -733
  62. package/mcp_client/config.py +484 -484
  63. package/mcp_client/registration.py +343 -343
  64. package/memories/manager.py +147 -147
  65. package/memories/tui_commands.py +319 -239
  66. package/package.json +303 -303
  67. package/project_init/config_files.py +282 -282
  68. package/project_init/generator.py +728 -728
  69. package/project_init/manifest.py +418 -418
  70. package/project_init/tui_commands.py +147 -147
  71. package/prompts/critic_pass.md +24 -24
  72. package/prompts/revalidate.md +25 -25
  73. package/prompts/revise.md +17 -17
  74. package/prompts/source_grounding.md +66 -66
  75. package/prompts/system_prompts.py +316 -316
  76. package/providers.json.example +94 -94
  77. package/pyproject.toml +149 -149
  78. package/requirements.txt +295 -295
  79. package/safety/policy_enforcement.py +1082 -1082
  80. package/security/askpass.py +419 -419
  81. package/security/capability.py +207 -207
  82. package/security/confine.py +484 -484
  83. package/security/hosted.py +182 -182
  84. package/security/permissions.py +237 -237
  85. package/security/protected_paths.py +501 -501
  86. package/security/sandbox.py +4108 -4108
  87. package/skills/builtin/math/formal-specification.md +67 -67
  88. package/skills/builtin/math/numerical-methods.md +66 -66
  89. package/skills/builtin/research/experiment-design.md +62 -62
  90. package/skills/builtin/research/reproducibility.md +55 -55
  91. package/skills/builtin/research/source-evaluation.md +70 -70
  92. package/skills/builtin/research/statistical-inference.md +63 -63
  93. package/skills/builtin/research/technical-writing.md +59 -59
  94. package/skills/builtin/software/code-review.md +61 -61
  95. package/skills/builtin/software/debugging.md +61 -61
  96. package/skills/builtin/software/test-design.md +67 -67
  97. package/skills/tui_commands.py +110 -110
  98. package/storage.py +1338 -1338
  99. package/tools/base.py +465 -465
  100. package/tools/builtin/_math_common.py +667 -667
  101. package/tools/builtin/_net_common.py +267 -267
  102. package/tools/builtin/arxiv.py +681 -681
  103. package/tools/builtin/fetch_url.py +371 -371
  104. package/tools/builtin/file_ops.py +720 -720
  105. package/tools/builtin/geometry.py +184 -184
  106. package/tools/builtin/load_skill.py +76 -76
  107. package/tools/builtin/pin.py +227 -227
  108. package/tools/builtin/probability_stats.py +224 -224
  109. package/tools/builtin/project_docs.py +464 -464
  110. package/tools/builtin/remember.py +175 -175
  111. package/tools/builtin/shell.py +980 -980
  112. package/tools/builtin/todo.py +149 -149
  113. package/tools/builtin/web_search.py +328 -328
  114. package/tools/context.py +159 -159
  115. package/tools/isolation.py +387 -387
  116. package/tools/registry.py +1242 -1242
  117. package/tui/app.py +8318 -8307
  118. package/tui/app.tcss +747 -739
  119. package/tui/commands.py +26 -26
  120. package/tui/diffs.py +286 -286
  121. package/tui/history.py +128 -128
  122. package/tui/hosted_driver.py +102 -102
  123. package/tui/markdown.py +1329 -1329
  124. package/tui/meters.py +364 -364
  125. package/tui/ravens.py +146 -146
  126. package/tui/screens.py +1255 -1197
  127. package/tui/themes.py +599 -599
  128. package/tui/widgets.py +3813 -3796
@@ -1,427 +1,427 @@
1
- """
2
- core/reasoning/pipeline_storage.py
3
-
4
- Persistence layer for deep-research pipeline runs (ROADMAP §5). A
5
- PipelineRunRecord row is created at the start of
6
- stream_deep_research_pipeline(), checkpointed with a data-only
7
- update_pipeline_run() call after every pass's trace-log line, and
8
- flipped to a terminal status (complete / failed) at the end. The
9
- load_pipeline_run() read API returns the stored state as a plain dict
10
- for inspection and debugging.
11
-
12
- Schema deliberately mirrors the codebase convention established by
13
- storage.py's MessageLog: semantically distinct data lives in separate
14
- typed columns (claims_json / trace_json / coverage_gaps_json /
15
- pass_threads_json / candidates_json / final_report), NOT one catch-all
16
- blob. The *_json columns hold json.dumps'd strings (same pattern as
17
- MessageLog.content); final_report is plain text. This keeps a serialization problem in one field from
18
- corrupting the others -- a persistence layer built for crash-resilience
19
- shouldn't itself be a single point of failure.
20
-
21
- Tables created here are registered in SQLModel.metadata and built by
22
- database.create_db_and_tables(), the same startup hook that creates
23
- ConversationThread / MessageLog.
24
- """
25
-
26
- from __future__ import annotations
27
-
28
- import json
29
- import logging
30
- from datetime import UTC, datetime
31
- from typing import Optional
32
- from uuid import UUID, uuid4
33
-
34
- from sqlmodel import Field, Session, SQLModel
35
-
36
- from database import engine
37
-
38
- # §27: the thread-kind vocabulary lives beside the table that stores it.
39
- # This import direction is the only one that exists -- storage.py knows
40
- # nothing about pipeline runs, and classify_legacy_pass_threads() below is
41
- # where the two meet.
42
- from storage import THREAD_KIND_CHAT, THREAD_KIND_RESEARCH_PASS
43
-
44
- logger = logging.getLogger(__name__)
45
-
46
-
47
- class PipelineRunRecord(SQLModel, table=True):
48
- """One row per stream_deep_research_pipeline() invocation. `status` is
49
- 'running' from creation until the wrapping try/except in the
50
- orchestrator flips it to 'complete' or 'failed'. The SIX data columns
51
- carry the run's structured output (or the partial state available at
52
- failure time) so an operator can inspect what happened without a
53
- separate trace log.
54
-
55
- SIX, not the four this paragraph said until batch 154: T2 added
56
- pass_threads_json and E13 added candidates_json, and neither batch
57
- found the enumerations written about them. The count is pinned by
58
- test_the_documented_record_shape_matches_what_load_returns, which
59
- reads it out of the prose rather than out of a second list here."""
60
-
61
- id: UUID = Field(default_factory=uuid4, primary_key=True)
62
- user_query: str = Field(index=True)
63
- status: str = Field(default="running") # "running" | "complete" | "failed"
64
- claims_json: str = "[]"
65
- trace_json: str = "[]"
66
- coverage_gaps_json: str = "[]"
67
- # §27 (T2): [{"pass": ..., "thread_id": ...}] for this run's pass
68
- # threads. Persisted rather than left to the output directory alone
69
- # (where granted_calls and subagent_reviews live) because those are
70
- # written once, at the end, by a run that finished -- and a run whose
71
- # passes are worth chasing through their threads is frequently one
72
- # that did not. Checkpointed like the other data columns, so an
73
- # abandoned run keeps the ids it had reached.
74
- pass_threads_json: str = "[]"
75
- # E13/#77: [{"candidate", "provider_name", "model", "chars"}] per
76
- # ensemble-Pass-1 SURVIVOR, so a stored claim's
77
- # asserted_by_candidates names models this row can still identify.
78
- # Metadata only: the full texts are per-candidate artifacts and pass
79
- # threads, and update_pipeline_run strips the in-memory `text` key
80
- # here rather than duplicating bodies into the database.
81
- # Checkpointed like the other data columns, so an abandoned run keeps
82
- # what it had reached.
83
- candidates_json: str = "[]"
84
- final_report: str = ""
85
- started_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
86
- finished_at: Optional[datetime] = None
87
-
88
-
89
- # ---------------------------------------------------------------------------
90
- # ---- Public write API ------------------------------------------------------
91
- # ---------------------------------------------------------------------------
92
-
93
- def create_pipeline_run(user_query: str) -> UUID:
94
- """Inserts a 'running' PipelineRunRecord row and returns its id.
95
- The orchestrator assigns this id to PipelineRun.run_id so later
96
- updates can find the row."""
97
- with Session(engine) as session:
98
- record = PipelineRunRecord(user_query=user_query, status="running")
99
- session.add(record)
100
- session.commit()
101
- session.refresh(record)
102
- return record.id
103
-
104
-
105
- def update_pipeline_run(
106
- run_id: UUID,
107
- run,
108
- status: Optional[str] = None,
109
- ) -> None:
110
- """Updates an existing row's data columns (claims / trace /
111
- coverage_gaps / pass_threads / candidates / final_report) from the
112
- live PipelineRun, and -- only when `status` is given -- flips status
113
- and sets finished_at on a terminal transition. Raises ValueError if run_id doesn't exist (the
114
- record was never created or was deleted), which is a real bug, not an
115
- expected path.
116
-
117
- `status` is Optional by design: the minimal core only ever passes a
118
- terminal status, but §5 proper's deferred per-pass checkpointing will
119
- call this for plain data snapshots WITHOUT a status. Defaulting status
120
- to a value here would silently reset a 'complete'/'failed' record back
121
- to that default on a data-only update; Optional=None means "don't touch
122
- status unless told to," which makes that future extension safe by
123
- construction.
124
-
125
- Claims are serialized with vars(c), matching every other
126
- claim-serialization call site in orchestrator.py (3a/3b/5/6a/6c/final
127
- synthesis). If a nested-dataclass field is ever added to Claim, ALL of
128
- those sites migrate to asdict(c) together -- see the note on Claim in
129
- core/reasoning/base.py.
130
- """
131
- with Session(engine) as session:
132
- record = session.get(PipelineRunRecord, run_id)
133
- if record is None:
134
- raise ValueError(f"No pipeline run found with id {run_id}")
135
- if status is not None:
136
- record.status = status
137
- if status in ("complete", "failed"):
138
- record.finished_at = datetime.now(UTC)
139
- record.claims_json = json.dumps([vars(c) for c in run.claims])
140
- record.trace_json = json.dumps(run.trace)
141
- record.coverage_gaps_json = json.dumps(run.coverage_gaps)
142
- record.pass_threads_json = json.dumps(run.pass_threads)
143
- # getattr, not run.candidates: this function is called with test
144
- # doubles and with runs reconstructed elsewhere, and a checkpoint
145
- # that raises on a missing optional field would take down the pass
146
- # that was only trying to record its progress.
147
- # `text` rides the in-memory entry (it feeds the artifacts and
148
- # Pass 3b) and is stripped HERE -- the database carries metadata
149
- # only; see PipelineRunRecord's column comment.
150
- record.candidates_json = json.dumps([
151
- {k: v for k, v in entry.items() if k != "text"}
152
- for entry in getattr(run, "candidates", []) or []
153
- ])
154
- record.final_report = run.final_report
155
- session.add(record)
156
- session.commit()
157
-
158
-
159
- # ---------------------------------------------------------------------------
160
- # ---- Public read API -------------------------------------------------------
161
- # ---------------------------------------------------------------------------
162
-
163
- def load_pipeline_run(run_id: UUID) -> dict:
164
- """Loads a PipelineRunRecord by id and returns its state as a plain
165
- dict. Raises ValueError if run_id doesn't exist (matching
166
- update_pipeline_run's convention -- a missing record is a real bug,
167
- not an expected path).
168
-
169
- Claims are returned as raw dicts (the shape vars(c) produced at write
170
- time), not reconstructed Claim dataclass instances. If/when
171
- programmatic pipeline resume is built, that function reconstructs
172
- Claim objects itself, using the same field-filtering pattern
173
- _claim_from_json already established in orchestrator.py.
174
- """
175
- with Session(engine) as session:
176
- record = session.get(PipelineRunRecord, run_id)
177
- if record is None:
178
- raise ValueError(f"No pipeline run found with id {run_id}")
179
- return {
180
- "id": record.id,
181
- "user_query": record.user_query,
182
- "status": record.status,
183
- "started_at": record.started_at,
184
- "finished_at": record.finished_at,
185
- "claims": json.loads(record.claims_json),
186
- "trace": json.loads(record.trace_json),
187
- "coverage_gaps": json.loads(record.coverage_gaps_json),
188
- # §27 AC6. The `or "[]"` is for a NULL written BY HAND, and
189
- # until #258 this said a migrated row was one. It is not:
190
- # ensure_columns adds the column with DEFAULT '[]', and SQLite
191
- # reads every row older than the ALTER as that default. A NULL
192
- # takes a database first migrated before #26 carried NOT NULL
193
- # (2026-08-06 to 08-22), where the column is nullable, and SQL
194
- # nobody here wrote -- on which json.loads(None) would raise.
195
- "pass_threads": json.loads(record.pass_threads_json or "[]"),
196
- # E13 arrived after #26, so this column is NOT NULL on every
197
- # database this project has built, and SQLite refuses the NULL
198
- # this would read past. Kept as the reader's own floor beside
199
- # its sibling's; tests/test_storage_e2e.py pins both facts.
200
- "candidates": json.loads(record.candidates_json or "[]"),
201
- "final_report": record.final_report,
202
- }
203
-
204
-
205
- # ---------------------------------------------------------------------------
206
- # ---- One-time classification of pre-§27 threads ---------------------------
207
- # ---------------------------------------------------------------------------
208
-
209
- def _as_uuid_key(value) -> Optional[str]:
210
- """One id's single spelling, or None for anything that is not a UUID.
211
-
212
- THE TWO SIDES OF TIER 1 ARE SPELT DIFFERENTLY, and until batch 154
213
- they were compared literally. `pass_threads_json` carries
214
- `str(thread_id)` -- the dashed 36-character form, which is also what
215
- output_writer.py ships as an artifact and what an operator reads --
216
- while `conversationthread.id` comes back from the raw DBAPI as the
217
- 32-character hex sqlmodel stores. `'a-b' in {'ab'}` is False for
218
- every id there has ever been, so the tier #82 calls authoritative
219
- matched nothing, ever. Measured end to end on real SQLite (the
220
- suite's fake sqlmodel cannot show a storage format): recorded
221
- 'acb451c5-9c50-...', stored 'acb451c59c50...', relabelled 0.
222
-
223
- NONE RATHER THAN THE STRING ITSELF for a value that does not parse. A
224
- thread id IS a UUID (`ConversationThread.id`), so a record holding
225
- something else is not an id this function can speak about, and
226
- deciding that two unparseable strings denote one thread is the
227
- over-classifying direction -- the one that hides a real conversation.
228
- The caller treats None as "no tier-1 answer", which is how an entry
229
- with no usable thread_id has always been treated.
230
-
231
- ONE SIDE, NOT TWO, AND THE OTHER SIDE IS PINNED. This normalises the
232
- RECORDED ids only; the row ids are compared as the database hands
233
- them over. Normalising both was written first, and the control run
234
- as mutation C1 found it unearned -- removing the row-side call left
235
- every test green, because `conversationthread.id` is hex and so is
236
- this. The remaining assumption is that the column holds hex, which
237
- is a fact about sqlmodel's UUID type rather than about this project,
238
- so it is ASSERTED where it can be:
239
- `test_a_thread_id_is_stored_in_a_spelling_str_does_not_produce` in
240
- tests/test_storage_e2e.py fails naming this function if the library
241
- ever starts storing the dashed form. A pinned dependency beats a
242
- branch for a spelling nothing here has ever written.
243
- """
244
- try:
245
- return UUID(value).hex
246
- except (ValueError, AttributeError, TypeError):
247
- return None
248
-
249
-
250
- def classify_legacy_pass_threads(connection) -> int:
251
- """Label pass threads that predate ConversationThread.kind. Returns the
252
- number of rows relabelled.
253
-
254
- WHY THIS IS NOT PART OF ensure_columns(). That function is additive
255
- only and explicitly never backfills (#21 M7) -- so every row already on
256
- disk takes `kind='chat'` from the column default, and the hundreds of
257
- pass threads in an existing database would still fill the picker #27
258
- exists to clear. Widening ensure_columns' contract to backfill would
259
- put data policy inside a schema migrator; if that function ever grows a
260
- DROP or an UPDATE, the project has outgrown it.
261
-
262
- WHY IT LIVES HERE rather than in storage.py. The signal is a pipeline
263
- fact -- which threads belong to which run -- and storage.py owns schema
264
- and CRUD without knowing what a run is. The dependency runs this way
265
- round only; storage.py does not import this module.
266
-
267
- TWO TIERS, EXACT FIRST (#82). Since T2, every run records the exact
268
- thread ids of its passes in `pass_threads_json`. That record is
269
- authoritative over any clock comparison -- so:
270
-
271
- * TIER 1 (exact): every thread id named in a populated
272
- `pass_threads_json` is a pass thread, whatever its timestamp says
273
- and whether or not the run ever finished. Membership was recorded
274
- at run time; there is nothing to infer. COMPARED AS UUIDS, NOT AS
275
- STRINGS -- the two sides are spelt differently and `_as_uuid_key`
276
- below is where they are made one thing (batch 154).
277
- * TIER 2 (window): only runs whose `pass_threads_json` is absent,
278
- NULL, empty, or unusable fall back to the time-window heuristic --
279
- "created between started_at and finished_at". That set is the
280
- pre-T2 population, WHICH CANNOT GROW: a run recorded after T2
281
- always names its own passes, so its window is never consulted and
282
- a chat thread another session creates during it is safe (#82's
283
- headline hazard).
284
-
285
- CORRUPTED JSON FALLS BACK, LOUDLY. An unparseable `pass_threads_json`
286
- puts THAT run in the window tier rather than crashing the launch --
287
- the under-classification direction, which this docstring already
288
- prefers -- and warns naming the run, because silently ignoring data
289
- the writer did write is how the corruption spreads unnoticed.
290
-
291
- A RUN WITH finished_at IS NULL GETS ONLY TIER 1. The window needs a
292
- closing bound: #22's abandoned-generator case deliberately leaves
293
- status='running', so an open window would swallow every thread created
294
- since -- including real conversations. Tier 1 needs no bound (exact
295
- ids), so an abandoned run's recorded passes ARE relabelled.
296
- Under-classifying leaves a straggler in the picker; over-classifying
297
- HIDES A REAL CONVERSATION, and only one of those is recoverable by the
298
- user.
299
-
300
- IDEMPOTENT BY CONSTRUCTION. It only ever moves rows still 'chat', so
301
- running it at every launch is a no-op after the first -- and a database
302
- last opened by an older build still gets classified, both for its
303
- pre-T2 rows (window tier) and its post-T2 rows (whose ids were recorded
304
- all along).
305
-
306
- Raw SQL over a DBAPI connection (`engine.raw_connection()` in
307
- production, stdlib `sqlite3.connect(...)` in tests), the same seam
308
- database.ensure_columns() uses and for the same reason: the suite's
309
- fake `sqlmodel` cannot execute DDL, and a seam that only works in
310
- production is not a seam. Does NOT close the connection -- the caller
311
- owns its lifetime. Computation happens in Python (fetch runs once,
312
- fetch chat rows once, one bounded UPDATE per chunk) so the two tiers'
313
- conditions are ordinary code a mutation can target, not a correlated
314
- subquery whose bounds the suite could only probe end-to-end.
315
-
316
- Threads a legacy compactor or subagent created are relabelled
317
- 'research_pass' here if they happen to fall inside a PRE-T2 run's
318
- window, and left as 'chat' otherwise. Both acceptable: the first is
319
- hidden either way, and the second is under-classification.
320
- """
321
- cursor = connection.cursor()
322
- cursor.execute("PRAGMA table_info(conversationthread)")
323
- if not any(row[1] == "kind" for row in cursor.fetchall()):
324
- # No column yet, so nothing to classify. create_all/ensure_columns
325
- # run before this in production; a caller that skipped them gets a
326
- # no-op rather than "no such column".
327
- return 0
328
- cursor.execute("PRAGMA table_info(pipelinerunrecord)")
329
- run_columns = {row[1] for row in cursor.fetchall()}
330
- if not run_columns:
331
- # A database that has never run research. Nothing to classify, and
332
- # neither query below would find its table.
333
- return 0
334
- has_recorded = "pass_threads_json" in run_columns
335
-
336
- # ---- Fetch the runs once, and sort each into a tier -------------------
337
- cursor.execute(
338
- "SELECT id, started_at, finished_at"
339
- + (", pass_threads_json" if has_recorded else "")
340
- + " FROM pipelinerunrecord")
341
- run_rows = cursor.fetchall()
342
-
343
- exact_ids = set() # tier 1: named in some run's pass_threads_json
344
- window_runs = [] # tier 2: (started_at, finished_at) pairs
345
- for row in run_rows:
346
- run_id, started_at, finished_at = row[0], row[1], row[2]
347
- raw = row[3] if has_recorded else None
348
- usable, corrupted = False, False
349
- if raw:
350
- try:
351
- entries = json.loads(raw)
352
- except ValueError:
353
- corrupted = True
354
- else:
355
- if not isinstance(entries, list):
356
- corrupted = True
357
- else:
358
- # T2's shape: [{"pass": ..., "thread_id": ...}]. Entries
359
- # without a usable thread_id contribute nothing either
360
- # way; an entry list that is merely EMPTY is the
361
- # legitimate pre-T2 shape, i.e. window tier.
362
- for entry in entries:
363
- if (isinstance(entry, dict)
364
- and isinstance(entry.get("thread_id"), str)):
365
- # POPULATED is what puts this run in tier 1,
366
- # whatever its ids turn out to parse as. An id
367
- # this function cannot read contributes no
368
- # membership, but it must not drop the run to
369
- # the window either: that would consult a
370
- # modern run's clock and hide the concurrent
371
- # chat thread #82's headline is about.
372
- usable = True
373
- key = _as_uuid_key(entry["thread_id"])
374
- if key is not None:
375
- exact_ids.add(key)
376
- # raw None/"" (or a missing column) falls through with usable=False:
377
- # the legitimate pre-T2 shape, i.e. window tier -- not corruption.
378
- if corrupted:
379
- # #82, owner decision B3: fall back for THIS run, and say so --
380
- # data the writer did write must not vanish without a trace.
381
- logger.warning(
382
- "Classify pass threads: run %s has unparseable "
383
- "pass_threads_json (%r); using its time window instead.",
384
- run_id, raw[:80] if isinstance(raw, str) else raw)
385
- if not usable:
386
- # Window tier -- but only a CLOSED window. The intent guard
387
- # against a future edit "kindly" treating NULL finished_at as
388
- # open-ended lives here, in ordinary code, where removing it
389
- # turns a test red (it never could under the old correlated
390
- # subquery: SQL's x <= NULL was already not true).
391
- if started_at is not None and finished_at is not None:
392
- window_runs.append((started_at, finished_at))
393
-
394
- # ---- Fetch the still-chat rows once ------------------------------------
395
- cursor.execute(
396
- "SELECT id, created_at FROM conversationthread WHERE kind = ?",
397
- (THREAD_KIND_CHAT,))
398
- chat_rows = cursor.fetchall()
399
-
400
- to_relabel = []
401
- for thread_id, created_at in chat_rows:
402
- # `exact_ids` holds hex, which is the spelling this column is in.
403
- # ONE normalisation, on the recorded side, and the assumption on
404
- # this side is PINNED rather than branched on -- see _as_uuid_key.
405
- if thread_id in exact_ids:
406
- to_relabel.append(thread_id)
407
- elif created_at is not None and any(
408
- start <= created_at <= finish for start, finish in window_runs):
409
- to_relabel.append(thread_id)
410
-
411
- relabelled = 0
412
- # Chunked: SQLite's host-parameter ceiling is finite, and a long-lived
413
- # database can carry thousands of pass threads.
414
- for offset in range(0, len(to_relabel), 500):
415
- chunk = to_relabel[offset:offset + 500]
416
- placeholders = ",".join("?" * len(chunk))
417
- cursor.execute(
418
- f"UPDATE conversationthread SET kind = ? "
419
- f" WHERE kind = ? AND id IN ({placeholders})",
420
- (THREAD_KIND_RESEARCH_PASS, THREAD_KIND_CHAT, *chunk))
421
- relabelled += cursor.rowcount if cursor.rowcount > 0 else 0
422
- connection.commit()
423
- if relabelled:
424
- logger.info(
425
- "Classified %d pre-\u00a727 thread(s) as research passes; they no "
426
- "longer appear in the thread picker.", relabelled)
427
- return relabelled
1
+ """
2
+ core/reasoning/pipeline_storage.py
3
+
4
+ Persistence layer for deep-research pipeline runs (ROADMAP §5). A
5
+ PipelineRunRecord row is created at the start of
6
+ stream_deep_research_pipeline(), checkpointed with a data-only
7
+ update_pipeline_run() call after every pass's trace-log line, and
8
+ flipped to a terminal status (complete / failed) at the end. The
9
+ load_pipeline_run() read API returns the stored state as a plain dict
10
+ for inspection and debugging.
11
+
12
+ Schema deliberately mirrors the codebase convention established by
13
+ storage.py's MessageLog: semantically distinct data lives in separate
14
+ typed columns (claims_json / trace_json / coverage_gaps_json /
15
+ pass_threads_json / candidates_json / final_report), NOT one catch-all
16
+ blob. The *_json columns hold json.dumps'd strings (same pattern as
17
+ MessageLog.content); final_report is plain text. This keeps a serialization problem in one field from
18
+ corrupting the others -- a persistence layer built for crash-resilience
19
+ shouldn't itself be a single point of failure.
20
+
21
+ Tables created here are registered in SQLModel.metadata and built by
22
+ database.create_db_and_tables(), the same startup hook that creates
23
+ ConversationThread / MessageLog.
24
+ """
25
+
26
+ from __future__ import annotations
27
+
28
+ import json
29
+ import logging
30
+ from datetime import UTC, datetime
31
+ from typing import Optional
32
+ from uuid import UUID, uuid4
33
+
34
+ from sqlmodel import Field, Session, SQLModel
35
+
36
+ from database import engine
37
+
38
+ # §27: the thread-kind vocabulary lives beside the table that stores it.
39
+ # This import direction is the only one that exists -- storage.py knows
40
+ # nothing about pipeline runs, and classify_legacy_pass_threads() below is
41
+ # where the two meet.
42
+ from storage import THREAD_KIND_CHAT, THREAD_KIND_RESEARCH_PASS
43
+
44
+ logger = logging.getLogger(__name__)
45
+
46
+
47
+ class PipelineRunRecord(SQLModel, table=True):
48
+ """One row per stream_deep_research_pipeline() invocation. `status` is
49
+ 'running' from creation until the wrapping try/except in the
50
+ orchestrator flips it to 'complete' or 'failed'. The SIX data columns
51
+ carry the run's structured output (or the partial state available at
52
+ failure time) so an operator can inspect what happened without a
53
+ separate trace log.
54
+
55
+ SIX, not the four this paragraph said until batch 154: T2 added
56
+ pass_threads_json and E13 added candidates_json, and neither batch
57
+ found the enumerations written about them. The count is pinned by
58
+ test_the_documented_record_shape_matches_what_load_returns, which
59
+ reads it out of the prose rather than out of a second list here."""
60
+
61
+ id: UUID = Field(default_factory=uuid4, primary_key=True)
62
+ user_query: str = Field(index=True)
63
+ status: str = Field(default="running") # "running" | "complete" | "failed"
64
+ claims_json: str = "[]"
65
+ trace_json: str = "[]"
66
+ coverage_gaps_json: str = "[]"
67
+ # §27 (T2): [{"pass": ..., "thread_id": ...}] for this run's pass
68
+ # threads. Persisted rather than left to the output directory alone
69
+ # (where granted_calls and subagent_reviews live) because those are
70
+ # written once, at the end, by a run that finished -- and a run whose
71
+ # passes are worth chasing through their threads is frequently one
72
+ # that did not. Checkpointed like the other data columns, so an
73
+ # abandoned run keeps the ids it had reached.
74
+ pass_threads_json: str = "[]"
75
+ # E13/#77: [{"candidate", "provider_name", "model", "chars"}] per
76
+ # ensemble-Pass-1 SURVIVOR, so a stored claim's
77
+ # asserted_by_candidates names models this row can still identify.
78
+ # Metadata only: the full texts are per-candidate artifacts and pass
79
+ # threads, and update_pipeline_run strips the in-memory `text` key
80
+ # here rather than duplicating bodies into the database.
81
+ # Checkpointed like the other data columns, so an abandoned run keeps
82
+ # what it had reached.
83
+ candidates_json: str = "[]"
84
+ final_report: str = ""
85
+ started_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
86
+ finished_at: Optional[datetime] = None
87
+
88
+
89
+ # ---------------------------------------------------------------------------
90
+ # ---- Public write API ------------------------------------------------------
91
+ # ---------------------------------------------------------------------------
92
+
93
+ def create_pipeline_run(user_query: str) -> UUID:
94
+ """Inserts a 'running' PipelineRunRecord row and returns its id.
95
+ The orchestrator assigns this id to PipelineRun.run_id so later
96
+ updates can find the row."""
97
+ with Session(engine) as session:
98
+ record = PipelineRunRecord(user_query=user_query, status="running")
99
+ session.add(record)
100
+ session.commit()
101
+ session.refresh(record)
102
+ return record.id
103
+
104
+
105
+ def update_pipeline_run(
106
+ run_id: UUID,
107
+ run,
108
+ status: Optional[str] = None,
109
+ ) -> None:
110
+ """Updates an existing row's data columns (claims / trace /
111
+ coverage_gaps / pass_threads / candidates / final_report) from the
112
+ live PipelineRun, and -- only when `status` is given -- flips status
113
+ and sets finished_at on a terminal transition. Raises ValueError if run_id doesn't exist (the
114
+ record was never created or was deleted), which is a real bug, not an
115
+ expected path.
116
+
117
+ `status` is Optional by design: the minimal core only ever passes a
118
+ terminal status, but §5 proper's deferred per-pass checkpointing will
119
+ call this for plain data snapshots WITHOUT a status. Defaulting status
120
+ to a value here would silently reset a 'complete'/'failed' record back
121
+ to that default on a data-only update; Optional=None means "don't touch
122
+ status unless told to," which makes that future extension safe by
123
+ construction.
124
+
125
+ Claims are serialized with vars(c), matching every other
126
+ claim-serialization call site in orchestrator.py (3a/3b/5/6a/6c/final
127
+ synthesis). If a nested-dataclass field is ever added to Claim, ALL of
128
+ those sites migrate to asdict(c) together -- see the note on Claim in
129
+ core/reasoning/base.py.
130
+ """
131
+ with Session(engine) as session:
132
+ record = session.get(PipelineRunRecord, run_id)
133
+ if record is None:
134
+ raise ValueError(f"No pipeline run found with id {run_id}")
135
+ if status is not None:
136
+ record.status = status
137
+ if status in ("complete", "failed"):
138
+ record.finished_at = datetime.now(UTC)
139
+ record.claims_json = json.dumps([vars(c) for c in run.claims])
140
+ record.trace_json = json.dumps(run.trace)
141
+ record.coverage_gaps_json = json.dumps(run.coverage_gaps)
142
+ record.pass_threads_json = json.dumps(run.pass_threads)
143
+ # getattr, not run.candidates: this function is called with test
144
+ # doubles and with runs reconstructed elsewhere, and a checkpoint
145
+ # that raises on a missing optional field would take down the pass
146
+ # that was only trying to record its progress.
147
+ # `text` rides the in-memory entry (it feeds the artifacts and
148
+ # Pass 3b) and is stripped HERE -- the database carries metadata
149
+ # only; see PipelineRunRecord's column comment.
150
+ record.candidates_json = json.dumps([
151
+ {k: v for k, v in entry.items() if k != "text"}
152
+ for entry in getattr(run, "candidates", []) or []
153
+ ])
154
+ record.final_report = run.final_report
155
+ session.add(record)
156
+ session.commit()
157
+
158
+
159
+ # ---------------------------------------------------------------------------
160
+ # ---- Public read API -------------------------------------------------------
161
+ # ---------------------------------------------------------------------------
162
+
163
+ def load_pipeline_run(run_id: UUID) -> dict:
164
+ """Loads a PipelineRunRecord by id and returns its state as a plain
165
+ dict. Raises ValueError if run_id doesn't exist (matching
166
+ update_pipeline_run's convention -- a missing record is a real bug,
167
+ not an expected path).
168
+
169
+ Claims are returned as raw dicts (the shape vars(c) produced at write
170
+ time), not reconstructed Claim dataclass instances. If/when
171
+ programmatic pipeline resume is built, that function reconstructs
172
+ Claim objects itself, using the same field-filtering pattern
173
+ _claim_from_json already established in orchestrator.py.
174
+ """
175
+ with Session(engine) as session:
176
+ record = session.get(PipelineRunRecord, run_id)
177
+ if record is None:
178
+ raise ValueError(f"No pipeline run found with id {run_id}")
179
+ return {
180
+ "id": record.id,
181
+ "user_query": record.user_query,
182
+ "status": record.status,
183
+ "started_at": record.started_at,
184
+ "finished_at": record.finished_at,
185
+ "claims": json.loads(record.claims_json),
186
+ "trace": json.loads(record.trace_json),
187
+ "coverage_gaps": json.loads(record.coverage_gaps_json),
188
+ # §27 AC6. The `or "[]"` is for a NULL written BY HAND, and
189
+ # until #258 this said a migrated row was one. It is not:
190
+ # ensure_columns adds the column with DEFAULT '[]', and SQLite
191
+ # reads every row older than the ALTER as that default. A NULL
192
+ # takes a database first migrated before #26 carried NOT NULL
193
+ # (2026-08-06 to 08-22), where the column is nullable, and SQL
194
+ # nobody here wrote -- on which json.loads(None) would raise.
195
+ "pass_threads": json.loads(record.pass_threads_json or "[]"),
196
+ # E13 arrived after #26, so this column is NOT NULL on every
197
+ # database this project has built, and SQLite refuses the NULL
198
+ # this would read past. Kept as the reader's own floor beside
199
+ # its sibling's; tests/test_storage_e2e.py pins both facts.
200
+ "candidates": json.loads(record.candidates_json or "[]"),
201
+ "final_report": record.final_report,
202
+ }
203
+
204
+
205
+ # ---------------------------------------------------------------------------
206
+ # ---- One-time classification of pre-§27 threads ---------------------------
207
+ # ---------------------------------------------------------------------------
208
+
209
+ def _as_uuid_key(value) -> Optional[str]:
210
+ """One id's single spelling, or None for anything that is not a UUID.
211
+
212
+ THE TWO SIDES OF TIER 1 ARE SPELT DIFFERENTLY, and until batch 154
213
+ they were compared literally. `pass_threads_json` carries
214
+ `str(thread_id)` -- the dashed 36-character form, which is also what
215
+ output_writer.py ships as an artifact and what an operator reads --
216
+ while `conversationthread.id` comes back from the raw DBAPI as the
217
+ 32-character hex sqlmodel stores. `'a-b' in {'ab'}` is False for
218
+ every id there has ever been, so the tier #82 calls authoritative
219
+ matched nothing, ever. Measured end to end on real SQLite (the
220
+ suite's fake sqlmodel cannot show a storage format): recorded
221
+ 'acb451c5-9c50-...', stored 'acb451c59c50...', relabelled 0.
222
+
223
+ NONE RATHER THAN THE STRING ITSELF for a value that does not parse. A
224
+ thread id IS a UUID (`ConversationThread.id`), so a record holding
225
+ something else is not an id this function can speak about, and
226
+ deciding that two unparseable strings denote one thread is the
227
+ over-classifying direction -- the one that hides a real conversation.
228
+ The caller treats None as "no tier-1 answer", which is how an entry
229
+ with no usable thread_id has always been treated.
230
+
231
+ ONE SIDE, NOT TWO, AND THE OTHER SIDE IS PINNED. This normalises the
232
+ RECORDED ids only; the row ids are compared as the database hands
233
+ them over. Normalising both was written first, and the control run
234
+ as mutation C1 found it unearned -- removing the row-side call left
235
+ every test green, because `conversationthread.id` is hex and so is
236
+ this. The remaining assumption is that the column holds hex, which
237
+ is a fact about sqlmodel's UUID type rather than about this project,
238
+ so it is ASSERTED where it can be:
239
+ `test_a_thread_id_is_stored_in_a_spelling_str_does_not_produce` in
240
+ tests/test_storage_e2e.py fails naming this function if the library
241
+ ever starts storing the dashed form. A pinned dependency beats a
242
+ branch for a spelling nothing here has ever written.
243
+ """
244
+ try:
245
+ return UUID(value).hex
246
+ except (ValueError, AttributeError, TypeError):
247
+ return None
248
+
249
+
250
+ def classify_legacy_pass_threads(connection) -> int:
251
+ """Label pass threads that predate ConversationThread.kind. Returns the
252
+ number of rows relabelled.
253
+
254
+ WHY THIS IS NOT PART OF ensure_columns(). That function is additive
255
+ only and explicitly never backfills (#21 M7) -- so every row already on
256
+ disk takes `kind='chat'` from the column default, and the hundreds of
257
+ pass threads in an existing database would still fill the picker #27
258
+ exists to clear. Widening ensure_columns' contract to backfill would
259
+ put data policy inside a schema migrator; if that function ever grows a
260
+ DROP or an UPDATE, the project has outgrown it.
261
+
262
+ WHY IT LIVES HERE rather than in storage.py. The signal is a pipeline
263
+ fact -- which threads belong to which run -- and storage.py owns schema
264
+ and CRUD without knowing what a run is. The dependency runs this way
265
+ round only; storage.py does not import this module.
266
+
267
+ TWO TIERS, EXACT FIRST (#82). Since T2, every run records the exact
268
+ thread ids of its passes in `pass_threads_json`. That record is
269
+ authoritative over any clock comparison -- so:
270
+
271
+ * TIER 1 (exact): every thread id named in a populated
272
+ `pass_threads_json` is a pass thread, whatever its timestamp says
273
+ and whether or not the run ever finished. Membership was recorded
274
+ at run time; there is nothing to infer. COMPARED AS UUIDS, NOT AS
275
+ STRINGS -- the two sides are spelt differently and `_as_uuid_key`
276
+ below is where they are made one thing (batch 154).
277
+ * TIER 2 (window): only runs whose `pass_threads_json` is absent,
278
+ NULL, empty, or unusable fall back to the time-window heuristic --
279
+ "created between started_at and finished_at". That set is the
280
+ pre-T2 population, WHICH CANNOT GROW: a run recorded after T2
281
+ always names its own passes, so its window is never consulted and
282
+ a chat thread another session creates during it is safe (#82's
283
+ headline hazard).
284
+
285
+ CORRUPTED JSON FALLS BACK, LOUDLY. An unparseable `pass_threads_json`
286
+ puts THAT run in the window tier rather than crashing the launch --
287
+ the under-classification direction, which this docstring already
288
+ prefers -- and warns naming the run, because silently ignoring data
289
+ the writer did write is how the corruption spreads unnoticed.
290
+
291
+ A RUN WITH finished_at IS NULL GETS ONLY TIER 1. The window needs a
292
+ closing bound: #22's abandoned-generator case deliberately leaves
293
+ status='running', so an open window would swallow every thread created
294
+ since -- including real conversations. Tier 1 needs no bound (exact
295
+ ids), so an abandoned run's recorded passes ARE relabelled.
296
+ Under-classifying leaves a straggler in the picker; over-classifying
297
+ HIDES A REAL CONVERSATION, and only one of those is recoverable by the
298
+ user.
299
+
300
+ IDEMPOTENT BY CONSTRUCTION. It only ever moves rows still 'chat', so
301
+ running it at every launch is a no-op after the first -- and a database
302
+ last opened by an older build still gets classified, both for its
303
+ pre-T2 rows (window tier) and its post-T2 rows (whose ids were recorded
304
+ all along).
305
+
306
+ Raw SQL over a DBAPI connection (`engine.raw_connection()` in
307
+ production, stdlib `sqlite3.connect(...)` in tests), the same seam
308
+ database.ensure_columns() uses and for the same reason: the suite's
309
+ fake `sqlmodel` cannot execute DDL, and a seam that only works in
310
+ production is not a seam. Does NOT close the connection -- the caller
311
+ owns its lifetime. Computation happens in Python (fetch runs once,
312
+ fetch chat rows once, one bounded UPDATE per chunk) so the two tiers'
313
+ conditions are ordinary code a mutation can target, not a correlated
314
+ subquery whose bounds the suite could only probe end-to-end.
315
+
316
+ Threads a legacy compactor or subagent created are relabelled
317
+ 'research_pass' here if they happen to fall inside a PRE-T2 run's
318
+ window, and left as 'chat' otherwise. Both acceptable: the first is
319
+ hidden either way, and the second is under-classification.
320
+ """
321
+ cursor = connection.cursor()
322
+ cursor.execute("PRAGMA table_info(conversationthread)")
323
+ if not any(row[1] == "kind" for row in cursor.fetchall()):
324
+ # No column yet, so nothing to classify. create_all/ensure_columns
325
+ # run before this in production; a caller that skipped them gets a
326
+ # no-op rather than "no such column".
327
+ return 0
328
+ cursor.execute("PRAGMA table_info(pipelinerunrecord)")
329
+ run_columns = {row[1] for row in cursor.fetchall()}
330
+ if not run_columns:
331
+ # A database that has never run research. Nothing to classify, and
332
+ # neither query below would find its table.
333
+ return 0
334
+ has_recorded = "pass_threads_json" in run_columns
335
+
336
+ # ---- Fetch the runs once, and sort each into a tier -------------------
337
+ cursor.execute(
338
+ "SELECT id, started_at, finished_at"
339
+ + (", pass_threads_json" if has_recorded else "")
340
+ + " FROM pipelinerunrecord")
341
+ run_rows = cursor.fetchall()
342
+
343
+ exact_ids = set() # tier 1: named in some run's pass_threads_json
344
+ window_runs = [] # tier 2: (started_at, finished_at) pairs
345
+ for row in run_rows:
346
+ run_id, started_at, finished_at = row[0], row[1], row[2]
347
+ raw = row[3] if has_recorded else None
348
+ usable, corrupted = False, False
349
+ if raw:
350
+ try:
351
+ entries = json.loads(raw)
352
+ except ValueError:
353
+ corrupted = True
354
+ else:
355
+ if not isinstance(entries, list):
356
+ corrupted = True
357
+ else:
358
+ # T2's shape: [{"pass": ..., "thread_id": ...}]. Entries
359
+ # without a usable thread_id contribute nothing either
360
+ # way; an entry list that is merely EMPTY is the
361
+ # legitimate pre-T2 shape, i.e. window tier.
362
+ for entry in entries:
363
+ if (isinstance(entry, dict)
364
+ and isinstance(entry.get("thread_id"), str)):
365
+ # POPULATED is what puts this run in tier 1,
366
+ # whatever its ids turn out to parse as. An id
367
+ # this function cannot read contributes no
368
+ # membership, but it must not drop the run to
369
+ # the window either: that would consult a
370
+ # modern run's clock and hide the concurrent
371
+ # chat thread #82's headline is about.
372
+ usable = True
373
+ key = _as_uuid_key(entry["thread_id"])
374
+ if key is not None:
375
+ exact_ids.add(key)
376
+ # raw None/"" (or a missing column) falls through with usable=False:
377
+ # the legitimate pre-T2 shape, i.e. window tier -- not corruption.
378
+ if corrupted:
379
+ # #82, owner decision B3: fall back for THIS run, and say so --
380
+ # data the writer did write must not vanish without a trace.
381
+ logger.warning(
382
+ "Classify pass threads: run %s has unparseable "
383
+ "pass_threads_json (%r); using its time window instead.",
384
+ run_id, raw[:80] if isinstance(raw, str) else raw)
385
+ if not usable:
386
+ # Window tier -- but only a CLOSED window. The intent guard
387
+ # against a future edit "kindly" treating NULL finished_at as
388
+ # open-ended lives here, in ordinary code, where removing it
389
+ # turns a test red (it never could under the old correlated
390
+ # subquery: SQL's x <= NULL was already not true).
391
+ if started_at is not None and finished_at is not None:
392
+ window_runs.append((started_at, finished_at))
393
+
394
+ # ---- Fetch the still-chat rows once ------------------------------------
395
+ cursor.execute(
396
+ "SELECT id, created_at FROM conversationthread WHERE kind = ?",
397
+ (THREAD_KIND_CHAT,))
398
+ chat_rows = cursor.fetchall()
399
+
400
+ to_relabel = []
401
+ for thread_id, created_at in chat_rows:
402
+ # `exact_ids` holds hex, which is the spelling this column is in.
403
+ # ONE normalisation, on the recorded side, and the assumption on
404
+ # this side is PINNED rather than branched on -- see _as_uuid_key.
405
+ if thread_id in exact_ids:
406
+ to_relabel.append(thread_id)
407
+ elif created_at is not None and any(
408
+ start <= created_at <= finish for start, finish in window_runs):
409
+ to_relabel.append(thread_id)
410
+
411
+ relabelled = 0
412
+ # Chunked: SQLite's host-parameter ceiling is finite, and a long-lived
413
+ # database can carry thousands of pass threads.
414
+ for offset in range(0, len(to_relabel), 500):
415
+ chunk = to_relabel[offset:offset + 500]
416
+ placeholders = ",".join("?" * len(chunk))
417
+ cursor.execute(
418
+ f"UPDATE conversationthread SET kind = ? "
419
+ f" WHERE kind = ? AND id IN ({placeholders})",
420
+ (THREAD_KIND_RESEARCH_PASS, THREAD_KIND_CHAT, *chunk))
421
+ relabelled += cursor.rowcount if cursor.rowcount > 0 else 0
422
+ connection.commit()
423
+ if relabelled:
424
+ logger.info(
425
+ "Classified %d pre-\u00a727 thread(s) as research passes; they no "
426
+ "longer appear in the thread picker.", relabelled)
427
+ return relabelled