@yahyahammad/venastine 0.1.0 → 0.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/.env.example +19 -14
  2. package/CONFIG_ARCHITECTURE.md +2002 -0
  3. package/NOTICE +1 -1
  4. package/README.md +249 -93
  5. package/agents/builtin/build.md +73 -0
  6. package/agents/builtin/general.md +80 -0
  7. package/agents/builtin/plan.md +23 -5
  8. package/agents/builtin/test.md +65 -0
  9. package/agents/builtin/writer.md +67 -0
  10. package/agents/manager.py +261 -195
  11. package/agents/subagent_tool.py +280 -23
  12. package/agents/tui_commands.py +31 -21
  13. package/bin/venastine.mjs +528 -382
  14. package/bin/venastine.py +71 -0
  15. package/cli_commands.py +835 -0
  16. package/config.py +95 -1198
  17. package/config.yaml +1090 -0
  18. package/config_edit.py +1301 -0
  19. package/config_schema.py +966 -0
  20. package/config_update.py +749 -0
  21. package/core/agent_activity.py +256 -0
  22. package/core/ansi.py +452 -0
  23. package/core/approval.py +27 -5
  24. package/core/call_view.py +364 -0
  25. package/core/cancel.py +121 -0
  26. package/core/chatgpt_auth.py +747 -0
  27. package/core/claims_view.py +107 -0
  28. package/core/client.py +1249 -82
  29. package/core/commands.py +454 -0
  30. package/core/compaction.py +1735 -959
  31. package/core/config_loader.py +516 -97
  32. package/core/connect.py +686 -0
  33. package/core/events.py +68 -6
  34. package/core/file_mentions.py +280 -0
  35. package/core/interaction.py +87 -4
  36. package/core/line_pattern.py +128 -0
  37. package/core/loop.py +2566 -1353
  38. package/core/memory.py +482 -370
  39. package/core/pipeline_models.py +23 -9
  40. package/core/provider_errors.py +234 -0
  41. package/core/reasoning/confidence_scoring.py +70 -54
  42. package/core/reasoning/json_retry.py +19 -1
  43. package/core/reasoning/orchestrator.py +232 -53
  44. package/core/reasoning/output_writer.py +90 -31
  45. package/core/reasoning/payload_validation.py +133 -7
  46. package/core/reasoning/pipeline_storage.py +427 -361
  47. package/core/reasoning/review.py +659 -615
  48. package/core/reasoning/scholar.py +38 -14
  49. package/core/reasoning/source_corpus.py +110 -4
  50. package/core/replay.py +315 -167
  51. package/core/session.py +36 -12
  52. package/core/session_view.py +259 -0
  53. package/core/session_wake.py +259 -0
  54. package/core/shell_sessions.py +1553 -0
  55. package/core/workspace_trust.py +1 -1
  56. package/credentials.py +308 -180
  57. package/database.py +82 -1
  58. package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
  59. package/docs/SECURITY.md +108 -0
  60. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  61. package/env_secrets.py +118 -77
  62. package/json_store.py +236 -16
  63. package/logging_setup.py +121 -11
  64. package/main.py +1672 -198
  65. package/mcp_client/client.py +257 -23
  66. package/mcp_client/config.py +484 -379
  67. package/mcp_client/registration.py +163 -2
  68. package/memories/manager.py +15 -0
  69. package/memories/tui_commands.py +21 -14
  70. package/package.json +303 -73
  71. package/project_init/config_files.py +4 -0
  72. package/project_init/generator.py +201 -37
  73. package/project_init/manifest.py +418 -406
  74. package/project_init/tui_commands.py +7 -4
  75. package/prompts/revalidate.md +1 -1
  76. package/prompts/revise.md +17 -15
  77. package/prompts/system_prompts.py +64 -10
  78. package/providers.json.example +94 -92
  79. package/pyproject.toml +149 -100
  80. package/requirements.txt +252 -29
  81. package/safety/policy_enforcement.py +1082 -682
  82. package/security/askpass.py +419 -0
  83. package/security/capability.py +37 -7
  84. package/security/confine.py +484 -0
  85. package/security/hosted.py +182 -0
  86. package/security/permissions.py +54 -7
  87. package/security/posture.py +124 -12
  88. package/security/protected_paths.py +198 -6
  89. package/security/sandbox.py +3262 -139
  90. package/security/secrets.py +855 -0
  91. package/skills/tui_commands.py +3 -4
  92. package/storage.py +1338 -1005
  93. package/tools/base.py +465 -305
  94. package/tools/builtin/_math_common.py +306 -16
  95. package/tools/builtin/_net_common.py +267 -61
  96. package/tools/builtin/arxiv.py +681 -223
  97. package/tools/builtin/ask_user.py +231 -213
  98. package/tools/builtin/discrete_math.py +35 -4
  99. package/tools/builtin/fetch_url.py +239 -21
  100. package/tools/builtin/file_ops.py +341 -33
  101. package/tools/builtin/geometry.py +184 -148
  102. package/tools/builtin/get_time.py +2 -2
  103. package/tools/builtin/linear_algebra.py +66 -23
  104. package/tools/builtin/load_skill.py +10 -0
  105. package/tools/builtin/logic.py +171 -12
  106. package/tools/builtin/pin.py +46 -12
  107. package/tools/builtin/probability_stats.py +224 -228
  108. package/tools/builtin/project_docs.py +80 -21
  109. package/tools/builtin/remember.py +21 -4
  110. package/tools/builtin/shell.py +586 -67
  111. package/tools/builtin/shell_sessions.py +613 -0
  112. package/tools/builtin/symbolic_math.py +80 -10
  113. package/tools/builtin/todo.py +5 -3
  114. package/tools/builtin/web_search.py +329 -151
  115. package/tools/context.py +23 -0
  116. package/tools/isolation.py +387 -314
  117. package/tools/registry.py +374 -29
  118. package/tui/app.py +5281 -397
  119. package/tui/app.tcss +290 -9
  120. package/tui/commands.py +26 -61
  121. package/tui/diffs.py +286 -265
  122. package/tui/history.py +128 -0
  123. package/tui/hosted_driver.py +102 -0
  124. package/tui/markdown.py +1329 -0
  125. package/tui/meters.py +364 -0
  126. package/tui/screens.py +346 -92
  127. package/tui/themes.py +599 -366
  128. package/tui/widgets.py +2673 -116
  129. package/SECURITY.md +0 -88
  130. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -1,361 +1,427 @@
1
- """
2
- core/reasoning/pipeline_storage.py
3
-
4
- Persistence layer for deep-research pipeline runs (ROADMAP §5). A
5
- PipelineRunRecord row is created at the start of
6
- stream_deep_research_pipeline(), checkpointed with a data-only
7
- update_pipeline_run() call after every pass's trace-log line, and
8
- flipped to a terminal status (complete / failed) at the end. The
9
- load_pipeline_run() read API returns the stored state as a plain dict
10
- for inspection and debugging.
11
-
12
- Schema deliberately mirrors the codebase convention established by
13
- storage.py's MessageLog: semantically distinct data lives in separate
14
- typed columns (claims_json / trace_json / coverage_gaps_json /
15
- final_report), NOT one catch-all blob. The *_json columns hold
16
- json.dumps'd strings (same pattern as MessageLog.content); final_report
17
- is plain text. This keeps a serialization problem in one field from
18
- corrupting the others -- a persistence layer built for crash-resilience
19
- shouldn't itself be a single point of failure.
20
-
21
- Tables created here are registered in SQLModel.metadata and built by
22
- database.create_db_and_tables(), the same startup hook that creates
23
- ConversationThread / MessageLog.
24
- """
25
-
26
- from __future__ import annotations
27
-
28
- import json
29
- import logging
30
- from datetime import datetime, timezone
31
- from typing import Optional
32
- from uuid import UUID, uuid4
33
-
34
- from sqlmodel import Field, Session, SQLModel
35
-
36
- from database import engine
37
- # §27: the thread-kind vocabulary lives beside the table that stores it.
38
- # This import direction is the only one that exists -- storage.py knows
39
- # nothing about pipeline runs, and classify_legacy_pass_threads() below is
40
- # where the two meet.
41
- from storage import THREAD_KIND_CHAT, THREAD_KIND_RESEARCH_PASS
42
-
43
- logger = logging.getLogger(__name__)
44
-
45
-
46
- class PipelineRunRecord(SQLModel, table=True):
47
- """One row per stream_deep_research_pipeline() invocation. `status` is
48
- 'running' from creation until the wrapping try/except in the
49
- orchestrator flips it to 'complete' or 'failed'. The four data
50
- columns carry the run's structured output (or the partial state
51
- available at failure time) so an operator can inspect what happened
52
- without a separate trace log."""
53
-
54
- id: UUID = Field(default_factory=uuid4, primary_key=True)
55
- user_query: str = Field(index=True)
56
- status: str = Field(default="running") # "running" | "complete" | "failed"
57
- claims_json: str = "[]"
58
- trace_json: str = "[]"
59
- coverage_gaps_json: str = "[]"
60
- # §27 (T2): [{"pass": ..., "thread_id": ...}] for this run's pass
61
- # threads. Persisted rather than left to the output directory alone
62
- # (where granted_calls and subagent_reviews live) because those are
63
- # written once, at the end, by a run that finished -- and a run whose
64
- # passes are worth chasing through their threads is frequently one
65
- # that did not. Checkpointed like the other data columns, so an
66
- # abandoned run keeps the ids it had reached.
67
- pass_threads_json: str = "[]"
68
- # E13/#77: [{"candidate", "provider_name", "model", "chars"}] per
69
- # ensemble-Pass-1 SURVIVOR, so a stored claim's
70
- # asserted_by_candidates names models this row can still identify.
71
- # Metadata only: the full texts are per-candidate artifacts and pass
72
- # threads, and update_pipeline_run strips the in-memory `text` key
73
- # here rather than duplicating bodies into the database.
74
- # Checkpointed like the other data columns, so an abandoned run keeps
75
- # what it had reached.
76
- candidates_json: str = "[]"
77
- final_report: str = ""
78
- started_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
79
- finished_at: Optional[datetime] = None
80
-
81
-
82
- # ---------------------------------------------------------------------------
83
- # ---- Public write API ------------------------------------------------------
84
- # ---------------------------------------------------------------------------
85
-
86
- def create_pipeline_run(user_query: str) -> UUID:
87
- """Inserts a 'running' PipelineRunRecord row and returns its id.
88
- The orchestrator assigns this id to PipelineRun.run_id so later
89
- updates can find the row."""
90
- with Session(engine) as session:
91
- record = PipelineRunRecord(user_query=user_query, status="running")
92
- session.add(record)
93
- session.commit()
94
- session.refresh(record)
95
- return record.id
96
-
97
-
98
- def update_pipeline_run(
99
- run_id: UUID,
100
- run,
101
- status: Optional[str] = None,
102
- ) -> None:
103
- """Updates an existing row's data columns (claims / trace /
104
- coverage_gaps / final_report) from the live PipelineRun, and -- only
105
- when `status` is given -- flips status and sets finished_at on a
106
- terminal transition. Raises ValueError if run_id doesn't exist (the
107
- record was never created or was deleted), which is a real bug, not an
108
- expected path.
109
-
110
- `status` is Optional by design: the minimal core only ever passes a
111
- terminal status, but §5 proper's deferred per-pass checkpointing will
112
- call this for plain data snapshots WITHOUT a status. Defaulting status
113
- to a value here would silently reset a 'complete'/'failed' record back
114
- to that default on a data-only update; Optional=None means "don't touch
115
- status unless told to," which makes that future extension safe by
116
- construction.
117
-
118
- Claims are serialized with vars(c), matching every other
119
- claim-serialization call site in orchestrator.py (3a/3b/5/6a/6c/final
120
- synthesis). If a nested-dataclass field is ever added to Claim, ALL of
121
- those sites migrate to asdict(c) together -- see the note on Claim in
122
- core/reasoning/base.py.
123
- """
124
- with Session(engine) as session:
125
- record = session.get(PipelineRunRecord, run_id)
126
- if record is None:
127
- raise ValueError(f"No pipeline run found with id {run_id}")
128
- if status is not None:
129
- record.status = status
130
- if status in ("complete", "failed"):
131
- record.finished_at = datetime.now(timezone.utc)
132
- record.claims_json = json.dumps([vars(c) for c in run.claims])
133
- record.trace_json = json.dumps(run.trace)
134
- record.coverage_gaps_json = json.dumps(run.coverage_gaps)
135
- record.pass_threads_json = json.dumps(run.pass_threads)
136
- # getattr, not run.candidates: this function is called with test
137
- # doubles and with runs reconstructed elsewhere, and a checkpoint
138
- # that raises on a missing optional field would take down the pass
139
- # that was only trying to record its progress.
140
- # `text` rides the in-memory entry (it feeds the artifacts and
141
- # Pass 3b) and is stripped HERE -- the database carries metadata
142
- # only; see PipelineRunRecord's column comment.
143
- record.candidates_json = json.dumps([
144
- {k: v for k, v in entry.items() if k != "text"}
145
- for entry in getattr(run, "candidates", []) or []
146
- ])
147
- record.final_report = run.final_report
148
- session.add(record)
149
- session.commit()
150
-
151
-
152
- # ---------------------------------------------------------------------------
153
- # ---- Public read API -------------------------------------------------------
154
- # ---------------------------------------------------------------------------
155
-
156
- def load_pipeline_run(run_id: UUID) -> dict:
157
- """Loads a PipelineRunRecord by id and returns its state as a plain
158
- dict. Raises ValueError if run_id doesn't exist (matching
159
- update_pipeline_run's convention -- a missing record is a real bug,
160
- not an expected path).
161
-
162
- Claims are returned as raw dicts (the shape vars(c) produced at write
163
- time), not reconstructed Claim dataclass instances. If/when
164
- programmatic pipeline resume is built, that function reconstructs
165
- Claim objects itself, using the same field-filtering pattern
166
- _claim_from_json already established in orchestrator.py.
167
- """
168
- with Session(engine) as session:
169
- record = session.get(PipelineRunRecord, run_id)
170
- if record is None:
171
- raise ValueError(f"No pipeline run found with id {run_id}")
172
- return {
173
- "id": record.id,
174
- "user_query": record.user_query,
175
- "status": record.status,
176
- "started_at": record.started_at,
177
- "finished_at": record.finished_at,
178
- "claims": json.loads(record.claims_json),
179
- "trace": json.loads(record.trace_json),
180
- "coverage_gaps": json.loads(record.coverage_gaps_json),
181
- # §27 AC6. Defaulted rather than read straight off the record:
182
- # a row written before this column existed has it as NULL after
183
- # ensure_columns' ALTER (the migration is additive and never
184
- # backfills), and json.loads(None) raises.
185
- "pass_threads": json.loads(record.pass_threads_json or "[]"),
186
- # Same NULL guard as pass_threads above: ensure_columns'
187
- # ALTER never backfills, so a row written before this column
188
- # existed reads as NULL, and json.loads(None) raises.
189
- "candidates": json.loads(record.candidates_json or "[]"),
190
- "final_report": record.final_report,
191
- }
192
-
193
-
194
- # ---------------------------------------------------------------------------
195
- # ---- One-time classification of pre-§27 threads ---------------------------
196
- # ---------------------------------------------------------------------------
197
-
198
- def classify_legacy_pass_threads(connection) -> int:
199
- """Label pass threads that predate ConversationThread.kind. Returns the
200
- number of rows relabelled.
201
-
202
- WHY THIS IS NOT PART OF ensure_columns(). That function is additive
203
- only and explicitly never backfills (#21 M7) -- so every row already on
204
- disk takes `kind='chat'` from the column default, and the hundreds of
205
- pass threads in an existing database would still fill the picker #27
206
- exists to clear. Widening ensure_columns' contract to backfill would
207
- put data policy inside a schema migrator; if that function ever grows a
208
- DROP or an UPDATE, the project has outgrown it.
209
-
210
- WHY IT LIVES HERE rather than in storage.py. The signal is a pipeline
211
- fact -- which threads belong to which run -- and storage.py owns schema
212
- and CRUD without knowing what a run is. The dependency runs this way
213
- round only; storage.py does not import this module.
214
-
215
- TWO TIERS, EXACT FIRST (#82). Since T2, every run records the exact
216
- thread ids of its passes in `pass_threads_json`. That record is
217
- authoritative over any clock comparison -- so:
218
-
219
- * TIER 1 (exact): every thread id named in a populated
220
- `pass_threads_json` is a pass thread, whatever its timestamp says
221
- and whether or not the run ever finished. Membership was recorded
222
- at run time; there is nothing to infer.
223
- * TIER 2 (window): only runs whose `pass_threads_json` is absent,
224
- NULL, empty, or unusable fall back to the time-window heuristic --
225
- "created between started_at and finished_at". That set is the
226
- pre-T2 population, WHICH CANNOT GROW: a run recorded after T2
227
- always names its own passes, so its window is never consulted and
228
- a chat thread another session creates during it is safe (#82's
229
- headline hazard).
230
-
231
- CORRUPTED JSON FALLS BACK, LOUDLY. An unparseable `pass_threads_json`
232
- puts THAT run in the window tier rather than crashing the launch --
233
- the under-classification direction, which this docstring already
234
- prefers -- and warns naming the run, because silently ignoring data
235
- the writer did write is how the corruption spreads unnoticed.
236
-
237
- A RUN WITH finished_at IS NULL GETS ONLY TIER 1. The window needs a
238
- closing bound: #22's abandoned-generator case deliberately leaves
239
- status='running', so an open window would swallow every thread created
240
- since -- including real conversations. Tier 1 needs no bound (exact
241
- ids), so an abandoned run's recorded passes ARE relabelled.
242
- Under-classifying leaves a straggler in the picker; over-classifying
243
- HIDES A REAL CONVERSATION, and only one of those is recoverable by the
244
- user.
245
-
246
- IDEMPOTENT BY CONSTRUCTION. It only ever moves rows still 'chat', so
247
- running it at every launch is a no-op after the first -- and a database
248
- last opened by an older build still gets classified, both for its
249
- pre-T2 rows (window tier) and its post-T2 rows (whose ids were recorded
250
- all along).
251
-
252
- Raw SQL over a DBAPI connection (`engine.raw_connection()` in
253
- production, stdlib `sqlite3.connect(...)` in tests), the same seam
254
- database.ensure_columns() uses and for the same reason: the suite's
255
- fake `sqlmodel` cannot execute DDL, and a seam that only works in
256
- production is not a seam. Does NOT close the connection -- the caller
257
- owns its lifetime. Computation happens in Python (fetch runs once,
258
- fetch chat rows once, one bounded UPDATE per chunk) so the two tiers'
259
- conditions are ordinary code a mutation can target, not a correlated
260
- subquery whose bounds the suite could only probe end-to-end.
261
-
262
- Threads a legacy compactor or subagent created are relabelled
263
- 'research_pass' here if they happen to fall inside a PRE-T2 run's
264
- window, and left as 'chat' otherwise. Both acceptable: the first is
265
- hidden either way, and the second is under-classification.
266
- """
267
- cursor = connection.cursor()
268
- cursor.execute("PRAGMA table_info(conversationthread)")
269
- if not any(row[1] == "kind" for row in cursor.fetchall()):
270
- # No column yet, so nothing to classify. create_all/ensure_columns
271
- # run before this in production; a caller that skipped them gets a
272
- # no-op rather than "no such column".
273
- return 0
274
- cursor.execute("PRAGMA table_info(pipelinerunrecord)")
275
- run_columns = {row[1] for row in cursor.fetchall()}
276
- if not run_columns:
277
- # A database that has never run research. Nothing to classify, and
278
- # neither query below would find its table.
279
- return 0
280
- has_recorded = "pass_threads_json" in run_columns
281
-
282
- # ---- Fetch the runs once, and sort each into a tier -------------------
283
- cursor.execute(
284
- "SELECT id, started_at, finished_at"
285
- + (", pass_threads_json" if has_recorded else "")
286
- + " FROM pipelinerunrecord")
287
- run_rows = cursor.fetchall()
288
-
289
- exact_ids = set() # tier 1: named in some run's pass_threads_json
290
- window_runs = [] # tier 2: (started_at, finished_at) pairs
291
- for row in run_rows:
292
- run_id, started_at, finished_at = row[0], row[1], row[2]
293
- raw = row[3] if has_recorded else None
294
- usable, corrupted = False, False
295
- if raw:
296
- try:
297
- entries = json.loads(raw)
298
- except ValueError:
299
- corrupted = True
300
- else:
301
- if not isinstance(entries, list):
302
- corrupted = True
303
- else:
304
- # T2's shape: [{"pass": ..., "thread_id": ...}]. Entries
305
- # without a usable thread_id contribute nothing either
306
- # way; an entry list that is merely EMPTY is the
307
- # legitimate pre-T2 shape, i.e. window tier.
308
- for entry in entries:
309
- if (isinstance(entry, dict)
310
- and isinstance(entry.get("thread_id"), str)):
311
- exact_ids.add(entry["thread_id"])
312
- usable = True
313
- # raw None/"" (or a missing column) falls through with usable=False:
314
- # the legitimate pre-T2 shape, i.e. window tier -- not corruption.
315
- if corrupted:
316
- # #82, owner decision B3: fall back for THIS run, and say so --
317
- # data the writer did write must not vanish without a trace.
318
- logger.warning(
319
- "Classify pass threads: run %s has unparseable "
320
- "pass_threads_json (%r); using its time window instead.",
321
- run_id, raw[:80] if isinstance(raw, str) else raw)
322
- if not usable:
323
- # Window tier -- but only a CLOSED window. The intent guard
324
- # against a future edit "kindly" treating NULL finished_at as
325
- # open-ended lives here, in ordinary code, where removing it
326
- # turns a test red (it never could under the old correlated
327
- # subquery: SQL's x <= NULL was already not true).
328
- if started_at is not None and finished_at is not None:
329
- window_runs.append((started_at, finished_at))
330
-
331
- # ---- Fetch the still-chat rows once ------------------------------------
332
- cursor.execute(
333
- "SELECT id, created_at FROM conversationthread WHERE kind = ?",
334
- (THREAD_KIND_CHAT,))
335
- chat_rows = cursor.fetchall()
336
-
337
- to_relabel = []
338
- for thread_id, created_at in chat_rows:
339
- if thread_id in exact_ids:
340
- to_relabel.append(thread_id)
341
- elif created_at is not None and any(
342
- start <= created_at <= finish for start, finish in window_runs):
343
- to_relabel.append(thread_id)
344
-
345
- relabelled = 0
346
- # Chunked: SQLite's host-parameter ceiling is finite, and a long-lived
347
- # database can carry thousands of pass threads.
348
- for offset in range(0, len(to_relabel), 500):
349
- chunk = to_relabel[offset:offset + 500]
350
- placeholders = ",".join("?" * len(chunk))
351
- cursor.execute(
352
- f"UPDATE conversationthread SET kind = ? "
353
- f" WHERE kind = ? AND id IN ({placeholders})",
354
- (THREAD_KIND_RESEARCH_PASS, THREAD_KIND_CHAT, *chunk))
355
- relabelled += cursor.rowcount if cursor.rowcount > 0 else 0
356
- connection.commit()
357
- if relabelled:
358
- logger.info(
359
- "Classified %d pre-\u00a727 thread(s) as research passes; they no "
360
- "longer appear in the thread picker.", relabelled)
361
- return relabelled
1
+ """
2
+ core/reasoning/pipeline_storage.py
3
+
4
+ Persistence layer for deep-research pipeline runs (ROADMAP §5). A
5
+ PipelineRunRecord row is created at the start of
6
+ stream_deep_research_pipeline(), checkpointed with a data-only
7
+ update_pipeline_run() call after every pass's trace-log line, and
8
+ flipped to a terminal status (complete / failed) at the end. The
9
+ load_pipeline_run() read API returns the stored state as a plain dict
10
+ for inspection and debugging.
11
+
12
+ Schema deliberately mirrors the codebase convention established by
13
+ storage.py's MessageLog: semantically distinct data lives in separate
14
+ typed columns (claims_json / trace_json / coverage_gaps_json /
15
+ pass_threads_json / candidates_json / final_report), NOT one catch-all
16
+ blob. The *_json columns hold json.dumps'd strings (same pattern as
17
+ MessageLog.content); final_report is plain text. This keeps a serialization problem in one field from
18
+ corrupting the others -- a persistence layer built for crash-resilience
19
+ shouldn't itself be a single point of failure.
20
+
21
+ Tables created here are registered in SQLModel.metadata and built by
22
+ database.create_db_and_tables(), the same startup hook that creates
23
+ ConversationThread / MessageLog.
24
+ """
25
+
26
+ from __future__ import annotations
27
+
28
+ import json
29
+ import logging
30
+ from datetime import UTC, datetime
31
+ from typing import Optional
32
+ from uuid import UUID, uuid4
33
+
34
+ from sqlmodel import Field, Session, SQLModel
35
+
36
+ from database import engine
37
+
38
+ # §27: the thread-kind vocabulary lives beside the table that stores it.
39
+ # This import direction is the only one that exists -- storage.py knows
40
+ # nothing about pipeline runs, and classify_legacy_pass_threads() below is
41
+ # where the two meet.
42
+ from storage import THREAD_KIND_CHAT, THREAD_KIND_RESEARCH_PASS
43
+
44
+ logger = logging.getLogger(__name__)
45
+
46
+
47
+ class PipelineRunRecord(SQLModel, table=True):
48
+ """One row per stream_deep_research_pipeline() invocation. `status` is
49
+ 'running' from creation until the wrapping try/except in the
50
+ orchestrator flips it to 'complete' or 'failed'. The SIX data columns
51
+ carry the run's structured output (or the partial state available at
52
+ failure time) so an operator can inspect what happened without a
53
+ separate trace log.
54
+
55
+ SIX, not the four this paragraph said until batch 154: T2 added
56
+ pass_threads_json and E13 added candidates_json, and neither batch
57
+ found the enumerations written about them. The count is pinned by
58
+ test_the_documented_record_shape_matches_what_load_returns, which
59
+ reads it out of the prose rather than out of a second list here."""
60
+
61
+ id: UUID = Field(default_factory=uuid4, primary_key=True)
62
+ user_query: str = Field(index=True)
63
+ status: str = Field(default="running") # "running" | "complete" | "failed"
64
+ claims_json: str = "[]"
65
+ trace_json: str = "[]"
66
+ coverage_gaps_json: str = "[]"
67
+ # §27 (T2): [{"pass": ..., "thread_id": ...}] for this run's pass
68
+ # threads. Persisted rather than left to the output directory alone
69
+ # (where granted_calls and subagent_reviews live) because those are
70
+ # written once, at the end, by a run that finished -- and a run whose
71
+ # passes are worth chasing through their threads is frequently one
72
+ # that did not. Checkpointed like the other data columns, so an
73
+ # abandoned run keeps the ids it had reached.
74
+ pass_threads_json: str = "[]"
75
+ # E13/#77: [{"candidate", "provider_name", "model", "chars"}] per
76
+ # ensemble-Pass-1 SURVIVOR, so a stored claim's
77
+ # asserted_by_candidates names models this row can still identify.
78
+ # Metadata only: the full texts are per-candidate artifacts and pass
79
+ # threads, and update_pipeline_run strips the in-memory `text` key
80
+ # here rather than duplicating bodies into the database.
81
+ # Checkpointed like the other data columns, so an abandoned run keeps
82
+ # what it had reached.
83
+ candidates_json: str = "[]"
84
+ final_report: str = ""
85
+ started_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
86
+ finished_at: Optional[datetime] = None
87
+
88
+
89
+ # ---------------------------------------------------------------------------
90
+ # ---- Public write API ------------------------------------------------------
91
+ # ---------------------------------------------------------------------------
92
+
93
+ def create_pipeline_run(user_query: str) -> UUID:
94
+ """Inserts a 'running' PipelineRunRecord row and returns its id.
95
+ The orchestrator assigns this id to PipelineRun.run_id so later
96
+ updates can find the row."""
97
+ with Session(engine) as session:
98
+ record = PipelineRunRecord(user_query=user_query, status="running")
99
+ session.add(record)
100
+ session.commit()
101
+ session.refresh(record)
102
+ return record.id
103
+
104
+
105
+ def update_pipeline_run(
106
+ run_id: UUID,
107
+ run,
108
+ status: Optional[str] = None,
109
+ ) -> None:
110
+ """Updates an existing row's data columns (claims / trace /
111
+ coverage_gaps / pass_threads / candidates / final_report) from the
112
+ live PipelineRun, and -- only when `status` is given -- flips status
113
+ and sets finished_at on a terminal transition. Raises ValueError if run_id doesn't exist (the
114
+ record was never created or was deleted), which is a real bug, not an
115
+ expected path.
116
+
117
+ `status` is Optional by design: the minimal core only ever passes a
118
+ terminal status, but §5 proper's deferred per-pass checkpointing will
119
+ call this for plain data snapshots WITHOUT a status. Defaulting status
120
+ to a value here would silently reset a 'complete'/'failed' record back
121
+ to that default on a data-only update; Optional=None means "don't touch
122
+ status unless told to," which makes that future extension safe by
123
+ construction.
124
+
125
+ Claims are serialized with vars(c), matching every other
126
+ claim-serialization call site in orchestrator.py (3a/3b/5/6a/6c/final
127
+ synthesis). If a nested-dataclass field is ever added to Claim, ALL of
128
+ those sites migrate to asdict(c) together -- see the note on Claim in
129
+ core/reasoning/base.py.
130
+ """
131
+ with Session(engine) as session:
132
+ record = session.get(PipelineRunRecord, run_id)
133
+ if record is None:
134
+ raise ValueError(f"No pipeline run found with id {run_id}")
135
+ if status is not None:
136
+ record.status = status
137
+ if status in ("complete", "failed"):
138
+ record.finished_at = datetime.now(UTC)
139
+ record.claims_json = json.dumps([vars(c) for c in run.claims])
140
+ record.trace_json = json.dumps(run.trace)
141
+ record.coverage_gaps_json = json.dumps(run.coverage_gaps)
142
+ record.pass_threads_json = json.dumps(run.pass_threads)
143
+ # getattr, not run.candidates: this function is called with test
144
+ # doubles and with runs reconstructed elsewhere, and a checkpoint
145
+ # that raises on a missing optional field would take down the pass
146
+ # that was only trying to record its progress.
147
+ # `text` rides the in-memory entry (it feeds the artifacts and
148
+ # Pass 3b) and is stripped HERE -- the database carries metadata
149
+ # only; see PipelineRunRecord's column comment.
150
+ record.candidates_json = json.dumps([
151
+ {k: v for k, v in entry.items() if k != "text"}
152
+ for entry in getattr(run, "candidates", []) or []
153
+ ])
154
+ record.final_report = run.final_report
155
+ session.add(record)
156
+ session.commit()
157
+
158
+
159
+ # ---------------------------------------------------------------------------
160
+ # ---- Public read API -------------------------------------------------------
161
+ # ---------------------------------------------------------------------------
162
+
163
+ def load_pipeline_run(run_id: UUID) -> dict:
164
+ """Loads a PipelineRunRecord by id and returns its state as a plain
165
+ dict. Raises ValueError if run_id doesn't exist (matching
166
+ update_pipeline_run's convention -- a missing record is a real bug,
167
+ not an expected path).
168
+
169
+ Claims are returned as raw dicts (the shape vars(c) produced at write
170
+ time), not reconstructed Claim dataclass instances. If/when
171
+ programmatic pipeline resume is built, that function reconstructs
172
+ Claim objects itself, using the same field-filtering pattern
173
+ _claim_from_json already established in orchestrator.py.
174
+ """
175
+ with Session(engine) as session:
176
+ record = session.get(PipelineRunRecord, run_id)
177
+ if record is None:
178
+ raise ValueError(f"No pipeline run found with id {run_id}")
179
+ return {
180
+ "id": record.id,
181
+ "user_query": record.user_query,
182
+ "status": record.status,
183
+ "started_at": record.started_at,
184
+ "finished_at": record.finished_at,
185
+ "claims": json.loads(record.claims_json),
186
+ "trace": json.loads(record.trace_json),
187
+ "coverage_gaps": json.loads(record.coverage_gaps_json),
188
+ # §27 AC6. The `or "[]"` is for a NULL written BY HAND, and
189
+ # until #258 this said a migrated row was one. It is not:
190
+ # ensure_columns adds the column with DEFAULT '[]', and SQLite
191
+ # reads every row older than the ALTER as that default. A NULL
192
+ # takes a database first migrated before #26 carried NOT NULL
193
+ # (2026-08-06 to 08-22), where the column is nullable, and SQL
194
+ # nobody here wrote -- on which json.loads(None) would raise.
195
+ "pass_threads": json.loads(record.pass_threads_json or "[]"),
196
+ # E13 arrived after #26, so this column is NOT NULL on every
197
+ # database this project has built, and SQLite refuses the NULL
198
+ # this would read past. Kept as the reader's own floor beside
199
+ # its sibling's; tests/test_storage_e2e.py pins both facts.
200
+ "candidates": json.loads(record.candidates_json or "[]"),
201
+ "final_report": record.final_report,
202
+ }
203
+
204
+
205
+ # ---------------------------------------------------------------------------
206
+ # ---- One-time classification of pre-§27 threads ---------------------------
207
+ # ---------------------------------------------------------------------------
208
+
209
+ def _as_uuid_key(value) -> Optional[str]:
210
+ """One id's single spelling, or None for anything that is not a UUID.
211
+
212
+ THE TWO SIDES OF TIER 1 ARE SPELT DIFFERENTLY, and until batch 154
213
+ they were compared literally. `pass_threads_json` carries
214
+ `str(thread_id)` -- the dashed 36-character form, which is also what
215
+ output_writer.py ships as an artifact and what an operator reads --
216
+ while `conversationthread.id` comes back from the raw DBAPI as the
217
+ 32-character hex sqlmodel stores. `'a-b' in {'ab'}` is False for
218
+ every id there has ever been, so the tier #82 calls authoritative
219
+ matched nothing, ever. Measured end to end on real SQLite (the
220
+ suite's fake sqlmodel cannot show a storage format): recorded
221
+ 'acb451c5-9c50-...', stored 'acb451c59c50...', relabelled 0.
222
+
223
+ NONE RATHER THAN THE STRING ITSELF for a value that does not parse. A
224
+ thread id IS a UUID (`ConversationThread.id`), so a record holding
225
+ something else is not an id this function can speak about, and
226
+ deciding that two unparseable strings denote one thread is the
227
+ over-classifying direction -- the one that hides a real conversation.
228
+ The caller treats None as "no tier-1 answer", which is how an entry
229
+ with no usable thread_id has always been treated.
230
+
231
+ ONE SIDE, NOT TWO, AND THE OTHER SIDE IS PINNED. This normalises the
232
+ RECORDED ids only; the row ids are compared as the database hands
233
+ them over. Normalising both was written first, and the control run
234
+ as mutation C1 found it unearned -- removing the row-side call left
235
+ every test green, because `conversationthread.id` is hex and so is
236
+ this. The remaining assumption is that the column holds hex, which
237
+ is a fact about sqlmodel's UUID type rather than about this project,
238
+ so it is ASSERTED where it can be:
239
+ `test_a_thread_id_is_stored_in_a_spelling_str_does_not_produce` in
240
+ tests/test_storage_e2e.py fails naming this function if the library
241
+ ever starts storing the dashed form. A pinned dependency beats a
242
+ branch for a spelling nothing here has ever written.
243
+ """
244
+ try:
245
+ return UUID(value).hex
246
+ except (ValueError, AttributeError, TypeError):
247
+ return None
248
+
249
+
250
+ def classify_legacy_pass_threads(connection) -> int:
251
+ """Label pass threads that predate ConversationThread.kind. Returns the
252
+ number of rows relabelled.
253
+
254
+ WHY THIS IS NOT PART OF ensure_columns(). That function is additive
255
+ only and explicitly never backfills (#21 M7) -- so every row already on
256
+ disk takes `kind='chat'` from the column default, and the hundreds of
257
+ pass threads in an existing database would still fill the picker #27
258
+ exists to clear. Widening ensure_columns' contract to backfill would
259
+ put data policy inside a schema migrator; if that function ever grows a
260
+ DROP or an UPDATE, the project has outgrown it.
261
+
262
+ WHY IT LIVES HERE rather than in storage.py. The signal is a pipeline
263
+ fact -- which threads belong to which run -- and storage.py owns schema
264
+ and CRUD without knowing what a run is. The dependency runs this way
265
+ round only; storage.py does not import this module.
266
+
267
+ TWO TIERS, EXACT FIRST (#82). Since T2, every run records the exact
268
+ thread ids of its passes in `pass_threads_json`. That record is
269
+ authoritative over any clock comparison -- so:
270
+
271
+ * TIER 1 (exact): every thread id named in a populated
272
+ `pass_threads_json` is a pass thread, whatever its timestamp says
273
+ and whether or not the run ever finished. Membership was recorded
274
+ at run time; there is nothing to infer. COMPARED AS UUIDS, NOT AS
275
+ STRINGS -- the two sides are spelt differently and `_as_uuid_key`
276
+ below is where they are made one thing (batch 154).
277
+ * TIER 2 (window): only runs whose `pass_threads_json` is absent,
278
+ NULL, empty, or unusable fall back to the time-window heuristic --
279
+ "created between started_at and finished_at". That set is the
280
+ pre-T2 population, WHICH CANNOT GROW: a run recorded after T2
281
+ always names its own passes, so its window is never consulted and
282
+ a chat thread another session creates during it is safe (#82's
283
+ headline hazard).
284
+
285
+ CORRUPTED JSON FALLS BACK, LOUDLY. An unparseable `pass_threads_json`
286
+ puts THAT run in the window tier rather than crashing the launch --
287
+ the under-classification direction, which this docstring already
288
+ prefers -- and warns naming the run, because silently ignoring data
289
+ the writer did write is how the corruption spreads unnoticed.
290
+
291
+ A RUN WITH finished_at IS NULL GETS ONLY TIER 1. The window needs a
292
+ closing bound: #22's abandoned-generator case deliberately leaves
293
+ status='running', so an open window would swallow every thread created
294
+ since -- including real conversations. Tier 1 needs no bound (exact
295
+ ids), so an abandoned run's recorded passes ARE relabelled.
296
+ Under-classifying leaves a straggler in the picker; over-classifying
297
+ HIDES A REAL CONVERSATION, and only one of those is recoverable by the
298
+ user.
299
+
300
+ IDEMPOTENT BY CONSTRUCTION. It only ever moves rows still 'chat', so
301
+ running it at every launch is a no-op after the first -- and a database
302
+ last opened by an older build still gets classified, both for its
303
+ pre-T2 rows (window tier) and its post-T2 rows (whose ids were recorded
304
+ all along).
305
+
306
+ Raw SQL over a DBAPI connection (`engine.raw_connection()` in
307
+ production, stdlib `sqlite3.connect(...)` in tests), the same seam
308
+ database.ensure_columns() uses and for the same reason: the suite's
309
+ fake `sqlmodel` cannot execute DDL, and a seam that only works in
310
+ production is not a seam. Does NOT close the connection -- the caller
311
+ owns its lifetime. Computation happens in Python (fetch runs once,
312
+ fetch chat rows once, one bounded UPDATE per chunk) so the two tiers'
313
+ conditions are ordinary code a mutation can target, not a correlated
314
+ subquery whose bounds the suite could only probe end-to-end.
315
+
316
+ Threads a legacy compactor or subagent created are relabelled
317
+ 'research_pass' here if they happen to fall inside a PRE-T2 run's
318
+ window, and left as 'chat' otherwise. Both acceptable: the first is
319
+ hidden either way, and the second is under-classification.
320
+ """
321
+ cursor = connection.cursor()
322
+ cursor.execute("PRAGMA table_info(conversationthread)")
323
+ if not any(row[1] == "kind" for row in cursor.fetchall()):
324
+ # No column yet, so nothing to classify. create_all/ensure_columns
325
+ # run before this in production; a caller that skipped them gets a
326
+ # no-op rather than "no such column".
327
+ return 0
328
+ cursor.execute("PRAGMA table_info(pipelinerunrecord)")
329
+ run_columns = {row[1] for row in cursor.fetchall()}
330
+ if not run_columns:
331
+ # A database that has never run research. Nothing to classify, and
332
+ # neither query below would find its table.
333
+ return 0
334
+ has_recorded = "pass_threads_json" in run_columns
335
+
336
+ # ---- Fetch the runs once, and sort each into a tier -------------------
337
+ cursor.execute(
338
+ "SELECT id, started_at, finished_at"
339
+ + (", pass_threads_json" if has_recorded else "")
340
+ + " FROM pipelinerunrecord")
341
+ run_rows = cursor.fetchall()
342
+
343
+ exact_ids = set() # tier 1: named in some run's pass_threads_json
344
+ window_runs = [] # tier 2: (started_at, finished_at) pairs
345
+ for row in run_rows:
346
+ run_id, started_at, finished_at = row[0], row[1], row[2]
347
+ raw = row[3] if has_recorded else None
348
+ usable, corrupted = False, False
349
+ if raw:
350
+ try:
351
+ entries = json.loads(raw)
352
+ except ValueError:
353
+ corrupted = True
354
+ else:
355
+ if not isinstance(entries, list):
356
+ corrupted = True
357
+ else:
358
+ # T2's shape: [{"pass": ..., "thread_id": ...}]. Entries
359
+ # without a usable thread_id contribute nothing either
360
+ # way; an entry list that is merely EMPTY is the
361
+ # legitimate pre-T2 shape, i.e. window tier.
362
+ for entry in entries:
363
+ if (isinstance(entry, dict)
364
+ and isinstance(entry.get("thread_id"), str)):
365
+ # POPULATED is what puts this run in tier 1,
366
+ # whatever its ids turn out to parse as. An id
367
+ # this function cannot read contributes no
368
+ # membership, but it must not drop the run to
369
+ # the window either: that would consult a
370
+ # modern run's clock and hide the concurrent
371
+ # chat thread #82's headline is about.
372
+ usable = True
373
+ key = _as_uuid_key(entry["thread_id"])
374
+ if key is not None:
375
+ exact_ids.add(key)
376
+ # raw None/"" (or a missing column) falls through with usable=False:
377
+ # the legitimate pre-T2 shape, i.e. window tier -- not corruption.
378
+ if corrupted:
379
+ # #82, owner decision B3: fall back for THIS run, and say so --
380
+ # data the writer did write must not vanish without a trace.
381
+ logger.warning(
382
+ "Classify pass threads: run %s has unparseable "
383
+ "pass_threads_json (%r); using its time window instead.",
384
+ run_id, raw[:80] if isinstance(raw, str) else raw)
385
+ if not usable:
386
+ # Window tier -- but only a CLOSED window. The intent guard
387
+ # against a future edit "kindly" treating NULL finished_at as
388
+ # open-ended lives here, in ordinary code, where removing it
389
+ # turns a test red (it never could under the old correlated
390
+ # subquery: SQL's x <= NULL was already not true).
391
+ if started_at is not None and finished_at is not None:
392
+ window_runs.append((started_at, finished_at))
393
+
394
+ # ---- Fetch the still-chat rows once ------------------------------------
395
+ cursor.execute(
396
+ "SELECT id, created_at FROM conversationthread WHERE kind = ?",
397
+ (THREAD_KIND_CHAT,))
398
+ chat_rows = cursor.fetchall()
399
+
400
+ to_relabel = []
401
+ for thread_id, created_at in chat_rows:
402
+ # `exact_ids` holds hex, which is the spelling this column is in.
403
+ # ONE normalisation, on the recorded side, and the assumption on
404
+ # this side is PINNED rather than branched on -- see _as_uuid_key.
405
+ if thread_id in exact_ids:
406
+ to_relabel.append(thread_id)
407
+ elif created_at is not None and any(
408
+ start <= created_at <= finish for start, finish in window_runs):
409
+ to_relabel.append(thread_id)
410
+
411
+ relabelled = 0
412
+ # Chunked: SQLite's host-parameter ceiling is finite, and a long-lived
413
+ # database can carry thousands of pass threads.
414
+ for offset in range(0, len(to_relabel), 500):
415
+ chunk = to_relabel[offset:offset + 500]
416
+ placeholders = ",".join("?" * len(chunk))
417
+ cursor.execute(
418
+ f"UPDATE conversationthread SET kind = ? "
419
+ f" WHERE kind = ? AND id IN ({placeholders})",
420
+ (THREAD_KIND_RESEARCH_PASS, THREAD_KIND_CHAT, *chunk))
421
+ relabelled += cursor.rowcount if cursor.rowcount > 0 else 0
422
+ connection.commit()
423
+ if relabelled:
424
+ logger.info(
425
+ "Classified %d pre-\u00a727 thread(s) as research passes; they no "
426
+ "longer appear in the thread picker.", relabelled)
427
+ return relabelled