@yahyahammad/venastine 0.2.0 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/.env.example +9 -4
  2. package/CONFIG_ARCHITECTURE.md +2030 -0
  3. package/LICENSE +201 -201
  4. package/NOTICE +1 -1
  5. package/README.md +1034 -895
  6. package/agents/builtin/build.md +73 -0
  7. package/agents/builtin/compactor.md +75 -75
  8. package/agents/builtin/explore.md +77 -77
  9. package/agents/builtin/general.md +80 -0
  10. package/agents/builtin/pipeline-reviewer.md +100 -100
  11. package/agents/builtin/plan.md +106 -88
  12. package/agents/builtin/review.md +74 -74
  13. package/agents/builtin/test.md +65 -0
  14. package/agents/builtin/writer.md +67 -0
  15. package/agents/manager.py +68 -2
  16. package/agents/subagent_tool.py +472 -238
  17. package/agents/tui_commands.py +238 -151
  18. package/bin/venastine.mjs +150 -4
  19. package/bin/venastine.py +71 -0
  20. package/cli_commands.py +838 -0
  21. package/config.py +95 -1198
  22. package/config.yaml +1103 -0
  23. package/config_edit.py +1301 -0
  24. package/config_schema.py +971 -0
  25. package/config_update.py +749 -0
  26. package/core/agent_activity.py +256 -118
  27. package/core/ansi.py +452 -0
  28. package/core/approval.py +137 -115
  29. package/core/call_view.py +364 -0
  30. package/core/cancel.py +121 -0
  31. package/core/chatgpt_auth.py +747 -0
  32. package/core/claims_view.py +107 -0
  33. package/core/client.py +2715 -1559
  34. package/core/commands.py +460 -0
  35. package/core/compaction.py +1735 -971
  36. package/core/config_loader.py +1982 -1563
  37. package/core/connect.py +686 -0
  38. package/core/events.py +123 -61
  39. package/core/file_mentions.py +280 -0
  40. package/core/interaction.py +411 -328
  41. package/core/line_pattern.py +128 -0
  42. package/core/loop.py +2566 -1490
  43. package/core/memory.py +482 -370
  44. package/core/pipeline_models.py +190 -176
  45. package/core/provider_errors.py +234 -0
  46. package/core/reasoning/authorization.py +130 -130
  47. package/core/reasoning/base.py +219 -219
  48. package/core/reasoning/confidence_scoring.py +484 -468
  49. package/core/reasoning/events.py +149 -149
  50. package/core/reasoning/json_retry.py +19 -1
  51. package/core/reasoning/orchestrator.py +1561 -1382
  52. package/core/reasoning/output_writer.py +345 -286
  53. package/core/reasoning/payload_validation.py +524 -398
  54. package/core/reasoning/pipeline_storage.py +88 -22
  55. package/core/reasoning/review.py +659 -615
  56. package/core/reasoning/scholar.py +458 -434
  57. package/core/reasoning/source_corpus.py +433 -327
  58. package/core/reasoning/source_scoring.py +855 -855
  59. package/core/replay.py +315 -183
  60. package/core/session.py +36 -12
  61. package/core/session_view.py +259 -0
  62. package/core/session_wake.py +259 -0
  63. package/core/shell_sessions.py +1553 -0
  64. package/core/workspace_trust.py +1 -1
  65. package/credentials.py +148 -20
  66. package/database.py +333 -252
  67. package/{PRIVACY.md → docs/PRIVACY.md} +78 -66
  68. package/docs/SECURITY.md +108 -0
  69. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  70. package/env_secrets.py +50 -9
  71. package/json_store.py +343 -123
  72. package/logging_setup.py +295 -185
  73. package/main.py +3433 -1961
  74. package/mcp_client/client.py +733 -499
  75. package/mcp_client/config.py +126 -21
  76. package/mcp_client/registration.py +343 -182
  77. package/memories/manager.py +147 -132
  78. package/memories/tui_commands.py +319 -244
  79. package/package.json +303 -73
  80. package/project_init/config_files.py +282 -278
  81. package/project_init/generator.py +728 -564
  82. package/project_init/manifest.py +18 -6
  83. package/project_init/tui_commands.py +147 -144
  84. package/prompts/critic_pass.md +24 -24
  85. package/prompts/revalidate.md +25 -25
  86. package/prompts/revise.md +2 -0
  87. package/prompts/source_grounding.md +66 -66
  88. package/prompts/system_prompts.py +316 -262
  89. package/providers.json.example +4 -2
  90. package/pyproject.toml +149 -100
  91. package/requirements.txt +295 -72
  92. package/safety/policy_enforcement.py +1082 -746
  93. package/security/askpass.py +419 -0
  94. package/security/capability.py +207 -177
  95. package/security/confine.py +484 -0
  96. package/security/hosted.py +182 -0
  97. package/security/permissions.py +237 -190
  98. package/security/posture.py +124 -12
  99. package/security/protected_paths.py +501 -309
  100. package/security/sandbox.py +4108 -985
  101. package/security/secrets.py +855 -0
  102. package/skills/builtin/math/formal-specification.md +67 -67
  103. package/skills/builtin/math/numerical-methods.md +66 -66
  104. package/skills/builtin/research/experiment-design.md +62 -62
  105. package/skills/builtin/research/reproducibility.md +55 -55
  106. package/skills/builtin/research/source-evaluation.md +70 -70
  107. package/skills/builtin/research/statistical-inference.md +63 -63
  108. package/skills/builtin/research/technical-writing.md +59 -59
  109. package/skills/builtin/software/code-review.md +61 -61
  110. package/skills/builtin/software/debugging.md +61 -61
  111. package/skills/builtin/software/test-design.md +67 -67
  112. package/skills/tui_commands.py +110 -111
  113. package/storage.py +1338 -1005
  114. package/tools/base.py +161 -1
  115. package/tools/builtin/_math_common.py +667 -377
  116. package/tools/builtin/_net_common.py +225 -19
  117. package/tools/builtin/arxiv.py +497 -39
  118. package/tools/builtin/ask_user.py +231 -213
  119. package/tools/builtin/discrete_math.py +35 -4
  120. package/tools/builtin/fetch_url.py +371 -153
  121. package/tools/builtin/file_ops.py +720 -412
  122. package/tools/builtin/geometry.py +48 -12
  123. package/tools/builtin/get_time.py +2 -2
  124. package/tools/builtin/linear_algebra.py +66 -23
  125. package/tools/builtin/load_skill.py +76 -66
  126. package/tools/builtin/logic.py +171 -12
  127. package/tools/builtin/pin.py +227 -193
  128. package/tools/builtin/probability_stats.py +12 -16
  129. package/tools/builtin/project_docs.py +464 -405
  130. package/tools/builtin/remember.py +175 -158
  131. package/tools/builtin/shell.py +980 -461
  132. package/tools/builtin/shell_sessions.py +613 -0
  133. package/tools/builtin/symbolic_math.py +80 -10
  134. package/tools/builtin/todo.py +149 -147
  135. package/tools/builtin/web_search.py +191 -13
  136. package/tools/context.py +159 -136
  137. package/tools/isolation.py +107 -34
  138. package/tools/registry.py +1242 -925
  139. package/tui/app.py +8318 -4100
  140. package/tui/app.tcss +747 -556
  141. package/tui/commands.py +26 -170
  142. package/tui/diffs.py +34 -13
  143. package/tui/history.py +128 -128
  144. package/tui/hosted_driver.py +102 -0
  145. package/tui/markdown.py +1329 -1279
  146. package/tui/meters.py +364 -268
  147. package/tui/ravens.py +146 -146
  148. package/tui/screens.py +1255 -943
  149. package/tui/themes.py +599 -559
  150. package/tui/widgets.py +3813 -2332
  151. package/SECURITY.md +0 -88
  152. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -1,219 +1,219 @@
1
- """
2
- core/reasoning/base.py
3
-
4
- Structured claim-level data model for the deep-research pipeline. This
5
- REPLACES the earlier free-text PipelineContext/PassResult design -- Pass 5
6
- (confidence tiering), gate D0 (claim-type routing), and D1/D2 (threshold and
7
- retry-cap checks) all need to reason about individual claims, not a
8
- concatenated paragraph of prior-pass prose. Passes that produce
9
- structured output (2, 3a, 3b, 4) mutate Claim objects in place; passes
10
- that produce free text (0, 1, 3c, final synthesis) populate the
11
- corresponding PipelineRun fields directly.
12
- """
13
-
14
- from __future__ import annotations
15
-
16
- from dataclasses import dataclass, field
17
- from typing import Literal, Optional, get_args
18
- from uuid import UUID
19
-
20
- ClaimType = Literal["factual", "synthesis", "speculative"]
21
- ConfidenceTier = Literal["HIGH", "MEDIUM", "LOW", "UNVERIFIED", "UNVERIFIED_COVERAGE"]
22
- GroundingStatus = Literal["grounded", "partial", "ungrounded"]
23
-
24
- # ROADMAP_v2 §30. The Literals above are ANNOTATIONS -- dataclasses do
25
- # not enforce them, and #75 is what that costs: `"Grounded"` scored a
26
- # well-sourced claim 0.35/LOW where `"grounded"` gives 0.85/HIGH, and
27
- # `"Ungrounded"` escaped the forced-UNVERIFIED rule to land LOW. Two
28
- # modules need the vocabulary as DATA -- the payload boundary rejects a
29
- # value outside it, the scorer coerces one that got through -- and a
30
- # second hand-typed copy of a vocabulary is a second thing to keep in
31
- # step. Derived from the Literal, beside the Literal.
32
- CLAIM_TYPES = frozenset(get_args(ClaimType))
33
- GROUNDING_STATUSES = frozenset(get_args(GroundingStatus))
34
-
35
-
36
- @dataclass
37
- class Claim:
38
- """One atomic, self-contained assertion extracted from Pass 1's raw
39
- response by Pass 2. Every later pass mutates fields on this same
40
- object rather than creating parallel data structures keyed by id.
41
-
42
- SERIALIZATION NOTE: EVERY `vars(c)` IN THE TREE serializes this object
43
- shallowly. That is correct only while every field below stays
44
- JSON-native (str / list / dict / float / int / None). If a
45
- nested-dataclass field is EVER added here, ALL of those sites must
46
- migrate to `dataclasses.asdict(c)` TOGETHER, not piecemeal -- hardening
47
- just one leaves the rest exposed and creates two different
48
- serialization mechanisms for the same object in the same codebase.
49
-
50
- grep -rn 'vars(c' --include='*.py' .
51
-
52
- Deliberately a GREP rather than a list. This note used to enumerate
53
- three sites -- orchestrator.py's passes, pipeline_storage and
54
- output_writer -- and §26 added two more in tui/app.py (show_claims and
55
- /copy claims) that fell out of it, and out of AGENTS.md's copy of the
56
- same list (audit #108). The TUI's failure would have been quieter than
57
- the persistence ones, since ClaimsScreen renders a repr into a cell and
58
- `json.dumps(..., default=str)` stringifies rather than raising -- which
59
- is precisely the "two different serialization mechanisms" outcome this
60
- note exists to prevent, arriving through the door the note did not
61
- cover. An enumeration that says ALL has to be complete to mean
62
- anything, and a new consumer in a new section will keep falling out of
63
- one.
64
- """
65
-
66
- id: str
67
- text: str
68
- type: ClaimType
69
- entities: list[str] = field(default_factory=list)
70
- source_span: str = ""
71
-
72
- # Populated by Pass 3a (factual claims only -- synthesis/speculative
73
- # claims skip grounding entirely per the pipeline's gate D0 routing, so
74
- # these stay at their defaults for those claims).
75
- grounding_sources: list[dict] = field(default_factory=list)
76
- grounding_status: Optional[GroundingStatus] = None
77
-
78
- # Populated by Pass 3b (factual claims only, same reason as above).
79
- fallacies: list[str] = field(default_factory=list)
80
- contradictions: list[str] = field(default_factory=list)
81
- critic_severity: float = 0.0
82
-
83
- # Populated by Pass 4 (ALL claims, factual or not).
84
- assumption_flags: list[str] = field(default_factory=list)
85
-
86
- # Populated by Pass 2 in ensemble mode (ROADMAP §10). Lists the
87
- # 1-indexed candidate numbers that independently asserted this claim.
88
- # Empty/unused when ensemble mode is off.
89
- asserted_by_candidates: list[int] = field(default_factory=list)
90
-
91
- # Populated by Pass 5 (pure code, no LLM call) -- see confidence_scoring.py.
92
- confidence_tier: Optional[ConfidenceTier] = None
93
- score_breakdown: dict = field(default_factory=dict)
94
-
95
- # Populated by the D1/6a/6b/D2/fallback retry loop.
96
- revision_text: Optional[str] = None
97
- retry_count: int = 0
98
-
99
- # Populated by 6c (annotate) / fallback -- the actual final text and
100
- # confidence tag a claim carries into the merged output.
101
- final_text: Optional[str] = None
102
- annotation: Optional[str] = None
103
-
104
-
105
- def resolve_by_id(claims: list) -> dict:
106
- """The pipeline's ONE claim-id resolution (ROADMAP_v2 §30, B6).
107
-
108
- There used to be two, and they disagreed. `_apply_grounding` and
109
- `_apply_critic` built `{c.id: c}` -- LAST wins -- while
110
- `_apply_assumption_flags` and `claim_by_id` used
111
- `next(c for c in claims if c.id == claim_id)` -- FIRST wins. With two
112
- claims sharing an id, driven against the real pipeline, that split one
113
- claim in half:
114
-
115
- text='first' grounding=None severity=0.0 flags=['a flag']
116
- text='second' grounding='grounded' severity=0.4 flags=[]
117
-
118
- The first copy is then ungrounded-but-factual, so D1 flags it, and
119
- Pass 6a revises the copy that has no sources while the grounded one
120
- keeps them.
121
-
122
- §30 also rejects a duplicate id at Pass 2's boundary, which stops that
123
- payload arriving. This exists anyway, because the boundary only covers
124
- claims that came through Pass 2 -- §20's corrections and any future
125
- resume build claims without passing this way, and a disagreement that
126
- can only be reached by a path nobody has written yet is still a
127
- disagreement waiting.
128
-
129
- FIRST wins, matching what claim_by_id and Pass 6a's in-batch lookup
130
- already did and documented. The choice is arbitrary once ids are
131
- unique; having one is not.
132
- """
133
- resolved = {}
134
- for claim in claims:
135
- resolved.setdefault(claim.id, claim)
136
- return resolved
137
-
138
-
139
- @dataclass
140
- class PipelineRun:
141
- """
142
- Holds every pass's structured output for one query. `trace` is a
143
- running human-readable audit log -- what was flagged, why, what
144
- changed -- meant to be at least as trustworthy an artifact as the
145
- final report itself.
146
- """
147
-
148
- user_query: str
149
- # Set by stream_deep_research_pipeline() once a PipelineRunRecord is
150
- # created in storage; stays None in tests that bypass persistence.
151
- # ROADMAP §5 (pipeline persistence minimal core): the record lets a
152
- # run that fails mid-pipeline be inspected and (future) resumed.
153
- run_id: Optional[UUID] = None
154
- plan: dict = field(default_factory=dict) # Pass 0
155
- raw_response: str = "" # Pass 1
156
- # E13/#77: one entry per SURVIVOR of ensemble Pass 1, in survivor
157
- # order -- {"candidate", "provider_name", "model", "chars", "text"}.
158
- # `text` is in-memory working data: pipeline_storage strips it when
159
- # persisting (the full texts live in the output directory's
160
- # 01_candidate_N.md files and in each candidate's own pass thread),
161
- # so the database carries metadata only. Empty for every run that
162
- # did not use ensemble mode -- raw_response alone is that run's
163
- # record, exactly as before this field existed.
164
- candidates: list[dict] = field(default_factory=list)
165
- claims: list[Claim] = field(default_factory=list) # Pass 2, mutated by 3a/3b/4/5/6a/6b/6c
166
- completeness: dict = field(default_factory=dict) # Pass 3c: {"gaps": [...], "coverage_score": ...}
167
- coverage_gaps: list[dict] = field(default_factory=list) # Pass 5-derived: gaps tagged UNVERIFIED_COVERAGE
168
- assumptions: dict = field(default_factory=dict) # Pass 4
169
- trace: list[str] = field(default_factory=list)
170
- # §25: tool calls that ran on a PRE-FLIGHT grant rather than a live
171
- # approval, one dict per call ({"pass", "tool", "params"}). Empty for
172
- # every run that granted nothing, which is every run by default.
173
- #
174
- # Authorization moved up-front, so accountability moves after: an
175
- # unattended ten-pass run is precisely where "the user said yes once"
176
- # needs a record of what that yes actually covered.
177
- granted_calls: list[dict] = field(default_factory=list)
178
- # §20: one record per finding the reviewer raised, carrying what it
179
- # proposed and what the human decided. Populated only on a run that
180
- # was reviewed, which is off by default (D9).
181
- #
182
- # A dict, not a dataclass, and deliberately: a nested dataclass field
183
- # here would require migrating EVERY vars(c)/vars(run) site to
184
- # asdict() in one change (see the Claim note above). It is also why
185
- # the reviewer's tier overrides are marked inside the existing
186
- # score_breakdown dict rather than in a new Claim field.
187
- subagent_reviews: list[dict] = field(default_factory=list)
188
- # §27 (T2): one entry per pass thread, {"pass": pass_id,
189
- # "thread_id": str}. Each pass runs in its own fresh thread -- a locked
190
- # invariant, since passes share distilled JSON and never raw history --
191
- # so §27 hides those threads from the picker. This is what keeps them
192
- # REACHABLE anyway: hiding a thread must not orphan it.
193
- #
194
- # A list[dict] for the same reason granted_calls and subagent_reviews
195
- # are: a nested dataclass here forces every vars(run)/vars(c) site to
196
- # asdict() in one change.
197
- pass_threads: list[dict] = field(default_factory=list)
198
- # §45 (SQ2/SQ8): one entry per URL this run actually retrieved text
199
- # from -- {"url", "title", "tool", "retrieved_at", "sha256", "chars",
200
- # "text"}. Populated from the live SourceCorpus once grounding has
201
- # finished; the corpus object itself stays inside the pipeline
202
- # generator.
203
- #
204
- # A list[dict] for the same reason granted_calls, subagent_reviews and
205
- # pass_threads are, and here the reason is sharper: the corpus IS a
206
- # dataclass, and putting it on this object directly is exactly the
207
- # nested-dataclass field the note on Claim above forbids.
208
- #
209
- # In-memory working data, like candidates[].text: pipeline_storage
210
- # names the fields it writes and this is not among them, so a run's
211
- # fetched pages reach output/<run_id>/sources/ and never the database.
212
- source_documents: list[dict] = field(default_factory=list)
213
- final_report: str = ""
214
-
215
- def log(self, message: str) -> None:
216
- self.trace.append(message)
217
-
218
- def claim_by_id(self, claim_id: str) -> Optional[Claim]:
219
- return resolve_by_id(self.claims).get(claim_id)
1
+ """
2
+ core/reasoning/base.py
3
+
4
+ Structured claim-level data model for the deep-research pipeline. This
5
+ REPLACES the earlier free-text PipelineContext/PassResult design -- Pass 5
6
+ (confidence tiering), gate D0 (claim-type routing), and D1/D2 (threshold and
7
+ retry-cap checks) all need to reason about individual claims, not a
8
+ concatenated paragraph of prior-pass prose. Passes that produce
9
+ structured output (2, 3a, 3b, 4) mutate Claim objects in place; passes
10
+ that produce free text (0, 1, 3c, final synthesis) populate the
11
+ corresponding PipelineRun fields directly.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ from dataclasses import dataclass, field
17
+ from typing import Literal, Optional, get_args
18
+ from uuid import UUID
19
+
20
+ ClaimType = Literal["factual", "synthesis", "speculative"]
21
+ ConfidenceTier = Literal["HIGH", "MEDIUM", "LOW", "UNVERIFIED", "UNVERIFIED_COVERAGE"]
22
+ GroundingStatus = Literal["grounded", "partial", "ungrounded"]
23
+
24
+ # ROADMAP_v2 §30. The Literals above are ANNOTATIONS -- dataclasses do
25
+ # not enforce them, and #75 is what that costs: `"Grounded"` scored a
26
+ # well-sourced claim 0.35/LOW where `"grounded"` gives 0.85/HIGH, and
27
+ # `"Ungrounded"` escaped the forced-UNVERIFIED rule to land LOW. Two
28
+ # modules need the vocabulary as DATA -- the payload boundary rejects a
29
+ # value outside it, the scorer coerces one that got through -- and a
30
+ # second hand-typed copy of a vocabulary is a second thing to keep in
31
+ # step. Derived from the Literal, beside the Literal.
32
+ CLAIM_TYPES = frozenset(get_args(ClaimType))
33
+ GROUNDING_STATUSES = frozenset(get_args(GroundingStatus))
34
+
35
+
36
+ @dataclass
37
+ class Claim:
38
+ """One atomic, self-contained assertion extracted from Pass 1's raw
39
+ response by Pass 2. Every later pass mutates fields on this same
40
+ object rather than creating parallel data structures keyed by id.
41
+
42
+ SERIALIZATION NOTE: EVERY `vars(c)` IN THE TREE serializes this object
43
+ shallowly. That is correct only while every field below stays
44
+ JSON-native (str / list / dict / float / int / None). If a
45
+ nested-dataclass field is EVER added here, ALL of those sites must
46
+ migrate to `dataclasses.asdict(c)` TOGETHER, not piecemeal -- hardening
47
+ just one leaves the rest exposed and creates two different
48
+ serialization mechanisms for the same object in the same codebase.
49
+
50
+ grep -rn 'vars(c' --include='*.py' .
51
+
52
+ Deliberately a GREP rather than a list. This note used to enumerate
53
+ three sites -- orchestrator.py's passes, pipeline_storage and
54
+ output_writer -- and §26 added two more in tui/app.py (show_claims and
55
+ /copy claims) that fell out of it, and out of AGENTS.md's copy of the
56
+ same list (audit #108). The TUI's failure would have been quieter than
57
+ the persistence ones, since ClaimsScreen renders a repr into a cell and
58
+ `json.dumps(..., default=str)` stringifies rather than raising -- which
59
+ is precisely the "two different serialization mechanisms" outcome this
60
+ note exists to prevent, arriving through the door the note did not
61
+ cover. An enumeration that says ALL has to be complete to mean
62
+ anything, and a new consumer in a new section will keep falling out of
63
+ one.
64
+ """
65
+
66
+ id: str
67
+ text: str
68
+ type: ClaimType
69
+ entities: list[str] = field(default_factory=list)
70
+ source_span: str = ""
71
+
72
+ # Populated by Pass 3a (factual claims only -- synthesis/speculative
73
+ # claims skip grounding entirely per the pipeline's gate D0 routing, so
74
+ # these stay at their defaults for those claims).
75
+ grounding_sources: list[dict] = field(default_factory=list)
76
+ grounding_status: Optional[GroundingStatus] = None
77
+
78
+ # Populated by Pass 3b (factual claims only, same reason as above).
79
+ fallacies: list[str] = field(default_factory=list)
80
+ contradictions: list[str] = field(default_factory=list)
81
+ critic_severity: float = 0.0
82
+
83
+ # Populated by Pass 4 (ALL claims, factual or not).
84
+ assumption_flags: list[str] = field(default_factory=list)
85
+
86
+ # Populated by Pass 2 in ensemble mode (ROADMAP §10). Lists the
87
+ # 1-indexed candidate numbers that independently asserted this claim.
88
+ # Empty/unused when ensemble mode is off.
89
+ asserted_by_candidates: list[int] = field(default_factory=list)
90
+
91
+ # Populated by Pass 5 (pure code, no LLM call) -- see confidence_scoring.py.
92
+ confidence_tier: Optional[ConfidenceTier] = None
93
+ score_breakdown: dict = field(default_factory=dict)
94
+
95
+ # Populated by the D1/6a/6b/D2/fallback retry loop.
96
+ revision_text: Optional[str] = None
97
+ retry_count: int = 0
98
+
99
+ # Populated by 6c (annotate) / fallback -- the actual final text and
100
+ # confidence tag a claim carries into the merged output.
101
+ final_text: Optional[str] = None
102
+ annotation: Optional[str] = None
103
+
104
+
105
+ def resolve_by_id(claims: list) -> dict:
106
+ """The pipeline's ONE claim-id resolution (ROADMAP_v2 §30, B6).
107
+
108
+ There used to be two, and they disagreed. `_apply_grounding` and
109
+ `_apply_critic` built `{c.id: c}` -- LAST wins -- while
110
+ `_apply_assumption_flags` and `claim_by_id` used
111
+ `next(c for c in claims if c.id == claim_id)` -- FIRST wins. With two
112
+ claims sharing an id, driven against the real pipeline, that split one
113
+ claim in half:
114
+
115
+ text='first' grounding=None severity=0.0 flags=['a flag']
116
+ text='second' grounding='grounded' severity=0.4 flags=[]
117
+
118
+ The first copy is then ungrounded-but-factual, so D1 flags it, and
119
+ Pass 6a revises the copy that has no sources while the grounded one
120
+ keeps them.
121
+
122
+ §30 also rejects a duplicate id at Pass 2's boundary, which stops that
123
+ payload arriving. This exists anyway, because the boundary only covers
124
+ claims that came through Pass 2 -- §20's corrections and any future
125
+ resume build claims without passing this way, and a disagreement that
126
+ can only be reached by a path nobody has written yet is still a
127
+ disagreement waiting.
128
+
129
+ FIRST wins, matching what claim_by_id and Pass 6a's in-batch lookup
130
+ already did and documented. The choice is arbitrary once ids are
131
+ unique; having one is not.
132
+ """
133
+ resolved = {}
134
+ for claim in claims:
135
+ resolved.setdefault(claim.id, claim)
136
+ return resolved
137
+
138
+
139
+ @dataclass
140
+ class PipelineRun:
141
+ """
142
+ Holds every pass's structured output for one query. `trace` is a
143
+ running human-readable audit log -- what was flagged, why, what
144
+ changed -- meant to be at least as trustworthy an artifact as the
145
+ final report itself.
146
+ """
147
+
148
+ user_query: str
149
+ # Set by stream_deep_research_pipeline() once a PipelineRunRecord is
150
+ # created in storage; stays None in tests that bypass persistence.
151
+ # ROADMAP §5 (pipeline persistence minimal core): the record lets a
152
+ # run that fails mid-pipeline be inspected and (future) resumed.
153
+ run_id: Optional[UUID] = None
154
+ plan: dict = field(default_factory=dict) # Pass 0
155
+ raw_response: str = "" # Pass 1
156
+ # E13/#77: one entry per SURVIVOR of ensemble Pass 1, in survivor
157
+ # order -- {"candidate", "provider_name", "model", "chars", "text"}.
158
+ # `text` is in-memory working data: pipeline_storage strips it when
159
+ # persisting (the full texts live in the output directory's
160
+ # 01_candidate_N.md files and in each candidate's own pass thread),
161
+ # so the database carries metadata only. Empty for every run that
162
+ # did not use ensemble mode -- raw_response alone is that run's
163
+ # record, exactly as before this field existed.
164
+ candidates: list[dict] = field(default_factory=list)
165
+ claims: list[Claim] = field(default_factory=list) # Pass 2, mutated by 3a/3b/4/5/6a/6b/6c
166
+ completeness: dict = field(default_factory=dict) # Pass 3c: {"gaps": [...], "coverage_score": ...}
167
+ coverage_gaps: list[dict] = field(default_factory=list) # Pass 5-derived: gaps tagged UNVERIFIED_COVERAGE
168
+ assumptions: dict = field(default_factory=dict) # Pass 4
169
+ trace: list[str] = field(default_factory=list)
170
+ # §25: tool calls that ran on a PRE-FLIGHT grant rather than a live
171
+ # approval, one dict per call ({"pass", "tool", "params"}). Empty for
172
+ # every run that granted nothing, which is every run by default.
173
+ #
174
+ # Authorization moved up-front, so accountability moves after: an
175
+ # unattended ten-pass run is precisely where "the user said yes once"
176
+ # needs a record of what that yes actually covered.
177
+ granted_calls: list[dict] = field(default_factory=list)
178
+ # §20: one record per finding the reviewer raised, carrying what it
179
+ # proposed and what the human decided. Populated only on a run that
180
+ # was reviewed, which is off by default (D9).
181
+ #
182
+ # A dict, not a dataclass, and deliberately: a nested dataclass field
183
+ # here would require migrating EVERY vars(c)/vars(run) site to
184
+ # asdict() in one change (see the Claim note above). It is also why
185
+ # the reviewer's tier overrides are marked inside the existing
186
+ # score_breakdown dict rather than in a new Claim field.
187
+ subagent_reviews: list[dict] = field(default_factory=list)
188
+ # §27 (T2): one entry per pass thread, {"pass": pass_id,
189
+ # "thread_id": str}. Each pass runs in its own fresh thread -- a locked
190
+ # invariant, since passes share distilled JSON and never raw history --
191
+ # so §27 hides those threads from the picker. This is what keeps them
192
+ # REACHABLE anyway: hiding a thread must not orphan it.
193
+ #
194
+ # A list[dict] for the same reason granted_calls and subagent_reviews
195
+ # are: a nested dataclass here forces every vars(run)/vars(c) site to
196
+ # asdict() in one change.
197
+ pass_threads: list[dict] = field(default_factory=list)
198
+ # §45 (SQ2/SQ8): one entry per URL this run actually retrieved text
199
+ # from -- {"url", "title", "tool", "retrieved_at", "sha256", "chars",
200
+ # "text"}. Populated from the live SourceCorpus once grounding has
201
+ # finished; the corpus object itself stays inside the pipeline
202
+ # generator.
203
+ #
204
+ # A list[dict] for the same reason granted_calls, subagent_reviews and
205
+ # pass_threads are, and here the reason is sharper: the corpus IS a
206
+ # dataclass, and putting it on this object directly is exactly the
207
+ # nested-dataclass field the note on Claim above forbids.
208
+ #
209
+ # In-memory working data, like candidates[].text: pipeline_storage
210
+ # names the fields it writes and this is not among them, so a run's
211
+ # fetched pages reach output/<run_id>/sources/ and never the database.
212
+ source_documents: list[dict] = field(default_factory=list)
213
+ final_report: str = ""
214
+
215
+ def log(self, message: str) -> None:
216
+ self.trace.append(message)
217
+
218
+ def claim_by_id(self, claim_id: str) -> Optional[Claim]:
219
+ return resolve_by_id(self.claims).get(claim_id)