@yahyahammad/venastine 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/.env.example +14 -0
  2. package/LICENSE +201 -0
  3. package/NOTICE +14 -0
  4. package/PRIVACY.md +66 -0
  5. package/README.md +878 -0
  6. package/SECURITY.md +88 -0
  7. package/THIRD_PARTY_NOTICES.md +100 -0
  8. package/agents/builtin/compactor.md +75 -0
  9. package/agents/builtin/explore.md +77 -0
  10. package/agents/builtin/grill-me.md +27 -0
  11. package/agents/builtin/initializer.md +94 -0
  12. package/agents/builtin/pipeline-reviewer.md +100 -0
  13. package/agents/builtin/plan.md +88 -0
  14. package/agents/builtin/review.md +74 -0
  15. package/agents/manager.py +195 -0
  16. package/agents/subagent_tool.py +215 -0
  17. package/agents/tui_commands.py +136 -0
  18. package/bin/venastine.mjs +382 -0
  19. package/config.py +1198 -0
  20. package/core/__init__.py +0 -0
  21. package/core/approval.py +115 -0
  22. package/core/client.py +1548 -0
  23. package/core/compaction.py +959 -0
  24. package/core/config_loader.py +1563 -0
  25. package/core/events.py +61 -0
  26. package/core/interaction.py +328 -0
  27. package/core/loop.py +1353 -0
  28. package/core/memory.py +370 -0
  29. package/core/model_windows.py +175 -0
  30. package/core/pipeline_models.py +176 -0
  31. package/core/reasoning/__init__.py +0 -0
  32. package/core/reasoning/authorization.py +130 -0
  33. package/core/reasoning/base.py +219 -0
  34. package/core/reasoning/confidence_scoring.py +468 -0
  35. package/core/reasoning/events.py +149 -0
  36. package/core/reasoning/json_retry.py +219 -0
  37. package/core/reasoning/orchestrator.py +1382 -0
  38. package/core/reasoning/output_writer.py +286 -0
  39. package/core/reasoning/payload_validation.py +398 -0
  40. package/core/reasoning/pipeline_storage.py +361 -0
  41. package/core/reasoning/review.py +615 -0
  42. package/core/reasoning/scholar.py +434 -0
  43. package/core/reasoning/source_corpus.py +327 -0
  44. package/core/reasoning/source_scoring.py +855 -0
  45. package/core/replay.py +167 -0
  46. package/core/session.py +153 -0
  47. package/core/workspace_trust.py +207 -0
  48. package/credentials.py +180 -0
  49. package/database.py +252 -0
  50. package/env_secrets.py +77 -0
  51. package/json_store.py +123 -0
  52. package/logging_setup.py +185 -0
  53. package/main.py +1959 -0
  54. package/mcp_client/__init__.py +15 -0
  55. package/mcp_client/client.py +499 -0
  56. package/mcp_client/config.py +379 -0
  57. package/mcp_client/registration.py +182 -0
  58. package/memories/manager.py +132 -0
  59. package/memories/tui_commands.py +232 -0
  60. package/package.json +73 -0
  61. package/project_init/config_files.py +278 -0
  62. package/project_init/doc_sets.py +330 -0
  63. package/project_init/generator.py +564 -0
  64. package/project_init/manifest.py +406 -0
  65. package/project_init/tui_commands.py +144 -0
  66. package/prompts/__init__.py +0 -0
  67. package/prompts/annotate.md +14 -0
  68. package/prompts/assumption_audit.md +27 -0
  69. package/prompts/claim_extraction.md +34 -0
  70. package/prompts/completeness.md +18 -0
  71. package/prompts/confidence_tiers.md +23 -0
  72. package/prompts/critic_pass.md +24 -0
  73. package/prompts/final_synthesis.md +13 -0
  74. package/prompts/initial_generation.md +14 -0
  75. package/prompts/preliminary_plan.md +23 -0
  76. package/prompts/revalidate.md +25 -0
  77. package/prompts/revise.md +15 -0
  78. package/prompts/source_grounding.md +66 -0
  79. package/prompts/system_prompts.py +262 -0
  80. package/prompts/universal_system_prompt +7 -0
  81. package/prompts/untrusted_content +1 -0
  82. package/providers.json.example +92 -0
  83. package/pyproject.toml +100 -0
  84. package/requirements.txt +72 -0
  85. package/safety/__init__.py +0 -0
  86. package/safety/policy_enforcement.py +682 -0
  87. package/security/__init__.py +0 -0
  88. package/security/capability.py +177 -0
  89. package/security/permissions.py +190 -0
  90. package/security/posture.py +302 -0
  91. package/security/protected_paths.py +309 -0
  92. package/security/sandbox.py +985 -0
  93. package/skills/builtin/crypto/cryptography-verification.md +66 -0
  94. package/skills/builtin/math/formal-specification.md +67 -0
  95. package/skills/builtin/math/numerical-methods.md +66 -0
  96. package/skills/builtin/math/proof-writing.md +61 -0
  97. package/skills/builtin/research/experiment-design.md +62 -0
  98. package/skills/builtin/research/literature-review.md +65 -0
  99. package/skills/builtin/research/reproducibility.md +55 -0
  100. package/skills/builtin/research/source-evaluation.md +70 -0
  101. package/skills/builtin/research/statistical-inference.md +63 -0
  102. package/skills/builtin/research/technical-writing.md +59 -0
  103. package/skills/builtin/security/cybersecurity-research.md +63 -0
  104. package/skills/builtin/software/code-review.md +61 -0
  105. package/skills/builtin/software/debugging.md +61 -0
  106. package/skills/builtin/software/test-design.md +67 -0
  107. package/skills/manager.py +138 -0
  108. package/skills/tui_commands.py +111 -0
  109. package/storage.py +1005 -0
  110. package/tools/base.py +305 -0
  111. package/tools/builtin/__init__.py +0 -0
  112. package/tools/builtin/_math_common.py +377 -0
  113. package/tools/builtin/_net_common.py +61 -0
  114. package/tools/builtin/arxiv.py +223 -0
  115. package/tools/builtin/ask_user.py +213 -0
  116. package/tools/builtin/discrete_math.py +134 -0
  117. package/tools/builtin/fetch_url.py +153 -0
  118. package/tools/builtin/file_ops.py +412 -0
  119. package/tools/builtin/geometry.py +148 -0
  120. package/tools/builtin/get_time.py +20 -0
  121. package/tools/builtin/linear_algebra.py +155 -0
  122. package/tools/builtin/load_skill.py +66 -0
  123. package/tools/builtin/logic.py +96 -0
  124. package/tools/builtin/pin.py +193 -0
  125. package/tools/builtin/probability_stats.py +228 -0
  126. package/tools/builtin/project_docs.py +405 -0
  127. package/tools/builtin/remember.py +158 -0
  128. package/tools/builtin/shell.py +461 -0
  129. package/tools/builtin/symbolic_math.py +119 -0
  130. package/tools/builtin/todo.py +147 -0
  131. package/tools/builtin/web_search.py +151 -0
  132. package/tools/context.py +136 -0
  133. package/tools/isolation.py +314 -0
  134. package/tools/registry.py +897 -0
  135. package/tui/app.py +3423 -0
  136. package/tui/app.tcss +458 -0
  137. package/tui/commands.py +61 -0
  138. package/tui/diffs.py +265 -0
  139. package/tui/preferences.py +182 -0
  140. package/tui/ravens.py +146 -0
  141. package/tui/screens.py +943 -0
  142. package/tui/themes.py +366 -0
  143. package/tui/widgets.py +1239 -0
@@ -0,0 +1,176 @@
1
+ """
2
+ core/pipeline_models.py
3
+
4
+ The two pipeline model roles that are not the session model, remembered.
5
+
6
+ ROADMAP_v2 §45 (SQ7). A research run reaches for three models: the
7
+ generator (the session pair, `/model`), the CRITIC that grounds and
8
+ critiques its claims (§11), and — since §45 — the EMBEDDER that scores how
9
+ close a source is to the claim it was cited for. Only the first has ever
10
+ been reachable without editing installed source. `CRITIC_MODEL` has sat in
11
+ `config.py` since §11, which means the routing this project's own
12
+ methodology calls essential ("a model checking its own output for errors
13
+ shares that model's blind spots") was available only to whoever could edit
14
+ the checkout.
15
+
16
+ IN `core/`, NOT `tui/preferences.py`, AND THAT IS A LAYERING FACT. The
17
+ pipeline reads these and D12 makes the TUI a shell that core must never
18
+ import — WS6's argument for `core/model_windows.py`, unchanged, and §25's
19
+ lesson about a capability the pipeline could not reach. It is equally not
20
+ `core/session.py`, whose contract is that nothing there reaches disk.
21
+
22
+ TWO ROLES IN ONE FILE, so both writers read-modify-write. RM4's shape and
23
+ RM4's reason: a writer serialising only its own role would silently drop
24
+ the other's, each correct in its own test. They are one file rather than
25
+ two modules because they are the same KIND of fact — which model plays
26
+ which part in a run — and `model_windows.py` is next door as the example
27
+ of when a second file is right instead (it holds a different kind of fact,
28
+ keyed on a pair rather than naming one).
29
+
30
+ PRECEDENCE: THE STORE OUTRANKS `config.py`. The opposite of
31
+ `model_windows`, and for the opposite reason — there, a configured value
32
+ could be confidently wrong about a deployment, so the person won. Here
33
+ `config.py` is the harness's own default and the store is a choice
34
+ someone made at a command prompt for this account, which is `/model`
35
+ outranking `config.MODEL_NAME` exactly.
36
+
37
+ WHY NOT `settings.json`. Choosing a provider is a grant (E2): a project's
38
+ settings.json outranks the user's, so a cloned repo could otherwise point
39
+ this harness's embedding calls — claim text and fetched page text — at a
40
+ provider of its choosing. `embedder_model` and `critic_model` are
41
+ therefore rejected there BY NAME, like `ensemble_models`, and this
42
+ user-tier store is the route instead.
43
+ """
44
+
45
+ import logging
46
+ import os
47
+ from typing import Optional
48
+
49
+ import config
50
+ from json_store import read_versioned, write_json_atomic
51
+
52
+ logger = logging.getLogger(__name__)
53
+
54
+ # Bumped when the SHAPE changes. An unknown version reads as an empty
55
+ # store, which costs the remembered roles and falls back to config.py --
56
+ # there is nothing here that was consented to, so the MCP store's careful
57
+ # legacy handling has no counterpart.
58
+ STORE_VERSION = 1
59
+
60
+ # The roles this store knows. A role outside it is refused rather than
61
+ # stored: a typo'd `/embeder` writing a key nothing reads would look like
62
+ # it worked and change nothing, which is the failure mode this whole
63
+ # module exists to remove for CRITIC_MODEL.
64
+ ROLES = ("critic", "embedder")
65
+
66
+ # Which config.py constant answers when the store is silent.
67
+ _FALLBACK_ATTR = {"critic": "CRITIC_MODEL", "embedder": "EMBEDDER_MODEL"}
68
+
69
+
70
+ def store_path() -> str:
71
+ # Call time, not import time: tests redirect it, and a moved config
72
+ # directory is picked up without a restart. Every sibling store
73
+ # resolves theirs the same way.
74
+ return os.path.expanduser("~/.config/venastine/pipeline_models.json")
75
+
76
+
77
+ def _read_raw() -> dict:
78
+ data = read_versioned(store_path(), STORE_VERSION,
79
+ "falling back to config.py for pipeline models.")
80
+ roles = data.get("roles")
81
+ return roles if isinstance(roles, dict) else {}
82
+
83
+
84
+ def _write(roles: dict, action: str) -> bool:
85
+ """Persist the whole map. True if it reached disk.
86
+
87
+ A failure WARNS and returns False rather than raising, matching
88
+ model_windows: the TUI routes WARNING+ into the transcript, so someone
89
+ whose choice could not be saved is told at the moment they make it
90
+ rather than discovering it at the next launch.
91
+ """
92
+ path = store_path()
93
+ try:
94
+ write_json_atomic(path, {"version": STORE_VERSION, "roles": roles},
95
+ sort_keys=True)
96
+ except OSError as exc:
97
+ logger.warning("Could not %s the pipeline model in %s (%s).",
98
+ action, path, exc)
99
+ return False
100
+ return True
101
+
102
+
103
+ def _valid(entry) -> bool:
104
+ """A whole pair or nothing.
105
+
106
+ tui/preferences.py's rule, for its reason: a model name means nothing
107
+ against the wrong provider, so half a record is no record. A
108
+ hand-edited file with only one of the two is ignored rather than
109
+ half-applied.
110
+ """
111
+ return (isinstance(entry, dict)
112
+ and isinstance(entry.get("provider_name"), str)
113
+ and isinstance(entry.get("model"), str)
114
+ and bool(entry["provider_name"]) and bool(entry["model"]))
115
+
116
+
117
+ def remembered(role: str) -> Optional[dict]:
118
+ """What the user chose for `role`, or None. Ignores config.py."""
119
+ entry = _read_raw().get(role)
120
+ if not _valid(entry):
121
+ return None
122
+ return {"provider_name": entry["provider_name"], "model": entry["model"]}
123
+
124
+
125
+ def resolve(role: str) -> Optional[dict]:
126
+ """The pair actually in force for `role`: the store, else config.py,
127
+ else None.
128
+
129
+ THE ONE READER FOR THE PIPELINE. `orchestrator` and every shell ask
130
+ this rather than reading `config.CRITIC_MODEL` directly, so a
131
+ remembered choice reaches the CLI, the TUI and a spawned run alike --
132
+ and so there is one precedence rule rather than one per caller.
133
+ """
134
+ if role not in ROLES:
135
+ raise ValueError(f"unknown pipeline model role {role!r}; "
136
+ f"expected one of {', '.join(ROLES)}")
137
+ chosen = remembered(role)
138
+ if chosen is not None:
139
+ return chosen
140
+ fallback = getattr(config, _FALLBACK_ATTR[role], None)
141
+ return fallback if _valid(fallback) else None
142
+
143
+
144
+ def remember(role: str, provider_name: str, model: str) -> bool:
145
+ """Record a pair for `role`. True if it reached disk."""
146
+ if role not in ROLES:
147
+ raise ValueError(f"unknown pipeline model role {role!r}; "
148
+ f"expected one of {', '.join(ROLES)}")
149
+ roles = _read_raw()
150
+ roles[role] = {"provider_name": provider_name, "model": model}
151
+ return _write(roles, "save")
152
+
153
+
154
+ def forget(role: str) -> str:
155
+ """Drop `role`'s entry. "removed", "absent" or "failed".
156
+
157
+ Three outcomes rather than a bool, following model_windows: the shell
158
+ says a different sentence for each, and a failed write reported as
159
+ "nothing was set" would be a lie about a file the user can go and
160
+ read. Note that "removed" does not mean the role is unset -- config.py
161
+ may still answer, and the shell says so.
162
+ """
163
+ if role not in ROLES:
164
+ raise ValueError(f"unknown pipeline model role {role!r}; "
165
+ f"expected one of {', '.join(ROLES)}")
166
+ roles = _read_raw()
167
+ if roles.pop(role, None) is None:
168
+ return "absent"
169
+ return "removed" if _write(roles, "clear") else "failed"
170
+
171
+
172
+ def all_roles() -> dict:
173
+ """{role: {"provider_name", "model"}} for every role in force, from
174
+ whichever tier answered. For a report."""
175
+ return {role: pair for role in ROLES
176
+ if (pair := resolve(role)) is not None}
File without changes
@@ -0,0 +1,130 @@
1
+ """
2
+ core/reasoning/authorization.py
3
+
4
+ ROADMAP_v2 §25. The pipeline's POLICY about pre-flight grants -- which
5
+ tools may be offered, and how a user's answer is parsed.
6
+
7
+ core/approval.py holds the data shapes and core/loop.py enforces them.
8
+ This file answers the two questions that are specific to the research
9
+ pipeline, and both shells (main.py's flag, tui/app.py's picker) consume it
10
+ so the CLI and the TUI cannot drift into offering different sets.
11
+
12
+ Why the pipeline needs its own answer at all: it is the harness's highest
13
+ prompt-injection surface by construction. It fetches web pages and feeds
14
+ them to a model that then chooses tools, unattended, across ten passes.
15
+ §23 reasoned the same way when it decided the question tool denies
16
+ headless -- "an injected question from fetched web content would be least
17
+ noticed" there.
18
+ """
19
+
20
+ import logging
21
+
22
+ logger = logging.getLogger(__name__)
23
+
24
+
25
+ # What both shells say when nothing in this run can be granted (#106).
26
+ #
27
+ # One string, two callers, for the same reason the module exists: they
28
+ # drifted, and the TUI's version was FALSE where it fired. It said "no
29
+ # approval-gated tool is AVAILABLE to this run" when two were -- they
30
+ # simply could not be pre-granted, which is a different sentence and the
31
+ # only one either shell is in a position to make.
32
+ #
33
+ # This module already kept the two from offering different SETS. What it
34
+ # had never been given is what they say about an empty one.
35
+ NOTHING_TO_GRANT = (
36
+ "No approval-gated tool in this run can be granted up front, so there "
37
+ "is nothing to authorise. Gated tools are still asked about per call "
38
+ "wherever something can answer."
39
+ )
40
+
41
+
42
+ # "The flag was given with no value -- ask me which." A distinct object
43
+ # rather than "" so an explicitly empty list (--grant-tools "") stays an
44
+ # empty GRANT rather than becoming a prompt. Defined here, not in either
45
+ # shell, because both compare against it and two sentinels would compare
46
+ # unequal while looking identical.
47
+ GRANT_PICKER = object()
48
+
49
+
50
+ def candidates(context=None) -> list[tuple[str, str]]:
51
+ """(name, description) for every tool a pipeline grant could cover.
52
+
53
+ Three filters, and the composition is the point:
54
+ * advertised and approval-gated -- i.e. exactly what the headless
55
+ rule hides today. Offering anything else would be offering a grant
56
+ for a tool that never needed one.
57
+ * registry.grantable -- a param-dependent gate was never consented
58
+ to by name (R2).
59
+ * grant_policy is GRANT_ANYWHERE (R13). This is the STRICT half of
60
+ the pair: a GRANT_SIGNOFF_ONLY tool may be pre-granted by a human
61
+ answering about one named agent for one turn, and may not be
62
+ pre-granted to ten unattended passes on one launch-time tick.
63
+
64
+ That third filter used to be a frozenset in this module, which is
65
+ exactly how #67 and #133 happened: agents/manager.py could not see it,
66
+ so the §18 sign-off and this function held different ideas of what a
67
+ grant covers, and a tool registered after the set was written
68
+ (write_project_doc, §24) joined the picker without anyone being asked.
69
+ The answer now lives on the tool and both callers read it.
70
+
71
+ On a default install this returns an EMPTY list, and that is correct
72
+ rather than a regression: every built-in is either ungated, param-
73
+ dependent, or excluded by policy. The population this exists for is
74
+ MCP tools -- allowed but approval-gated by default (§17 decision D),
75
+ no approval_check, and named at connection time.
76
+
77
+ Descriptions come from the tool's own schema because they are the only
78
+ signal available: MCP exposes no read-only/write metadata, so nothing
79
+ can tell the user that one granted tool searches a library and another
80
+ posts to a channel. Showing what a tool says it does is the whole of
81
+ informed consent here, which is also why the grant is per-tool (R1)
82
+ rather than one yes covering the set.
83
+ """
84
+ from tools.base import GRANT_ANYWHERE
85
+ from tools.registry import registry
86
+
87
+ out = []
88
+ for name in sorted(registry.headless_hidden(context)):
89
+ if (not registry.grantable(name)
90
+ or registry.grant_policy(name) != GRANT_ANYWHERE):
91
+ continue
92
+ spec = registry._tools.get(name)
93
+ description = ""
94
+ if spec is not None:
95
+ description = (spec.schema or {}).get("description", "") or ""
96
+ out.append((name, description.strip().splitlines()[0]
97
+ if description.strip() else ""))
98
+ return out
99
+
100
+
101
+ class GrantSpecError(ValueError):
102
+ """A --grant-tools value naming something that cannot be granted."""
103
+
104
+
105
+ def parse_grant_spec(text: str, offered) -> set:
106
+ """Turn a comma-separated --grant-tools value into a set of names.
107
+
108
+ RAISES on an unknown name rather than dropping it. A typo'd tool would
109
+ otherwise produce an empty or partial grant, the run would proceed
110
+ looking authorised, and the pass would report that a tool it was
111
+ supposed to have was unavailable -- with the actual cause three layers
112
+ away. The same argument D24 settled for undeclared permissions: an
113
+ authorization mistake must be loud at the point it is made.
114
+
115
+ `offered` is candidates()' output (or any iterable of names), so what a
116
+ flag may name and what a picker may show are the same set by
117
+ construction.
118
+ """
119
+ allowed = {n if isinstance(n, str) else n[0] for n in offered}
120
+ names = {part.strip() for part in (text or "").split(",") if part.strip()}
121
+ unknown = sorted(names - allowed)
122
+ if unknown:
123
+ raise GrantSpecError(
124
+ f"cannot grant {', '.join(unknown)}: "
125
+ + (f"grantable tools are {', '.join(sorted(allowed))}"
126
+ if allowed else
127
+ "no tool in this run needs a grant, so there is nothing to "
128
+ "authorise")
129
+ )
130
+ return names
@@ -0,0 +1,219 @@
1
+ """
2
+ core/reasoning/base.py
3
+
4
+ Structured claim-level data model for the deep-research pipeline. This
5
+ REPLACES the earlier free-text PipelineContext/PassResult design -- Pass 5
6
+ (confidence tiering), gate D0 (claim-type routing), and D1/D2 (threshold and
7
+ retry-cap checks) all need to reason about individual claims, not a
8
+ concatenated paragraph of prior-pass prose. Passes that produce
9
+ structured output (2, 3a, 3b, 4) mutate Claim objects in place; passes
10
+ that produce free text (0, 1, 3c, final synthesis) populate the
11
+ corresponding PipelineRun fields directly.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ from dataclasses import dataclass, field
17
+ from typing import Literal, Optional, get_args
18
+ from uuid import UUID
19
+
20
+ ClaimType = Literal["factual", "synthesis", "speculative"]
21
+ ConfidenceTier = Literal["HIGH", "MEDIUM", "LOW", "UNVERIFIED", "UNVERIFIED_COVERAGE"]
22
+ GroundingStatus = Literal["grounded", "partial", "ungrounded"]
23
+
24
+ # ROADMAP_v2 §30. The Literals above are ANNOTATIONS -- dataclasses do
25
+ # not enforce them, and #75 is what that costs: `"Grounded"` scored a
26
+ # well-sourced claim 0.35/LOW where `"grounded"` gives 0.85/HIGH, and
27
+ # `"Ungrounded"` escaped the forced-UNVERIFIED rule to land LOW. Two
28
+ # modules need the vocabulary as DATA -- the payload boundary rejects a
29
+ # value outside it, the scorer coerces one that got through -- and a
30
+ # second hand-typed copy of a vocabulary is a second thing to keep in
31
+ # step. Derived from the Literal, beside the Literal.
32
+ CLAIM_TYPES = frozenset(get_args(ClaimType))
33
+ GROUNDING_STATUSES = frozenset(get_args(GroundingStatus))
34
+
35
+
36
+ @dataclass
37
+ class Claim:
38
+ """One atomic, self-contained assertion extracted from Pass 1's raw
39
+ response by Pass 2. Every later pass mutates fields on this same
40
+ object rather than creating parallel data structures keyed by id.
41
+
42
+ SERIALIZATION NOTE: EVERY `vars(c)` IN THE TREE serializes this object
43
+ shallowly. That is correct only while every field below stays
44
+ JSON-native (str / list / dict / float / int / None). If a
45
+ nested-dataclass field is EVER added here, ALL of those sites must
46
+ migrate to `dataclasses.asdict(c)` TOGETHER, not piecemeal -- hardening
47
+ just one leaves the rest exposed and creates two different
48
+ serialization mechanisms for the same object in the same codebase.
49
+
50
+ grep -rn 'vars(c' --include='*.py' .
51
+
52
+ Deliberately a GREP rather than a list. This note used to enumerate
53
+ three sites -- orchestrator.py's passes, pipeline_storage and
54
+ output_writer -- and §26 added two more in tui/app.py (show_claims and
55
+ /copy claims) that fell out of it, and out of AGENTS.md's copy of the
56
+ same list (audit #108). The TUI's failure would have been quieter than
57
+ the persistence ones, since ClaimsScreen renders a repr into a cell and
58
+ `json.dumps(..., default=str)` stringifies rather than raising -- which
59
+ is precisely the "two different serialization mechanisms" outcome this
60
+ note exists to prevent, arriving through the door the note did not
61
+ cover. An enumeration that says ALL has to be complete to mean
62
+ anything, and a new consumer in a new section will keep falling out of
63
+ one.
64
+ """
65
+
66
+ id: str
67
+ text: str
68
+ type: ClaimType
69
+ entities: list[str] = field(default_factory=list)
70
+ source_span: str = ""
71
+
72
+ # Populated by Pass 3a (factual claims only -- synthesis/speculative
73
+ # claims skip grounding entirely per the pipeline's gate D0 routing, so
74
+ # these stay at their defaults for those claims).
75
+ grounding_sources: list[dict] = field(default_factory=list)
76
+ grounding_status: Optional[GroundingStatus] = None
77
+
78
+ # Populated by Pass 3b (factual claims only, same reason as above).
79
+ fallacies: list[str] = field(default_factory=list)
80
+ contradictions: list[str] = field(default_factory=list)
81
+ critic_severity: float = 0.0
82
+
83
+ # Populated by Pass 4 (ALL claims, factual or not).
84
+ assumption_flags: list[str] = field(default_factory=list)
85
+
86
+ # Populated by Pass 2 in ensemble mode (ROADMAP §10). Lists the
87
+ # 1-indexed candidate numbers that independently asserted this claim.
88
+ # Empty/unused when ensemble mode is off.
89
+ asserted_by_candidates: list[int] = field(default_factory=list)
90
+
91
+ # Populated by Pass 5 (pure code, no LLM call) -- see confidence_scoring.py.
92
+ confidence_tier: Optional[ConfidenceTier] = None
93
+ score_breakdown: dict = field(default_factory=dict)
94
+
95
+ # Populated by the D1/6a/6b/D2/fallback retry loop.
96
+ revision_text: Optional[str] = None
97
+ retry_count: int = 0
98
+
99
+ # Populated by 6c (annotate) / fallback -- the actual final text and
100
+ # confidence tag a claim carries into the merged output.
101
+ final_text: Optional[str] = None
102
+ annotation: Optional[str] = None
103
+
104
+
105
+ def resolve_by_id(claims: list) -> dict:
106
+ """The pipeline's ONE claim-id resolution (ROADMAP_v2 §30, B6).
107
+
108
+ There used to be two, and they disagreed. `_apply_grounding` and
109
+ `_apply_critic` built `{c.id: c}` -- LAST wins -- while
110
+ `_apply_assumption_flags` and `claim_by_id` used
111
+ `next(c for c in claims if c.id == claim_id)` -- FIRST wins. With two
112
+ claims sharing an id, driven against the real pipeline, that split one
113
+ claim in half:
114
+
115
+ text='first' grounding=None severity=0.0 flags=['a flag']
116
+ text='second' grounding='grounded' severity=0.4 flags=[]
117
+
118
+ The first copy is then ungrounded-but-factual, so D1 flags it, and
119
+ Pass 6a revises the copy that has no sources while the grounded one
120
+ keeps them.
121
+
122
+ §30 also rejects a duplicate id at Pass 2's boundary, which stops that
123
+ payload arriving. This exists anyway, because the boundary only covers
124
+ claims that came through Pass 2 -- §20's corrections and any future
125
+ resume build claims without passing this way, and a disagreement that
126
+ can only be reached by a path nobody has written yet is still a
127
+ disagreement waiting.
128
+
129
+ FIRST wins, matching what claim_by_id and Pass 6a's in-batch lookup
130
+ already did and documented. The choice is arbitrary once ids are
131
+ unique; having one is not.
132
+ """
133
+ resolved = {}
134
+ for claim in claims:
135
+ resolved.setdefault(claim.id, claim)
136
+ return resolved
137
+
138
+
139
+ @dataclass
140
+ class PipelineRun:
141
+ """
142
+ Holds every pass's structured output for one query. `trace` is a
143
+ running human-readable audit log -- what was flagged, why, what
144
+ changed -- meant to be at least as trustworthy an artifact as the
145
+ final report itself.
146
+ """
147
+
148
+ user_query: str
149
+ # Set by stream_deep_research_pipeline() once a PipelineRunRecord is
150
+ # created in storage; stays None in tests that bypass persistence.
151
+ # ROADMAP §5 (pipeline persistence minimal core): the record lets a
152
+ # run that fails mid-pipeline be inspected and (future) resumed.
153
+ run_id: Optional[UUID] = None
154
+ plan: dict = field(default_factory=dict) # Pass 0
155
+ raw_response: str = "" # Pass 1
156
+ # E13/#77: one entry per SURVIVOR of ensemble Pass 1, in survivor
157
+ # order -- {"candidate", "provider_name", "model", "chars", "text"}.
158
+ # `text` is in-memory working data: pipeline_storage strips it when
159
+ # persisting (the full texts live in the output directory's
160
+ # 01_candidate_N.md files and in each candidate's own pass thread),
161
+ # so the database carries metadata only. Empty for every run that
162
+ # did not use ensemble mode -- raw_response alone is that run's
163
+ # record, exactly as before this field existed.
164
+ candidates: list[dict] = field(default_factory=list)
165
+ claims: list[Claim] = field(default_factory=list) # Pass 2, mutated by 3a/3b/4/5/6a/6b/6c
166
+ completeness: dict = field(default_factory=dict) # Pass 3c: {"gaps": [...], "coverage_score": ...}
167
+ coverage_gaps: list[dict] = field(default_factory=list) # Pass 5-derived: gaps tagged UNVERIFIED_COVERAGE
168
+ assumptions: dict = field(default_factory=dict) # Pass 4
169
+ trace: list[str] = field(default_factory=list)
170
+ # §25: tool calls that ran on a PRE-FLIGHT grant rather than a live
171
+ # approval, one dict per call ({"pass", "tool", "params"}). Empty for
172
+ # every run that granted nothing, which is every run by default.
173
+ #
174
+ # Authorization moved up-front, so accountability moves after: an
175
+ # unattended ten-pass run is precisely where "the user said yes once"
176
+ # needs a record of what that yes actually covered.
177
+ granted_calls: list[dict] = field(default_factory=list)
178
+ # §20: one record per finding the reviewer raised, carrying what it
179
+ # proposed and what the human decided. Populated only on a run that
180
+ # was reviewed, which is off by default (D9).
181
+ #
182
+ # A dict, not a dataclass, and deliberately: a nested dataclass field
183
+ # here would require migrating EVERY vars(c)/vars(run) site to
184
+ # asdict() in one change (see the Claim note above). It is also why
185
+ # the reviewer's tier overrides are marked inside the existing
186
+ # score_breakdown dict rather than in a new Claim field.
187
+ subagent_reviews: list[dict] = field(default_factory=list)
188
+ # §27 (T2): one entry per pass thread, {"pass": pass_id,
189
+ # "thread_id": str}. Each pass runs in its own fresh thread -- a locked
190
+ # invariant, since passes share distilled JSON and never raw history --
191
+ # so §27 hides those threads from the picker. This is what keeps them
192
+ # REACHABLE anyway: hiding a thread must not orphan it.
193
+ #
194
+ # A list[dict] for the same reason granted_calls and subagent_reviews
195
+ # are: a nested dataclass here forces every vars(run)/vars(c) site to
196
+ # asdict() in one change.
197
+ pass_threads: list[dict] = field(default_factory=list)
198
+ # §45 (SQ2/SQ8): one entry per URL this run actually retrieved text
199
+ # from -- {"url", "title", "tool", "retrieved_at", "sha256", "chars",
200
+ # "text"}. Populated from the live SourceCorpus once grounding has
201
+ # finished; the corpus object itself stays inside the pipeline
202
+ # generator.
203
+ #
204
+ # A list[dict] for the same reason granted_calls, subagent_reviews and
205
+ # pass_threads are, and here the reason is sharper: the corpus IS a
206
+ # dataclass, and putting it on this object directly is exactly the
207
+ # nested-dataclass field the note on Claim above forbids.
208
+ #
209
+ # In-memory working data, like candidates[].text: pipeline_storage
210
+ # names the fields it writes and this is not among them, so a run's
211
+ # fetched pages reach output/<run_id>/sources/ and never the database.
212
+ source_documents: list[dict] = field(default_factory=list)
213
+ final_report: str = ""
214
+
215
+ def log(self, message: str) -> None:
216
+ self.trace.append(message)
217
+
218
+ def claim_by_id(self, claim_id: str) -> Optional[Claim]:
219
+ return resolve_by_id(self.claims).get(claim_id)