@yahyahammad/venastine 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/.env.example +14 -0
  2. package/LICENSE +201 -0
  3. package/NOTICE +14 -0
  4. package/PRIVACY.md +66 -0
  5. package/README.md +878 -0
  6. package/SECURITY.md +88 -0
  7. package/THIRD_PARTY_NOTICES.md +100 -0
  8. package/agents/builtin/compactor.md +75 -0
  9. package/agents/builtin/explore.md +77 -0
  10. package/agents/builtin/grill-me.md +27 -0
  11. package/agents/builtin/initializer.md +94 -0
  12. package/agents/builtin/pipeline-reviewer.md +100 -0
  13. package/agents/builtin/plan.md +88 -0
  14. package/agents/builtin/review.md +74 -0
  15. package/agents/manager.py +195 -0
  16. package/agents/subagent_tool.py +215 -0
  17. package/agents/tui_commands.py +136 -0
  18. package/bin/venastine.mjs +382 -0
  19. package/config.py +1198 -0
  20. package/core/__init__.py +0 -0
  21. package/core/approval.py +115 -0
  22. package/core/client.py +1548 -0
  23. package/core/compaction.py +959 -0
  24. package/core/config_loader.py +1563 -0
  25. package/core/events.py +61 -0
  26. package/core/interaction.py +328 -0
  27. package/core/loop.py +1353 -0
  28. package/core/memory.py +370 -0
  29. package/core/model_windows.py +175 -0
  30. package/core/pipeline_models.py +176 -0
  31. package/core/reasoning/__init__.py +0 -0
  32. package/core/reasoning/authorization.py +130 -0
  33. package/core/reasoning/base.py +219 -0
  34. package/core/reasoning/confidence_scoring.py +468 -0
  35. package/core/reasoning/events.py +149 -0
  36. package/core/reasoning/json_retry.py +219 -0
  37. package/core/reasoning/orchestrator.py +1382 -0
  38. package/core/reasoning/output_writer.py +286 -0
  39. package/core/reasoning/payload_validation.py +398 -0
  40. package/core/reasoning/pipeline_storage.py +361 -0
  41. package/core/reasoning/review.py +615 -0
  42. package/core/reasoning/scholar.py +434 -0
  43. package/core/reasoning/source_corpus.py +327 -0
  44. package/core/reasoning/source_scoring.py +855 -0
  45. package/core/replay.py +167 -0
  46. package/core/session.py +153 -0
  47. package/core/workspace_trust.py +207 -0
  48. package/credentials.py +180 -0
  49. package/database.py +252 -0
  50. package/env_secrets.py +77 -0
  51. package/json_store.py +123 -0
  52. package/logging_setup.py +185 -0
  53. package/main.py +1959 -0
  54. package/mcp_client/__init__.py +15 -0
  55. package/mcp_client/client.py +499 -0
  56. package/mcp_client/config.py +379 -0
  57. package/mcp_client/registration.py +182 -0
  58. package/memories/manager.py +132 -0
  59. package/memories/tui_commands.py +232 -0
  60. package/package.json +73 -0
  61. package/project_init/config_files.py +278 -0
  62. package/project_init/doc_sets.py +330 -0
  63. package/project_init/generator.py +564 -0
  64. package/project_init/manifest.py +406 -0
  65. package/project_init/tui_commands.py +144 -0
  66. package/prompts/__init__.py +0 -0
  67. package/prompts/annotate.md +14 -0
  68. package/prompts/assumption_audit.md +27 -0
  69. package/prompts/claim_extraction.md +34 -0
  70. package/prompts/completeness.md +18 -0
  71. package/prompts/confidence_tiers.md +23 -0
  72. package/prompts/critic_pass.md +24 -0
  73. package/prompts/final_synthesis.md +13 -0
  74. package/prompts/initial_generation.md +14 -0
  75. package/prompts/preliminary_plan.md +23 -0
  76. package/prompts/revalidate.md +25 -0
  77. package/prompts/revise.md +15 -0
  78. package/prompts/source_grounding.md +66 -0
  79. package/prompts/system_prompts.py +262 -0
  80. package/prompts/universal_system_prompt +7 -0
  81. package/prompts/untrusted_content +1 -0
  82. package/providers.json.example +92 -0
  83. package/pyproject.toml +100 -0
  84. package/requirements.txt +72 -0
  85. package/safety/__init__.py +0 -0
  86. package/safety/policy_enforcement.py +682 -0
  87. package/security/__init__.py +0 -0
  88. package/security/capability.py +177 -0
  89. package/security/permissions.py +190 -0
  90. package/security/posture.py +302 -0
  91. package/security/protected_paths.py +309 -0
  92. package/security/sandbox.py +985 -0
  93. package/skills/builtin/crypto/cryptography-verification.md +66 -0
  94. package/skills/builtin/math/formal-specification.md +67 -0
  95. package/skills/builtin/math/numerical-methods.md +66 -0
  96. package/skills/builtin/math/proof-writing.md +61 -0
  97. package/skills/builtin/research/experiment-design.md +62 -0
  98. package/skills/builtin/research/literature-review.md +65 -0
  99. package/skills/builtin/research/reproducibility.md +55 -0
  100. package/skills/builtin/research/source-evaluation.md +70 -0
  101. package/skills/builtin/research/statistical-inference.md +63 -0
  102. package/skills/builtin/research/technical-writing.md +59 -0
  103. package/skills/builtin/security/cybersecurity-research.md +63 -0
  104. package/skills/builtin/software/code-review.md +61 -0
  105. package/skills/builtin/software/debugging.md +61 -0
  106. package/skills/builtin/software/test-design.md +67 -0
  107. package/skills/manager.py +138 -0
  108. package/skills/tui_commands.py +111 -0
  109. package/storage.py +1005 -0
  110. package/tools/base.py +305 -0
  111. package/tools/builtin/__init__.py +0 -0
  112. package/tools/builtin/_math_common.py +377 -0
  113. package/tools/builtin/_net_common.py +61 -0
  114. package/tools/builtin/arxiv.py +223 -0
  115. package/tools/builtin/ask_user.py +213 -0
  116. package/tools/builtin/discrete_math.py +134 -0
  117. package/tools/builtin/fetch_url.py +153 -0
  118. package/tools/builtin/file_ops.py +412 -0
  119. package/tools/builtin/geometry.py +148 -0
  120. package/tools/builtin/get_time.py +20 -0
  121. package/tools/builtin/linear_algebra.py +155 -0
  122. package/tools/builtin/load_skill.py +66 -0
  123. package/tools/builtin/logic.py +96 -0
  124. package/tools/builtin/pin.py +193 -0
  125. package/tools/builtin/probability_stats.py +228 -0
  126. package/tools/builtin/project_docs.py +405 -0
  127. package/tools/builtin/remember.py +158 -0
  128. package/tools/builtin/shell.py +461 -0
  129. package/tools/builtin/symbolic_math.py +119 -0
  130. package/tools/builtin/todo.py +147 -0
  131. package/tools/builtin/web_search.py +151 -0
  132. package/tools/context.py +136 -0
  133. package/tools/isolation.py +314 -0
  134. package/tools/registry.py +897 -0
  135. package/tui/app.py +3423 -0
  136. package/tui/app.tcss +458 -0
  137. package/tui/commands.py +61 -0
  138. package/tui/diffs.py +265 -0
  139. package/tui/preferences.py +182 -0
  140. package/tui/ravens.py +146 -0
  141. package/tui/screens.py +943 -0
  142. package/tui/themes.py +366 -0
  143. package/tui/widgets.py +1239 -0
@@ -0,0 +1,468 @@
1
+ """
2
+ core/reasoning/confidence_scoring.py
3
+
4
+ Pass 5 -- confidence tiering. ZERO LLM calls, by design: this is a
5
+ deterministic, documented weighted formula over already-computed data
6
+ (grounding status, critic severity, assumption flags). Kept as its own
7
+ module, separate from orchestrator.py, so the weights can be tuned later
8
+ without touching any prompt or any sequencing logic.
9
+
10
+ Interpretation worth flagging: Pass 3c's "gaps" (things the response
11
+ never addressed at all) are NOT claims that exist and failed
12
+ verification -- they're an absence. Rather than forcing them into the
13
+ Claim model, they're represented separately as coverage_gaps, each
14
+ tagged UNVERIFIED_COVERAGE. This is my reading of what that tier is for;
15
+ flag it if you intended something different.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ from core.reasoning.base import (
21
+ CLAIM_TYPES, GROUNDING_STATUSES, Claim, ConfidenceTier, PipelineRun,
22
+ )
23
+
24
+ # --- Tunable weights -- change these, not the logic below, to retune scoring ---
25
+ GROUNDING_WEIGHTS = {"grounded": 1.0, "partial": 0.5, "ungrounded": 0.0, None: 0.0}
26
+ GROUNDING_WEIGHT_FACTOR = 0.5
27
+ CRITIC_WEIGHT_FACTOR = 0.35
28
+ ASSUMPTION_FLAG_PENALTY = 0.15 # per flag
29
+ NON_FACTUAL_SCORE_CAP = 0.65 # claims never grounded can't reach HIGH
30
+
31
+ # ROADMAP §10 (revisit, E8). Cross-candidate disagreement is a PENALTY,
32
+ # subtracted like ASSUMPTION_FLAG_PENALTY above -- not a fourth weighted
33
+ # term with the others scaled down to make room for it.
34
+ #
35
+ # §10 shipped the other shape: 0.5/0.35 redistributed to 0.4/0.3 with a
36
+ # 0.15 consistency term added. Two consequences, neither intended:
37
+ #
38
+ # 1. Turning ensemble mode on made GROUNDING count for less. Grounding is
39
+ # the pipeline's strongest evidence, and which generation strategy
40
+ # produced a claim is no reason to weigh its verification differently.
41
+ # 2. Because the maximum stayed 0.85, consistency could only ever cost a
42
+ # well-grounded claim. A claim Pass 3a grounded and Pass 3b found
43
+ # nothing wrong with dropped from HIGH to MEDIUM at 2-of-3 agreement.
44
+ #
45
+ # As a penalty, unanimity changes nothing and dissent subtracts, which is
46
+ # also the honest reading of the signal: agreement is the null hypothesis
47
+ # (candidates asked the same question usually answer alike), so disagreement
48
+ # is the part that carries information. It makes the non-ensemble formula
49
+ # literally unchanged rather than coincidentally equal, so §10's
50
+ # byte-for-byte regression guard holds by construction.
51
+ DISAGREEMENT_PENALTY_FACTOR = 0.15
52
+
53
+ # ROADMAP_v2 §45 (SQ3). What a grounding is WORTH.
54
+ #
55
+ # MULTIPLICATIVE, NOT A FOURTH TERM, and that is the whole design. §10
56
+ # shipped a fourth weighted term for consistency and reversed it (see
57
+ # E8 above): redistributing 0.5/0.35 to make room meant grounding counted
58
+ # for less whenever the feature was on, and the new term could only ever
59
+ # subtract. Scaling the grounding component instead says what is actually
60
+ # meant -- how much is this grounding worth -- and leaves the non-scored
61
+ # formula LITERALLY unchanged rather than coincidentally equal, so the
62
+ # byte-for-byte regression guard holds by construction here too.
63
+ #
64
+ # BOTH CONSTANTS BELOW WERE SET BY MEASUREMENT, not by taste, and the
65
+ # first draft of each was wrong in a way only a sweep showed.
66
+ #
67
+ # AUTHORITY_FULL_CREDIT exists because the domain table is a RANKING and
68
+ # this formula needs an EVIDENTIAL scale. Feeding `authority` in raw
69
+ # multiplied two rank-scales together and landed everything in the middle:
70
+ # swept across the whole table, only a `.gov` at similarity 1.0 still
71
+ # reached HIGH, and Nature, arXiv, Reddit and a Pinterest pin were
72
+ # indistinguishable at MEDIUM -- a tier that no longer discriminates,
73
+ # which is the opposite of the point. At or above a peer-reviewed
74
+ # publisher, authority is not what limits a claim's confidence, so that is
75
+ # where full credit starts.
76
+ #
77
+ # SOURCE_QUALITY_FLOOR was 0.4 in the first draft, and 0.4 is exactly the
78
+ # worst available value: 0.5*0.4 + 0.35 == 0.55, which IS the MEDIUM
79
+ # threshold, so the floor guaranteed MEDIUM for every grounded claim
80
+ # however bad its sources were. At 0.25 a claim grounded only in a content
81
+ # farm lands LOW -- flagged, and sent to the 6a/6b loop to find something
82
+ # better, which is the measured Pinterest case getting the treatment it
83
+ # always should have.
84
+ #
85
+ # NOT ZERO, though: a weak-but-real grounding is still better evidenced
86
+ # than one nothing supports, and the forced-UNVERIFIED rule below already
87
+ # handles the latter.
88
+ AUTHORITY_FULL_CREDIT = 0.85
89
+ SOURCE_QUALITY_FLOOR = 0.25
90
+
91
+
92
+ TIER_THRESHOLDS = [
93
+ (0.8, "HIGH"),
94
+ (0.55, "MEDIUM"),
95
+ (0.3, "LOW"),
96
+ ] # anything below the lowest threshold, or an ungrounded factual claim, is UNVERIFIED
97
+
98
+
99
+ # ROADMAP_v2 §45 (SQ6). The constants above are the DEFAULTS; a run may
100
+ # carry its own, resolved once at pipeline start from settings.json by
101
+ # core.config_loader.effective_confidence.
102
+ #
103
+ # THE PARAMETER DEFAULTS TO THESE, which is what keeps every existing
104
+ # caller -- and the byte-for-byte regression tests this file is built
105
+ # around -- scoring exactly what they always did. A retuned run is an
106
+ # explicit act at one call site, never a silent re-reading of global
107
+ # state inside a pure function.
108
+ #
109
+ # Built by a FUNCTION rather than bound at import, so a test or a tool
110
+ # that patches one of the constants above still sees its own value here.
111
+ def default_weights() -> dict:
112
+ return {
113
+ "grounding_weight_factor": GROUNDING_WEIGHT_FACTOR,
114
+ "critic_weight_factor": CRITIC_WEIGHT_FACTOR,
115
+ "assumption_flag_penalty": ASSUMPTION_FLAG_PENALTY,
116
+ "disagreement_penalty_factor": DISAGREEMENT_PENALTY_FACTOR,
117
+ "non_factual_score_cap": NON_FACTUAL_SCORE_CAP,
118
+ "source_quality_floor": SOURCE_QUALITY_FLOOR,
119
+ "authority_full_credit": AUTHORITY_FULL_CREDIT,
120
+ "grounding_weights": dict(GROUNDING_WEIGHTS),
121
+ "tier_thresholds": {name: value for value, name in TIER_THRESHOLDS},
122
+ }
123
+
124
+
125
+ def _tier_for(raw_score: float, thresholds: dict) -> str:
126
+ """The tier `raw_score` earns, highest first.
127
+
128
+ SORTED HERE rather than trusting the mapping's order: the resolver
129
+ already refuses a table that does not descend, and sorting again means
130
+ a caller passing one by hand cannot make a tier unreachable by
131
+ writing its keys in the wrong order.
132
+ """
133
+ for name, threshold in sorted(thresholds.items(), key=lambda kv: -kv[1]):
134
+ if raw_score >= threshold:
135
+ return name
136
+ return "UNVERIFIED"
137
+
138
+
139
+ def source_quality(claim: Claim, scored: bool,
140
+ full_credit: float = None):
141
+ """The best source this claim has, as authority x similarity, or None.
142
+
143
+ A PRODUCT, not an average (§45, SQ3). An authoritative source that
144
+ does not discuss the claim and a perfect match on a content farm are
145
+ both worth nothing, and only multiplication says so. MAX across
146
+ sources, because grounding asks whether ANY credible source supports
147
+ the claim -- a noisy-OR that rewarded corroboration would let five
148
+ mediocre blogs outscore one primary source.
149
+
150
+ `scored` IS THE WHOLE SAFETY PROPERTY, and it is passed rather than
151
+ inferred. An empty source list is ambiguous on its own: it means "the
152
+ model claimed this was grounded and cited nothing" on a run that went
153
+ through §45's scoring stage, and it means "this claim was built
154
+ without one" everywhere else -- a test, a §20 correction, a resume.
155
+ Reading 0.0 from the second would re-tier claims nothing has measured,
156
+ which is exactly the silent rescoring this module's own history is
157
+ about. So None (leave the formula alone) is the answer whenever the
158
+ caller has not said scoring ran.
159
+ """
160
+ if not scored:
161
+ return None
162
+ if full_credit is None:
163
+ full_credit = AUTHORITY_FULL_CREDIT
164
+ sources = claim.grounding_sources
165
+ if not isinstance(sources, list):
166
+ return 0.0
167
+ values = []
168
+ for source in sources:
169
+ if not isinstance(source, dict):
170
+ continue
171
+ authority = source.get("authority_score")
172
+ relevance = source.get("similarity_score")
173
+ if any(isinstance(v, bool) or not isinstance(v, (int, float))
174
+ for v in (authority, relevance)):
175
+ continue
176
+ # Authority is normalised against AUTHORITY_FULL_CREDIT before the
177
+ # product: the table ranks source types, and a rank multiplied by
178
+ # a rank is not evidence. Above the reference it is capped rather
179
+ # than allowed to compensate for weak relevance -- a government
180
+ # site that does not discuss the claim is still not evidence for
181
+ # it, which is the property the product exists to express.
182
+ credit = min(1.0, max(0.0, float(authority)) / full_credit)
183
+ values.append(credit * min(1.0, max(0.0, float(relevance))))
184
+ # 0.0 rather than None for a claim with no scorable source at all:
185
+ # scoring DID run, and it found nothing behind a claim the model
186
+ # asserted. source_scoring already says so in the trace; this is the
187
+ # same finding reaching the score.
188
+ return max(values) if values else 0.0
189
+
190
+
191
+ def score_claim(claim: Claim, ensemble_n: int = 0,
192
+ sources_scored: bool = False,
193
+ weights: dict | None = None) -> tuple[ConfidenceTier, dict]:
194
+ """Returns (tier, score_breakdown). Pure function -- no side effects,
195
+ easy to unit test claim-by-claim without running any pass.
196
+
197
+ When ensemble_n > 0 (ROADMAP §10), cross-candidate DISAGREEMENT -- the
198
+ fraction of candidates that did NOT independently assert this claim --
199
+ is subtracted from a factual claim's score. ensemble_n is the number of
200
+ candidates that actually produced text, which on a run where a
201
+ candidate's provider failed is fewer than the configured roster.
202
+
203
+ ROADMAP_v2 §30 (B8, B9, B10) hardened all three of this function's
204
+ model-supplied inputs. Every coercion below is a LAST RESORT: the
205
+ payload boundary rejects each of these values at the pass that sent
206
+ them, and re-asks. What reaches here is a value that survived §3's
207
+ corrective retries, or a claim built by some path that never went
208
+ through Pass 2 -- and silently scoring it wrong is the outcome #75
209
+ measured.
210
+ """
211
+ # §45 (SQ6). Resolved weights or this module's own defaults -- and
212
+ # None is the default, so every caller that predates §45 scores
213
+ # exactly what it always did.
214
+ w = default_weights() if weights is None else weights
215
+
216
+ # B9. base.py's Literals are ANNOTATIONS. An unrecognised status used
217
+ # to take GROUNDING_WEIGHTS' 0.0 default AND slip past the
218
+ # forced-UNVERIFIED rule below, which compares == "ungrounded" -- so a
219
+ # claim the model said it could not source was published as merely
220
+ # LOW. Coerced to the CONSERVATIVE end, which is the one that cannot
221
+ # overstate confidence.
222
+ status = claim.grounding_status
223
+ unrecognised_status = None
224
+ if status is not None and status not in GROUNDING_STATUSES:
225
+ unrecognised_status, status = status, "ungrounded"
226
+
227
+ # Same, for the type. Factual is the conservative branch: it has no
228
+ # NON_FACTUAL_SCORE_CAP floor to sit on, and it is the only branch the
229
+ # forced-UNVERIFIED rule can reach. A capitalised `type` used to skip
230
+ # gate D0's routing entirely and then score MEDIUM off the cap.
231
+ claim_type = claim.type
232
+ unrecognised_type = None
233
+ if claim_type not in CLAIM_TYPES:
234
+ unrecognised_type, claim_type = claim_type, "factual"
235
+
236
+ # §45 (SQ3). What the grounding is WORTH, not a fourth axis beside it.
237
+ # None -- no scoring stage ran for this claim -- gives a multiplier of
238
+ # 1.0, which is the pre-§45 arithmetic exactly.
239
+ # ROUNDED ONCE, HERE, then used for both the formula and the
240
+ # breakdown -- E10's rule applied to a second derived value. The
241
+ # unrounded product of two fractions records as
242
+ # `grounding_component: 0.7264705882352941` in an artifact whose other
243
+ # fields are all 4 decimal places, and rounding it on the way OUT
244
+ # instead is exactly how E10's two values came to disagree: the score
245
+ # would be computed from one number and published as another.
246
+ quality = source_quality(claim, sources_scored, w["authority_full_credit"])
247
+ quality_multiplier = (1.0 if quality is None
248
+ else round(max(w["source_quality_floor"], quality), 4))
249
+ grounding_weight = w["grounding_weights"].get(status, 0.0)
250
+ grounding_component = grounding_weight * quality_multiplier
251
+ critic_component = 1.0 - claim.critic_severity
252
+ assumption_penalty = w["assumption_flag_penalty"] * len(claim.assumption_flags)
253
+
254
+ # None when ensemble mode is off, which is what makes the formula below
255
+ # ONE formula rather than a pair that can drift -- and (B7) also when
256
+ # ensemble mode is ON and Pass 2 reported nothing for this claim, which
257
+ # is a different fact from "no candidate asserted it".
258
+ consistency_score = None
259
+ consistency_reported = None
260
+ disagreement_penalty = 0.0
261
+ if ensemble_n > 0 and claim.asserted_by_candidates:
262
+ # B8. Deduplicated and clamped. `[1, 2, 3, 3]` made
263
+ # consistency 1.3333, so the PENALTY went NEGATIVE and a MEDIUM
264
+ # claim scored HIGH -- the exact property E8's comment says holds
265
+ # "by construction". `consistency_reported` keeps the unclamped
266
+ # value when it fired, because a numerator above the denominator is
267
+ # itself evidence that Pass 2 misread the candidate labelling, and
268
+ # clamping silently would hide the evidence along with the bug.
269
+ reported = len(set(claim.asserted_by_candidates)) / ensemble_n
270
+ consistency_score = min(1.0, reported)
271
+ if reported > 1.0:
272
+ consistency_reported = round(reported, 4)
273
+ # E9: computed only where it is applied, so the breakdown records
274
+ # what was subtracted rather than what was available (B10).
275
+ if claim_type == "factual":
276
+ disagreement_penalty = w["disagreement_penalty_factor"] * (1.0 - consistency_score)
277
+
278
+ if claim_type != "factual":
279
+ # Cap applies to the critic component BEFORE subtracting the
280
+ # assumption penalty -- capping the final score instead would let
281
+ # the cap swallow the penalty whenever the pre-penalty score
282
+ # already exceeded it, making assumption flags invisible on any
283
+ # claim that was already clean under critique. Found this via a
284
+ # direct test case (two speculative claims, one flagged one not,
285
+ # both landing at the same tier) before it shipped.
286
+ #
287
+ # E9: no disagreement penalty here. §10 decided consistency does not
288
+ # RESCUE a non-factual claim from the cap; whether dissent should
289
+ # penalise a speculative claim is a separate judgment about what
290
+ # candidates declining to speculate means, and it was not made.
291
+ capped_critic = min(critic_component, w["non_factual_score_cap"])
292
+ raw_score = capped_critic - assumption_penalty
293
+ formula = "non_factual"
294
+ else:
295
+ capped_critic = None
296
+ formula = "factual"
297
+ raw_score = (
298
+ (w["grounding_weight_factor"] * grounding_component)
299
+ + (w["critic_weight_factor"] * critic_component)
300
+ - assumption_penalty
301
+ - disagreement_penalty
302
+ )
303
+
304
+ # ROUND ONCE, then use that one value for BOTH the tier comparison and
305
+ # the breakdown. These used to be two values: the tier compared the raw
306
+ # float while the breakdown stored round(raw, 4), so a claim scoring
307
+ # 0.7999999999999999 was recorded as `raw_score: 0.8` and tiered MEDIUM
308
+ # -- against a TIER_THRESHOLDS table that says >= 0.8 is HIGH. Anyone
309
+ # auditing a run artifact read a number that contradicted the published
310
+ # thresholds, and the tier was the one that mattered.
311
+ #
312
+ # Binary floating point makes this reachable from ordinary weights, not
313
+ # from an exotic input: 0.5 + 0.35 - 0.15 * (1/3) is exactly that value.
314
+ # Sweeping the whole non-ensemble input space, exactly ONE shape changes
315
+ # tier under this fix -- a speculative claim at severity 0.55 with one
316
+ # assumption flag, which records 0.3 and now tiers LOW instead of
317
+ # UNVERIFIED. That is what the threshold table always specified.
318
+ raw_score = round(max(0.0, min(1.0, raw_score)), 4)
319
+
320
+ if claim_type == "factual" and status == "ungrounded":
321
+ tier: ConfidenceTier = "UNVERIFIED"
322
+ else:
323
+ tier = _tier_for(raw_score, w["tier_thresholds"])
324
+
325
+ # ROADMAP_v2 §30 (B10). THE BREAKDOWN RECORDS WHAT WAS APPLIED, AND
326
+ # NAMES THE BRANCH THAT APPLIED IT, so a reader can recompute raw_score
327
+ # from these fields alone. It could not before: a speculative claim
328
+ # asserted by 1 of 3 candidates stored
329
+ # `grounding_component: 0.0, disagreement_penalty: 0.1, raw_score: 0.65`
330
+ # -- and recomputing from those gives 0.25, because E9 subtracts the
331
+ # penalty only in the factual branch and grounding is not in the
332
+ # non-factual formula at all. Both fields were inert and neither said
333
+ # so.
334
+ #
335
+ # Same class as E10 on the same object, which is why it is worth the
336
+ # extra key: "anyone auditing a run artifact read a number that
337
+ # contradicted the published thresholds."
338
+ #
339
+ # factual 0.5*grounding + 0.35*critic - assumption - disagreement
340
+ # non_factual capped_critic - assumption
341
+ breakdown = {
342
+ "formula": formula,
343
+ "critic_component": critic_component,
344
+ "assumption_penalty": assumption_penalty,
345
+ # Already rounded above -- rounding again here is what let the two
346
+ # values drift apart in the first place.
347
+ "raw_score": raw_score,
348
+ }
349
+ if formula == "factual":
350
+ breakdown["grounding_component"] = grounding_component
351
+ # B10's property, extended: the breakdown alone must let a reader
352
+ # recompute grounding_component. Written ONLY on the factual
353
+ # branch and only when scoring ran, so a field that did nothing is
354
+ # never present -- which is the exact defect B10 records for
355
+ # `grounding_component` and `disagreement_penalty` on a
356
+ # speculative claim.
357
+ if quality is not None:
358
+ breakdown["grounding_weight"] = grounding_weight
359
+ breakdown["source_quality"] = round(quality, 4)
360
+ breakdown["source_quality_multiplier"] = quality_multiplier
361
+ breakdown["source_count"] = len(claim.grounding_sources) \
362
+ if isinstance(claim.grounding_sources, list) else 0
363
+ else:
364
+ breakdown["capped_critic_component"] = capped_critic
365
+ if ensemble_n > 0:
366
+ # E12: the DENOMINATOR, so a stored claim is self-describing on a run
367
+ # where the roster shrank. Without it, "consistency 0.5" is
368
+ # ambiguous between one of two candidates and two of four, and the
369
+ # roster that produced it lives only in the trace.
370
+ breakdown["consistency_denominator"] = ensemble_n
371
+ # B7. None means PASS 2 REPORTED NOTHING, which is not 0.0 -- and
372
+ # the two were indistinguishable, so a claim one candidate raised
373
+ # uncontested took the maximum 0.15 penalty rather than the 0.05 it
374
+ # earned. #12 reads this field; it needs to be able to tell them
375
+ # apart too.
376
+ breakdown["consistency_score"] = (
377
+ None if consistency_score is None else round(consistency_score, 4))
378
+ breakdown["disagreement_penalty"] = round(disagreement_penalty, 4)
379
+ if consistency_reported is not None:
380
+ breakdown["consistency_reported"] = consistency_reported
381
+ if unrecognised_status is not None:
382
+ breakdown["unrecognised_grounding_status"] = unrecognised_status
383
+ if unrecognised_type is not None:
384
+ breakdown["unrecognised_type"] = unrecognised_type
385
+ return tier, breakdown
386
+
387
+
388
+ def build_coverage_gap_entries(completeness: dict) -> list[dict]:
389
+ """Gaps from Pass 3c become separate coverage-gap entries, not Claim
390
+ objects -- they were never asserted, so there's nothing to re-tier."""
391
+ return [{"gap": gap, "tier": "UNVERIFIED_COVERAGE"} for gap in completeness.get("gaps", [])]
392
+
393
+
394
+ def run_confidence_tiering(
395
+ run: PipelineRun, ensemble_n: int = 0,
396
+ claims: list[Claim] | None = None,
397
+ sources_scored: bool = False,
398
+ weights: dict | None = None,
399
+ ) -> None:
400
+ """The actual Pass 5 entry point the orchestrator calls. Mutates
401
+ the claims in place; zero LLM calls. Pass ensemble_n > 0 to use
402
+ the ensemble formula variant (ROADMAP §10).
403
+
404
+ `claims` narrows WHICH claims are re-tiered, and defaults to all of
405
+ them -- which is Pass 5 itself, where every claim is in play. The
406
+ 6a/6c loop passes its current batch instead: it calls this once per
407
+ round, and re-tiering the whole run there recomputes claims that are
408
+ already finished. A claim D2 had exhausted got its tier restored from
409
+ unchanged score data while the fallback annotation _apply_fallback
410
+ wrote stayed put, so it shipped as (say) LOW beside a note saying it
411
+ could not be verified -- and a claim D1 never flagged could be
412
+ re-tiered by any round that touched its grounding.
413
+
414
+ `run.coverage_gaps` is rebuilt either way: it is a pure function of
415
+ run.completeness, which does not change across the loop, so the
416
+ narrowed call leaves it exactly as the full one would.
417
+ """
418
+ for claim in (run.claims if claims is None else claims):
419
+ tier, breakdown = score_claim(claim, ensemble_n=ensemble_n,
420
+ sources_scored=sources_scored,
421
+ weights=weights)
422
+ claim.confidence_tier = tier
423
+ claim.score_breakdown = breakdown
424
+ _report_coercions(run, claim, breakdown, ensemble_n)
425
+
426
+ run.coverage_gaps = build_coverage_gap_entries(run.completeness)
427
+
428
+
429
+ def _report_coercions(run: PipelineRun, claim: Claim, breakdown: dict,
430
+ ensemble_n: int) -> None:
431
+ """ROADMAP_v2 §30. Pass 5 makes no model call, so a value it had to
432
+ correct has nowhere else to surface.
433
+
434
+ Every line here describes something that should have been impossible:
435
+ the payload boundary rejects each of these at the pass that sent it and
436
+ re-asks. Reaching Pass 5 means the value survived §3's retries, or the
437
+ claim did not come through Pass 2 at all. `run.trace` is the artifact
438
+ base.py calls as trustworthy as the report -- silently scoring around a
439
+ bad value is precisely what #75 measured, so the correction is stated
440
+ even though it is also the safe one.
441
+ """
442
+ if "consistency_reported" in breakdown:
443
+ run.log(
444
+ f"Pass 5: claim {claim.id} was asserted by candidates "
445
+ f"{claim.asserted_by_candidates} against a denominator of "
446
+ f"{ensemble_n} -- consistency clamped to 1.0. A numerator above "
447
+ f"the denominator means Pass 2 misread the candidate labelling."
448
+ )
449
+ if ensemble_n > 0 and breakdown.get("consistency_score") is None:
450
+ run.log(
451
+ f"Pass 5: claim {claim.id} carries no asserted_by_candidates on "
452
+ f"an ensemble run -- scored WITHOUT a disagreement penalty, "
453
+ f"since an unreported numerator is not a numerator of zero."
454
+ )
455
+ if "unrecognised_grounding_status" in breakdown:
456
+ run.log(
457
+ f"Pass 5: claim {claim.id} has grounding_status="
458
+ f"{breakdown['unrecognised_grounding_status']!r}, which is not "
459
+ f"one of {', '.join(sorted(GROUNDING_STATUSES))} -- scored as "
460
+ f"ungrounded."
461
+ )
462
+ if "unrecognised_type" in breakdown:
463
+ run.log(
464
+ f"Pass 5: claim {claim.id} has type="
465
+ f"{breakdown['unrecognised_type']!r}, which is not one of "
466
+ f"{', '.join(sorted(CLAIM_TYPES))} -- scored on the factual "
467
+ f"formula."
468
+ )
@@ -0,0 +1,149 @@
1
+ """
2
+ core/reasoning/events.py
3
+
4
+ PipelineEvent is the single event type yielded by
5
+ core.reasoning.orchestrator.stream_deep_research_pipeline() as the
6
+ ten-pass research pipeline progresses. Consumers (the CLI's research
7
+ mode, the TUI's /research worker) iterate the generator and render what
8
+ they care about; run_pipeline_to_completion() drains it for callers that
9
+ only need the finished PipelineRun.
10
+
11
+ WHY THIS IS A SEPARATE TYPE FROM core.events.LoopEvent (ROADMAP_v2 §22
12
+ §22 AC4, decision P1). §22 required this decision be made and recorded
13
+ BEFORE any event was added, precisely so nobody discovered the problem at
14
+ §23. Three shapes were considered:
15
+
16
+ 1. Add fields to LoopEvent. Rejected. LoopEvent is a flat dataclass
17
+ with an "exactly one field is populated" convention documented in a
18
+ docstring rather than in the type. That was right for six kinds;
19
+ §22 adds seven of a different family and §23 adds more, at which
20
+ point ~15 optional fields whose valid combinations live in prose
21
+ stops paying for itself.
22
+ 2. A tagged union for both. Rejected as disproportionate: it rewrites
23
+ every `if event.token_delta:` read in tui/app.py, core/loop.py and
24
+ the streaming tests -- a §13-scale breakage inside a §22 change.
25
+ 3. A separate, KIND-DISCRIMINATED type. Chosen. The two families have
26
+ different consumers and different lifetimes: a LoopEvent describes
27
+ one model call's progress and lives for a turn, a PipelineEvent
28
+ describes a ten-pass run and lives for the run. The `kind` field
29
+ puts the discriminator in the data rather than in a convention, so
30
+ adding a kind cannot silently widen what "exactly one field" means.
31
+
32
+ The two types deliberately do NOT compose, and ROADMAP_v2 §26 did not
33
+ change that. §22's P2 said a pass's LoopEvents are not forwarded up, on
34
+ the premise that a pass's internals are not worth seeing; a real ten-pass
35
+ run disproved the premise (a pass making fourteen tool calls over several
36
+ minutes was indistinguishable from one that had hung). So §26 amends P2:
37
+ core/reasoning/orchestrator.py now iterates
38
+ RunAgentLoop.stream_deep_research_mode() and TRANSLATES a chosen subset of
39
+ its LoopEvents into the pipeline kinds below.
40
+
41
+ Translated, not forwarded -- and the distinction is the whole amendment. A
42
+ consumer of this generator still receives exactly one event type, so no
43
+ `if event.token_delta:` read exists anywhere outside core/loop.py and the
44
+ streaming tests, which is what P1 and P2 were jointly protecting.
45
+ core/client.py, core/reasoning/json_retry.py and review.py's internals
46
+ remain untouched.
47
+
48
+ KNOWN LIMIT, stated rather than fixed: json_retry.py reaches the model via
49
+ continue_conversation(), which drains its own loop internally, so tool
50
+ calls made during a corrective JSON retry produce no tool_call event. The
51
+ retry itself is still visible -- that module appends a trace line per
52
+ failed parse, which _Progress's watermark carries as a trace_line.
53
+
54
+ ERROR HANDLING: there is deliberately no error variant, matching
55
+ core/events.py for the same reason. Exceptions raised mid-pipeline
56
+ propagate out of the generator unchanged, so the orchestrator's
57
+ `except Exception: update_pipeline_run(..., status="failed"); raise`
58
+ keeps working exactly as it did when the function was synchronous.
59
+ Turning failures into data-shaped events would require every consumer to
60
+ know to unpack and re-raise them.
61
+ """
62
+
63
+ from dataclasses import dataclass
64
+ from typing import Any, Optional
65
+
66
+ # Every kind stream_deep_research_pipeline() can yield. Named here rather
67
+ # than left implicit in the emission sites so a consumer can be written
68
+ # against the list, and so an unknown kind in a test is a typo rather
69
+ # than a silently-ignored event.
70
+ PIPELINE_EVENT_KINDS = (
71
+ "pass_start", # pass_id -- an LLM-backed pass is about to run
72
+ "pass_complete", # pass_id (+ ok/text) -- it returned. ok=False
73
+ # marks a pass that ENDED BADLY (#115/E14): text
74
+ # carries a short reason, mirroring tool_result,
75
+ # and no success line may be printed for it. A
76
+ # skipped ensemble candidate resolves this way
77
+ # while the run continues; a fatal pass emits it
78
+ # before its exception propagates.
79
+ "trace_line", # text -- one line appended to PipelineRun.trace
80
+ "claim_extracted", # claim_id + text -- Pass 2 produced a claim
81
+ "claim_tiered", # claim_id + tier -- Pass 5 (or a 6b round) scored it
82
+ "retry", # claim_id + attempt -- Pass 6a revised a claim
83
+ "run_complete", # run -- the terminal event, carrying the PipelineRun
84
+ # --- §26 ---------------------------------------------------------
85
+ "stage", # pass_id -- a ZERO-LLM stage completed. Pass 5 and
86
+ # 6c are passes by the pipeline's numbering and
87
+ # stages by their cost: they make no model call,
88
+ # so they complete instantly and can never be
89
+ # meaningfully "running". Consumers distinguish
90
+ # them by THIS kind (tui/widgets.py's is_stage,
91
+ # main.py's '..'), never by matching pass-id
92
+ # strings -- #84 deleted CODE_STAGES, an unused
93
+ # list of those strings whose only future was to
94
+ # drift away from the orchestrator's _stage()
95
+ # call sites while looking like a contract.
96
+ "tool_call", # pass_id + tool + text -- a pass invoked a tool
97
+ "tool_result", # pass_id + tool + ok (+ text when it failed)
98
+ "pass_activity", # pass_id + chars -- output streamed so far, throttled
99
+ )
100
+
101
+ # There is deliberately NO "step" kind. §26 set out to report which model
102
+ # call a pass was on, and _run() has no step marker to read: within one
103
+ # step it yields tool_call_start/tool_result in pairs, and BETWEEN steps it
104
+ # yields nothing at all unless the model happened to stream text. Inferring
105
+ # a boundary from that interleaving would be a guess presented as a fact,
106
+ # and growing LoopEvent to carry a real one is what P1's test forbids.
107
+ #
108
+ # So a consumer counts tool_call events itself -- derivation, not a second
109
+ # writer of related data -- and pass_activity covers the other half: a pass
110
+ # that calls no tools (Pass 1, final synthesis) still proves it is alive by
111
+ # how much it has written.
112
+
113
+
114
+ @dataclass
115
+ class PipelineEvent:
116
+ """One event yielded by stream_deep_research_pipeline().
117
+
118
+ `kind` says which of the fields below are meaningful; the rest stay
119
+ None. See PIPELINE_EVENT_KINDS for the mapping.
120
+
121
+ This type GROWS with its kinds, which is what P1 chose it for. The
122
+ contrast is LoopEvent, whose "exactly one field is populated" rule
123
+ lives in a docstring -- so an EIGHTH field there (it has seven: six
124
+ payload fields plus stop_reason) would silently widen a convention
125
+ nothing checks, and a test fails if one appears. Here the
126
+ discriminator is in the data, so a reader of `kind` cannot be wrong
127
+ about which fields to trust.
128
+ """
129
+
130
+ kind: str
131
+ pass_id: Optional[str] = None
132
+ text: Optional[str] = None
133
+ claim_id: Optional[str] = None
134
+ tier: Optional[str] = None
135
+ attempt: Optional[int] = None
136
+ # §26. `text` carries the tool's redacted param digest on a tool_call
137
+ # and the error message on a failed tool_result -- one field for "the
138
+ # human-readable detail", rather than two more that are each set on one
139
+ # kind.
140
+ tool: Optional[str] = None
141
+ ok: Optional[bool] = None
142
+ # Running total of characters the pass has streamed, on pass_activity.
143
+ # A total rather than a delta, so a consumer that drops an event (or
144
+ # attaches late) still renders the right number.
145
+ chars: Optional[int] = None
146
+ # PipelineRun on the terminal event. Typed Any to keep this module
147
+ # importable from anywhere without dragging base.py in behind it --
148
+ # the same reason LoopEvent.final_response is Any.
149
+ run: Optional[Any] = None