@yahyahammad/venastine 0.2.0 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/.env.example +9 -4
  2. package/CONFIG_ARCHITECTURE.md +2030 -0
  3. package/LICENSE +201 -201
  4. package/NOTICE +1 -1
  5. package/README.md +1034 -895
  6. package/agents/builtin/build.md +73 -0
  7. package/agents/builtin/compactor.md +75 -75
  8. package/agents/builtin/explore.md +77 -77
  9. package/agents/builtin/general.md +80 -0
  10. package/agents/builtin/pipeline-reviewer.md +100 -100
  11. package/agents/builtin/plan.md +106 -88
  12. package/agents/builtin/review.md +74 -74
  13. package/agents/builtin/test.md +65 -0
  14. package/agents/builtin/writer.md +67 -0
  15. package/agents/manager.py +68 -2
  16. package/agents/subagent_tool.py +472 -238
  17. package/agents/tui_commands.py +238 -151
  18. package/bin/venastine.mjs +150 -4
  19. package/bin/venastine.py +71 -0
  20. package/cli_commands.py +838 -0
  21. package/config.py +95 -1198
  22. package/config.yaml +1103 -0
  23. package/config_edit.py +1301 -0
  24. package/config_schema.py +971 -0
  25. package/config_update.py +749 -0
  26. package/core/agent_activity.py +256 -118
  27. package/core/ansi.py +452 -0
  28. package/core/approval.py +137 -115
  29. package/core/call_view.py +364 -0
  30. package/core/cancel.py +121 -0
  31. package/core/chatgpt_auth.py +747 -0
  32. package/core/claims_view.py +107 -0
  33. package/core/client.py +2715 -1559
  34. package/core/commands.py +460 -0
  35. package/core/compaction.py +1735 -971
  36. package/core/config_loader.py +1982 -1563
  37. package/core/connect.py +686 -0
  38. package/core/events.py +123 -61
  39. package/core/file_mentions.py +280 -0
  40. package/core/interaction.py +411 -328
  41. package/core/line_pattern.py +128 -0
  42. package/core/loop.py +2566 -1490
  43. package/core/memory.py +482 -370
  44. package/core/pipeline_models.py +190 -176
  45. package/core/provider_errors.py +234 -0
  46. package/core/reasoning/authorization.py +130 -130
  47. package/core/reasoning/base.py +219 -219
  48. package/core/reasoning/confidence_scoring.py +484 -468
  49. package/core/reasoning/events.py +149 -149
  50. package/core/reasoning/json_retry.py +19 -1
  51. package/core/reasoning/orchestrator.py +1561 -1382
  52. package/core/reasoning/output_writer.py +345 -286
  53. package/core/reasoning/payload_validation.py +524 -398
  54. package/core/reasoning/pipeline_storage.py +88 -22
  55. package/core/reasoning/review.py +659 -615
  56. package/core/reasoning/scholar.py +458 -434
  57. package/core/reasoning/source_corpus.py +433 -327
  58. package/core/reasoning/source_scoring.py +855 -855
  59. package/core/replay.py +315 -183
  60. package/core/session.py +36 -12
  61. package/core/session_view.py +259 -0
  62. package/core/session_wake.py +259 -0
  63. package/core/shell_sessions.py +1553 -0
  64. package/core/workspace_trust.py +1 -1
  65. package/credentials.py +148 -20
  66. package/database.py +333 -252
  67. package/{PRIVACY.md → docs/PRIVACY.md} +78 -66
  68. package/docs/SECURITY.md +108 -0
  69. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  70. package/env_secrets.py +50 -9
  71. package/json_store.py +343 -123
  72. package/logging_setup.py +295 -185
  73. package/main.py +3433 -1961
  74. package/mcp_client/client.py +733 -499
  75. package/mcp_client/config.py +126 -21
  76. package/mcp_client/registration.py +343 -182
  77. package/memories/manager.py +147 -132
  78. package/memories/tui_commands.py +319 -244
  79. package/package.json +303 -73
  80. package/project_init/config_files.py +282 -278
  81. package/project_init/generator.py +728 -564
  82. package/project_init/manifest.py +18 -6
  83. package/project_init/tui_commands.py +147 -144
  84. package/prompts/critic_pass.md +24 -24
  85. package/prompts/revalidate.md +25 -25
  86. package/prompts/revise.md +2 -0
  87. package/prompts/source_grounding.md +66 -66
  88. package/prompts/system_prompts.py +316 -262
  89. package/providers.json.example +4 -2
  90. package/pyproject.toml +149 -100
  91. package/requirements.txt +295 -72
  92. package/safety/policy_enforcement.py +1082 -746
  93. package/security/askpass.py +419 -0
  94. package/security/capability.py +207 -177
  95. package/security/confine.py +484 -0
  96. package/security/hosted.py +182 -0
  97. package/security/permissions.py +237 -190
  98. package/security/posture.py +124 -12
  99. package/security/protected_paths.py +501 -309
  100. package/security/sandbox.py +4108 -985
  101. package/security/secrets.py +855 -0
  102. package/skills/builtin/math/formal-specification.md +67 -67
  103. package/skills/builtin/math/numerical-methods.md +66 -66
  104. package/skills/builtin/research/experiment-design.md +62 -62
  105. package/skills/builtin/research/reproducibility.md +55 -55
  106. package/skills/builtin/research/source-evaluation.md +70 -70
  107. package/skills/builtin/research/statistical-inference.md +63 -63
  108. package/skills/builtin/research/technical-writing.md +59 -59
  109. package/skills/builtin/software/code-review.md +61 -61
  110. package/skills/builtin/software/debugging.md +61 -61
  111. package/skills/builtin/software/test-design.md +67 -67
  112. package/skills/tui_commands.py +110 -111
  113. package/storage.py +1338 -1005
  114. package/tools/base.py +161 -1
  115. package/tools/builtin/_math_common.py +667 -377
  116. package/tools/builtin/_net_common.py +225 -19
  117. package/tools/builtin/arxiv.py +497 -39
  118. package/tools/builtin/ask_user.py +231 -213
  119. package/tools/builtin/discrete_math.py +35 -4
  120. package/tools/builtin/fetch_url.py +371 -153
  121. package/tools/builtin/file_ops.py +720 -412
  122. package/tools/builtin/geometry.py +48 -12
  123. package/tools/builtin/get_time.py +2 -2
  124. package/tools/builtin/linear_algebra.py +66 -23
  125. package/tools/builtin/load_skill.py +76 -66
  126. package/tools/builtin/logic.py +171 -12
  127. package/tools/builtin/pin.py +227 -193
  128. package/tools/builtin/probability_stats.py +12 -16
  129. package/tools/builtin/project_docs.py +464 -405
  130. package/tools/builtin/remember.py +175 -158
  131. package/tools/builtin/shell.py +980 -461
  132. package/tools/builtin/shell_sessions.py +613 -0
  133. package/tools/builtin/symbolic_math.py +80 -10
  134. package/tools/builtin/todo.py +149 -147
  135. package/tools/builtin/web_search.py +191 -13
  136. package/tools/context.py +159 -136
  137. package/tools/isolation.py +107 -34
  138. package/tools/registry.py +1242 -925
  139. package/tui/app.py +8318 -4100
  140. package/tui/app.tcss +747 -556
  141. package/tui/commands.py +26 -170
  142. package/tui/diffs.py +34 -13
  143. package/tui/history.py +128 -128
  144. package/tui/hosted_driver.py +102 -0
  145. package/tui/markdown.py +1329 -1279
  146. package/tui/meters.py +364 -268
  147. package/tui/ravens.py +146 -146
  148. package/tui/screens.py +1255 -943
  149. package/tui/themes.py +599 -559
  150. package/tui/widgets.py +3813 -2332
  151. package/SECURITY.md +0 -88
  152. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -1,468 +1,484 @@
1
- """
2
- core/reasoning/confidence_scoring.py
3
-
4
- Pass 5 -- confidence tiering. ZERO LLM calls, by design: this is a
5
- deterministic, documented weighted formula over already-computed data
6
- (grounding status, critic severity, assumption flags). Kept as its own
7
- module, separate from orchestrator.py, so the weights can be tuned later
8
- without touching any prompt or any sequencing logic.
9
-
10
- Interpretation worth flagging: Pass 3c's "gaps" (things the response
11
- never addressed at all) are NOT claims that exist and failed
12
- verification -- they're an absence. Rather than forcing them into the
13
- Claim model, they're represented separately as coverage_gaps, each
14
- tagged UNVERIFIED_COVERAGE. This is my reading of what that tier is for;
15
- flag it if you intended something different.
16
- """
17
-
18
- from __future__ import annotations
19
-
20
- from core.reasoning.base import (
21
- CLAIM_TYPES, GROUNDING_STATUSES, Claim, ConfidenceTier, PipelineRun,
22
- )
23
-
24
- # --- Tunable weights -- change these, not the logic below, to retune scoring ---
25
- GROUNDING_WEIGHTS = {"grounded": 1.0, "partial": 0.5, "ungrounded": 0.0, None: 0.0}
26
- GROUNDING_WEIGHT_FACTOR = 0.5
27
- CRITIC_WEIGHT_FACTOR = 0.35
28
- ASSUMPTION_FLAG_PENALTY = 0.15 # per flag
29
- NON_FACTUAL_SCORE_CAP = 0.65 # claims never grounded can't reach HIGH
30
-
31
- # ROADMAP §10 (revisit, E8). Cross-candidate disagreement is a PENALTY,
32
- # subtracted like ASSUMPTION_FLAG_PENALTY above -- not a fourth weighted
33
- # term with the others scaled down to make room for it.
34
- #
35
- # §10 shipped the other shape: 0.5/0.35 redistributed to 0.4/0.3 with a
36
- # 0.15 consistency term added. Two consequences, neither intended:
37
- #
38
- # 1. Turning ensemble mode on made GROUNDING count for less. Grounding is
39
- # the pipeline's strongest evidence, and which generation strategy
40
- # produced a claim is no reason to weigh its verification differently.
41
- # 2. Because the maximum stayed 0.85, consistency could only ever cost a
42
- # well-grounded claim. A claim Pass 3a grounded and Pass 3b found
43
- # nothing wrong with dropped from HIGH to MEDIUM at 2-of-3 agreement.
44
- #
45
- # As a penalty, unanimity changes nothing and dissent subtracts, which is
46
- # also the honest reading of the signal: agreement is the null hypothesis
47
- # (candidates asked the same question usually answer alike), so disagreement
48
- # is the part that carries information. It makes the non-ensemble formula
49
- # literally unchanged rather than coincidentally equal, so §10's
50
- # byte-for-byte regression guard holds by construction.
51
- DISAGREEMENT_PENALTY_FACTOR = 0.15
52
-
53
- # ROADMAP_v2 §45 (SQ3). What a grounding is WORTH.
54
- #
55
- # MULTIPLICATIVE, NOT A FOURTH TERM, and that is the whole design. §10
56
- # shipped a fourth weighted term for consistency and reversed it (see
57
- # E8 above): redistributing 0.5/0.35 to make room meant grounding counted
58
- # for less whenever the feature was on, and the new term could only ever
59
- # subtract. Scaling the grounding component instead says what is actually
60
- # meant -- how much is this grounding worth -- and leaves the non-scored
61
- # formula LITERALLY unchanged rather than coincidentally equal, so the
62
- # byte-for-byte regression guard holds by construction here too.
63
- #
64
- # BOTH CONSTANTS BELOW WERE SET BY MEASUREMENT, not by taste, and the
65
- # first draft of each was wrong in a way only a sweep showed.
66
- #
67
- # AUTHORITY_FULL_CREDIT exists because the domain table is a RANKING and
68
- # this formula needs an EVIDENTIAL scale. Feeding `authority` in raw
69
- # multiplied two rank-scales together and landed everything in the middle:
70
- # swept across the whole table, only a `.gov` at similarity 1.0 still
71
- # reached HIGH, and Nature, arXiv, Reddit and a Pinterest pin were
72
- # indistinguishable at MEDIUM -- a tier that no longer discriminates,
73
- # which is the opposite of the point. At or above a peer-reviewed
74
- # publisher, authority is not what limits a claim's confidence, so that is
75
- # where full credit starts.
76
- #
77
- # SOURCE_QUALITY_FLOOR was 0.4 in the first draft, and 0.4 is exactly the
78
- # worst available value: 0.5*0.4 + 0.35 == 0.55, which IS the MEDIUM
79
- # threshold, so the floor guaranteed MEDIUM for every grounded claim
80
- # however bad its sources were. At 0.25 a claim grounded only in a content
81
- # farm lands LOW -- flagged, and sent to the 6a/6b loop to find something
82
- # better, which is the measured Pinterest case getting the treatment it
83
- # always should have.
84
- #
85
- # NOT ZERO, though: a weak-but-real grounding is still better evidenced
86
- # than one nothing supports, and the forced-UNVERIFIED rule below already
87
- # handles the latter.
88
- AUTHORITY_FULL_CREDIT = 0.85
89
- SOURCE_QUALITY_FLOOR = 0.25
90
-
91
-
92
- TIER_THRESHOLDS = [
93
- (0.8, "HIGH"),
94
- (0.55, "MEDIUM"),
95
- (0.3, "LOW"),
96
- ] # anything below the lowest threshold, or an ungrounded factual claim, is UNVERIFIED
97
-
98
-
99
- # ROADMAP_v2 §45 (SQ6). The constants above are the DEFAULTS; a run may
100
- # carry its own, resolved once at pipeline start from settings.json by
101
- # core.config_loader.effective_confidence.
102
- #
103
- # THE PARAMETER DEFAULTS TO THESE, which is what keeps every existing
104
- # caller -- and the byte-for-byte regression tests this file is built
105
- # around -- scoring exactly what they always did. A retuned run is an
106
- # explicit act at one call site, never a silent re-reading of global
107
- # state inside a pure function.
108
- #
109
- # Built by a FUNCTION rather than bound at import, so a test or a tool
110
- # that patches one of the constants above still sees its own value here.
111
- def default_weights() -> dict:
112
- return {
113
- "grounding_weight_factor": GROUNDING_WEIGHT_FACTOR,
114
- "critic_weight_factor": CRITIC_WEIGHT_FACTOR,
115
- "assumption_flag_penalty": ASSUMPTION_FLAG_PENALTY,
116
- "disagreement_penalty_factor": DISAGREEMENT_PENALTY_FACTOR,
117
- "non_factual_score_cap": NON_FACTUAL_SCORE_CAP,
118
- "source_quality_floor": SOURCE_QUALITY_FLOOR,
119
- "authority_full_credit": AUTHORITY_FULL_CREDIT,
120
- "grounding_weights": dict(GROUNDING_WEIGHTS),
121
- "tier_thresholds": {name: value for value, name in TIER_THRESHOLDS},
122
- }
123
-
124
-
125
- def _tier_for(raw_score: float, thresholds: dict) -> str:
126
- """The tier `raw_score` earns, highest first.
127
-
128
- SORTED HERE rather than trusting the mapping's order: the resolver
129
- already refuses a table that does not descend, and sorting again means
130
- a caller passing one by hand cannot make a tier unreachable by
131
- writing its keys in the wrong order.
132
- """
133
- for name, threshold in sorted(thresholds.items(), key=lambda kv: -kv[1]):
134
- if raw_score >= threshold:
135
- return name
136
- return "UNVERIFIED"
137
-
138
-
139
- def source_quality(claim: Claim, scored: bool,
140
- full_credit: float = None):
141
- """The best source this claim has, as authority x similarity, or None.
142
-
143
- A PRODUCT, not an average (§45, SQ3). An authoritative source that
144
- does not discuss the claim and a perfect match on a content farm are
145
- both worth nothing, and only multiplication says so. MAX across
146
- sources, because grounding asks whether ANY credible source supports
147
- the claim -- a noisy-OR that rewarded corroboration would let five
148
- mediocre blogs outscore one primary source.
149
-
150
- `scored` IS THE WHOLE SAFETY PROPERTY, and it is passed rather than
151
- inferred. An empty source list is ambiguous on its own: it means "the
152
- model claimed this was grounded and cited nothing" on a run that went
153
- through §45's scoring stage, and it means "this claim was built
154
- without one" everywhere else -- a test, a §20 correction, a resume.
155
- Reading 0.0 from the second would re-tier claims nothing has measured,
156
- which is exactly the silent rescoring this module's own history is
157
- about. So None (leave the formula alone) is the answer whenever the
158
- caller has not said scoring ran.
159
- """
160
- if not scored:
161
- return None
162
- if full_credit is None:
163
- full_credit = AUTHORITY_FULL_CREDIT
164
- sources = claim.grounding_sources
165
- if not isinstance(sources, list):
166
- return 0.0
167
- values = []
168
- for source in sources:
169
- if not isinstance(source, dict):
170
- continue
171
- authority = source.get("authority_score")
172
- relevance = source.get("similarity_score")
173
- if any(isinstance(v, bool) or not isinstance(v, (int, float))
174
- for v in (authority, relevance)):
175
- continue
176
- # Authority is normalised against AUTHORITY_FULL_CREDIT before the
177
- # product: the table ranks source types, and a rank multiplied by
178
- # a rank is not evidence. Above the reference it is capped rather
179
- # than allowed to compensate for weak relevance -- a government
180
- # site that does not discuss the claim is still not evidence for
181
- # it, which is the property the product exists to express.
182
- credit = min(1.0, max(0.0, float(authority)) / full_credit)
183
- values.append(credit * min(1.0, max(0.0, float(relevance))))
184
- # 0.0 rather than None for a claim with no scorable source at all:
185
- # scoring DID run, and it found nothing behind a claim the model
186
- # asserted. source_scoring already says so in the trace; this is the
187
- # same finding reaching the score.
188
- return max(values) if values else 0.0
189
-
190
-
191
- def score_claim(claim: Claim, ensemble_n: int = 0,
192
- sources_scored: bool = False,
193
- weights: dict | None = None) -> tuple[ConfidenceTier, dict]:
194
- """Returns (tier, score_breakdown). Pure function -- no side effects,
195
- easy to unit test claim-by-claim without running any pass.
196
-
197
- When ensemble_n > 0 (ROADMAP §10), cross-candidate DISAGREEMENT -- the
198
- fraction of candidates that did NOT independently assert this claim --
199
- is subtracted from a factual claim's score. ensemble_n is the number of
200
- candidates that actually produced text, which on a run where a
201
- candidate's provider failed is fewer than the configured roster.
202
-
203
- ROADMAP_v2 §30 (B8, B9, B10) hardened all three of this function's
204
- model-supplied inputs. Every coercion below is a LAST RESORT: the
205
- payload boundary rejects each of these values at the pass that sent
206
- them, and re-asks. What reaches here is a value that survived §3's
207
- corrective retries, or a claim built by some path that never went
208
- through Pass 2 -- and silently scoring it wrong is the outcome #75
209
- measured.
210
- """
211
- # §45 (SQ6). Resolved weights or this module's own defaults -- and
212
- # None is the default, so every caller that predates §45 scores
213
- # exactly what it always did.
214
- w = default_weights() if weights is None else weights
215
-
216
- # B9. base.py's Literals are ANNOTATIONS. An unrecognised status used
217
- # to take GROUNDING_WEIGHTS' 0.0 default AND slip past the
218
- # forced-UNVERIFIED rule below, which compares == "ungrounded" -- so a
219
- # claim the model said it could not source was published as merely
220
- # LOW. Coerced to the CONSERVATIVE end, which is the one that cannot
221
- # overstate confidence.
222
- status = claim.grounding_status
223
- unrecognised_status = None
224
- if status is not None and status not in GROUNDING_STATUSES:
225
- unrecognised_status, status = status, "ungrounded"
226
-
227
- # Same, for the type. Factual is the conservative branch: it has no
228
- # NON_FACTUAL_SCORE_CAP floor to sit on, and it is the only branch the
229
- # forced-UNVERIFIED rule can reach. A capitalised `type` used to skip
230
- # gate D0's routing entirely and then score MEDIUM off the cap.
231
- claim_type = claim.type
232
- unrecognised_type = None
233
- if claim_type not in CLAIM_TYPES:
234
- unrecognised_type, claim_type = claim_type, "factual"
235
-
236
- # §45 (SQ3). What the grounding is WORTH, not a fourth axis beside it.
237
- # None -- no scoring stage ran for this claim -- gives a multiplier of
238
- # 1.0, which is the pre-§45 arithmetic exactly.
239
- # ROUNDED ONCE, HERE, then used for both the formula and the
240
- # breakdown -- E10's rule applied to a second derived value. The
241
- # unrounded product of two fractions records as
242
- # `grounding_component: 0.7264705882352941` in an artifact whose other
243
- # fields are all 4 decimal places, and rounding it on the way OUT
244
- # instead is exactly how E10's two values came to disagree: the score
245
- # would be computed from one number and published as another.
246
- quality = source_quality(claim, sources_scored, w["authority_full_credit"])
247
- quality_multiplier = (1.0 if quality is None
248
- else round(max(w["source_quality_floor"], quality), 4))
249
- grounding_weight = w["grounding_weights"].get(status, 0.0)
250
- grounding_component = grounding_weight * quality_multiplier
251
- critic_component = 1.0 - claim.critic_severity
252
- assumption_penalty = w["assumption_flag_penalty"] * len(claim.assumption_flags)
253
-
254
- # None when ensemble mode is off, which is what makes the formula below
255
- # ONE formula rather than a pair that can drift -- and (B7) also when
256
- # ensemble mode is ON and Pass 2 reported nothing for this claim, which
257
- # is a different fact from "no candidate asserted it".
258
- consistency_score = None
259
- consistency_reported = None
260
- disagreement_penalty = 0.0
261
- if ensemble_n > 0 and claim.asserted_by_candidates:
262
- # B8. Deduplicated and clamped. `[1, 2, 3, 3]` made
263
- # consistency 1.3333, so the PENALTY went NEGATIVE and a MEDIUM
264
- # claim scored HIGH -- the exact property E8's comment says holds
265
- # "by construction". `consistency_reported` keeps the unclamped
266
- # value when it fired, because a numerator above the denominator is
267
- # itself evidence that Pass 2 misread the candidate labelling, and
268
- # clamping silently would hide the evidence along with the bug.
269
- reported = len(set(claim.asserted_by_candidates)) / ensemble_n
270
- consistency_score = min(1.0, reported)
271
- if reported > 1.0:
272
- consistency_reported = round(reported, 4)
273
- # E9: computed only where it is applied, so the breakdown records
274
- # what was subtracted rather than what was available (B10).
275
- if claim_type == "factual":
276
- disagreement_penalty = w["disagreement_penalty_factor"] * (1.0 - consistency_score)
277
-
278
- if claim_type != "factual":
279
- # Cap applies to the critic component BEFORE subtracting the
280
- # assumption penalty -- capping the final score instead would let
281
- # the cap swallow the penalty whenever the pre-penalty score
282
- # already exceeded it, making assumption flags invisible on any
283
- # claim that was already clean under critique. Found this via a
284
- # direct test case (two speculative claims, one flagged one not,
285
- # both landing at the same tier) before it shipped.
286
- #
287
- # E9: no disagreement penalty here. §10 decided consistency does not
288
- # RESCUE a non-factual claim from the cap; whether dissent should
289
- # penalise a speculative claim is a separate judgment about what
290
- # candidates declining to speculate means, and it was not made.
291
- capped_critic = min(critic_component, w["non_factual_score_cap"])
292
- raw_score = capped_critic - assumption_penalty
293
- formula = "non_factual"
294
- else:
295
- capped_critic = None
296
- formula = "factual"
297
- raw_score = (
298
- (w["grounding_weight_factor"] * grounding_component)
299
- + (w["critic_weight_factor"] * critic_component)
300
- - assumption_penalty
301
- - disagreement_penalty
302
- )
303
-
304
- # ROUND ONCE, then use that one value for BOTH the tier comparison and
305
- # the breakdown. These used to be two values: the tier compared the raw
306
- # float while the breakdown stored round(raw, 4), so a claim scoring
307
- # 0.7999999999999999 was recorded as `raw_score: 0.8` and tiered MEDIUM
308
- # -- against a TIER_THRESHOLDS table that says >= 0.8 is HIGH. Anyone
309
- # auditing a run artifact read a number that contradicted the published
310
- # thresholds, and the tier was the one that mattered.
311
- #
312
- # Binary floating point makes this reachable from ordinary weights, not
313
- # from an exotic input: 0.5 + 0.35 - 0.15 * (1/3) is exactly that value.
314
- # Sweeping the whole non-ensemble input space, exactly ONE shape changes
315
- # tier under this fix -- a speculative claim at severity 0.55 with one
316
- # assumption flag, which records 0.3 and now tiers LOW instead of
317
- # UNVERIFIED. That is what the threshold table always specified.
318
- raw_score = round(max(0.0, min(1.0, raw_score)), 4)
319
-
320
- if claim_type == "factual" and status == "ungrounded":
321
- tier: ConfidenceTier = "UNVERIFIED"
322
- else:
323
- tier = _tier_for(raw_score, w["tier_thresholds"])
324
-
325
- # ROADMAP_v2 §30 (B10). THE BREAKDOWN RECORDS WHAT WAS APPLIED, AND
326
- # NAMES THE BRANCH THAT APPLIED IT, so a reader can recompute raw_score
327
- # from these fields alone. It could not before: a speculative claim
328
- # asserted by 1 of 3 candidates stored
329
- # `grounding_component: 0.0, disagreement_penalty: 0.1, raw_score: 0.65`
330
- # -- and recomputing from those gives 0.25, because E9 subtracts the
331
- # penalty only in the factual branch and grounding is not in the
332
- # non-factual formula at all. Both fields were inert and neither said
333
- # so.
334
- #
335
- # Same class as E10 on the same object, which is why it is worth the
336
- # extra key: "anyone auditing a run artifact read a number that
337
- # contradicted the published thresholds."
338
- #
339
- # factual 0.5*grounding + 0.35*critic - assumption - disagreement
340
- # non_factual capped_critic - assumption
341
- breakdown = {
342
- "formula": formula,
343
- "critic_component": critic_component,
344
- "assumption_penalty": assumption_penalty,
345
- # Already rounded above -- rounding again here is what let the two
346
- # values drift apart in the first place.
347
- "raw_score": raw_score,
348
- }
349
- if formula == "factual":
350
- breakdown["grounding_component"] = grounding_component
351
- # B10's property, extended: the breakdown alone must let a reader
352
- # recompute grounding_component. Written ONLY on the factual
353
- # branch and only when scoring ran, so a field that did nothing is
354
- # never present -- which is the exact defect B10 records for
355
- # `grounding_component` and `disagreement_penalty` on a
356
- # speculative claim.
357
- if quality is not None:
358
- breakdown["grounding_weight"] = grounding_weight
359
- breakdown["source_quality"] = round(quality, 4)
360
- breakdown["source_quality_multiplier"] = quality_multiplier
361
- breakdown["source_count"] = len(claim.grounding_sources) \
362
- if isinstance(claim.grounding_sources, list) else 0
363
- else:
364
- breakdown["capped_critic_component"] = capped_critic
365
- if ensemble_n > 0:
366
- # E12: the DENOMINATOR, so a stored claim is self-describing on a run
367
- # where the roster shrank. Without it, "consistency 0.5" is
368
- # ambiguous between one of two candidates and two of four, and the
369
- # roster that produced it lives only in the trace.
370
- breakdown["consistency_denominator"] = ensemble_n
371
- # B7. None means PASS 2 REPORTED NOTHING, which is not 0.0 -- and
372
- # the two were indistinguishable, so a claim one candidate raised
373
- # uncontested took the maximum 0.15 penalty rather than the 0.05 it
374
- # earned. #12 reads this field; it needs to be able to tell them
375
- # apart too.
376
- breakdown["consistency_score"] = (
377
- None if consistency_score is None else round(consistency_score, 4))
378
- breakdown["disagreement_penalty"] = round(disagreement_penalty, 4)
379
- if consistency_reported is not None:
380
- breakdown["consistency_reported"] = consistency_reported
381
- if unrecognised_status is not None:
382
- breakdown["unrecognised_grounding_status"] = unrecognised_status
383
- if unrecognised_type is not None:
384
- breakdown["unrecognised_type"] = unrecognised_type
385
- return tier, breakdown
386
-
387
-
388
- def build_coverage_gap_entries(completeness: dict) -> list[dict]:
389
- """Gaps from Pass 3c become separate coverage-gap entries, not Claim
390
- objects -- they were never asserted, so there's nothing to re-tier."""
391
- return [{"gap": gap, "tier": "UNVERIFIED_COVERAGE"} for gap in completeness.get("gaps", [])]
392
-
393
-
394
- def run_confidence_tiering(
395
- run: PipelineRun, ensemble_n: int = 0,
396
- claims: list[Claim] | None = None,
397
- sources_scored: bool = False,
398
- weights: dict | None = None,
399
- ) -> None:
400
- """The actual Pass 5 entry point the orchestrator calls. Mutates
401
- the claims in place; zero LLM calls. Pass ensemble_n > 0 to use
402
- the ensemble formula variant (ROADMAP §10).
403
-
404
- `claims` narrows WHICH claims are re-tiered, and defaults to all of
405
- them -- which is Pass 5 itself, where every claim is in play. The
406
- 6a/6c loop passes its current batch instead: it calls this once per
407
- round, and re-tiering the whole run there recomputes claims that are
408
- already finished. A claim D2 had exhausted got its tier restored from
409
- unchanged score data while the fallback annotation _apply_fallback
410
- wrote stayed put, so it shipped as (say) LOW beside a note saying it
411
- could not be verified -- and a claim D1 never flagged could be
412
- re-tiered by any round that touched its grounding.
413
-
414
- `run.coverage_gaps` is rebuilt either way: it is a pure function of
415
- run.completeness, which does not change across the loop, so the
416
- narrowed call leaves it exactly as the full one would.
417
- """
418
- for claim in (run.claims if claims is None else claims):
419
- tier, breakdown = score_claim(claim, ensemble_n=ensemble_n,
420
- sources_scored=sources_scored,
421
- weights=weights)
422
- claim.confidence_tier = tier
423
- claim.score_breakdown = breakdown
424
- _report_coercions(run, claim, breakdown, ensemble_n)
425
-
426
- run.coverage_gaps = build_coverage_gap_entries(run.completeness)
427
-
428
-
429
- def _report_coercions(run: PipelineRun, claim: Claim, breakdown: dict,
430
- ensemble_n: int) -> None:
431
- """ROADMAP_v2 §30. Pass 5 makes no model call, so a value it had to
432
- correct has nowhere else to surface.
433
-
434
- Every line here describes something that should have been impossible:
435
- the payload boundary rejects each of these at the pass that sent it and
436
- re-asks. Reaching Pass 5 means the value survived §3's retries, or the
437
- claim did not come through Pass 2 at all. `run.trace` is the artifact
438
- base.py calls as trustworthy as the report -- silently scoring around a
439
- bad value is precisely what #75 measured, so the correction is stated
440
- even though it is also the safe one.
441
- """
442
- if "consistency_reported" in breakdown:
443
- run.log(
444
- f"Pass 5: claim {claim.id} was asserted by candidates "
445
- f"{claim.asserted_by_candidates} against a denominator of "
446
- f"{ensemble_n} -- consistency clamped to 1.0. A numerator above "
447
- f"the denominator means Pass 2 misread the candidate labelling."
448
- )
449
- if ensemble_n > 0 and breakdown.get("consistency_score") is None:
450
- run.log(
451
- f"Pass 5: claim {claim.id} carries no asserted_by_candidates on "
452
- f"an ensemble run -- scored WITHOUT a disagreement penalty, "
453
- f"since an unreported numerator is not a numerator of zero."
454
- )
455
- if "unrecognised_grounding_status" in breakdown:
456
- run.log(
457
- f"Pass 5: claim {claim.id} has grounding_status="
458
- f"{breakdown['unrecognised_grounding_status']!r}, which is not "
459
- f"one of {', '.join(sorted(GROUNDING_STATUSES))} -- scored as "
460
- f"ungrounded."
461
- )
462
- if "unrecognised_type" in breakdown:
463
- run.log(
464
- f"Pass 5: claim {claim.id} has type="
465
- f"{breakdown['unrecognised_type']!r}, which is not one of "
466
- f"{', '.join(sorted(CLAIM_TYPES))} -- scored on the factual "
467
- f"formula."
468
- )
1
+ """
2
+ core/reasoning/confidence_scoring.py
3
+
4
+ Pass 5 -- confidence tiering. ZERO LLM calls, by design: this is a
5
+ deterministic, documented weighted formula over already-computed data
6
+ (grounding status, critic severity, assumption flags). Kept as its own
7
+ module, separate from orchestrator.py, so the weights can be tuned later
8
+ without touching any prompt or any sequencing logic.
9
+
10
+ Interpretation worth flagging: Pass 3c's "gaps" (things the response
11
+ never addressed at all) are NOT claims that exist and failed
12
+ verification -- they're an absence. Rather than forcing them into the
13
+ Claim model, they're represented separately as coverage_gaps, each
14
+ tagged UNVERIFIED_COVERAGE. This is my reading of what that tier is for;
15
+ flag it if you intended something different.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ from core.reasoning.base import (
21
+ CLAIM_TYPES,
22
+ GROUNDING_STATUSES,
23
+ Claim,
24
+ ConfidenceTier,
25
+ PipelineRun,
26
+ )
27
+
28
+ # --- Tunable weights -- change these, not the logic below, to retune scoring ---
29
+ GROUNDING_WEIGHTS = {"grounded": 1.0, "partial": 0.5, "ungrounded": 0.0, None: 0.0}
30
+ GROUNDING_WEIGHT_FACTOR = 0.5
31
+ CRITIC_WEIGHT_FACTOR = 0.35
32
+ ASSUMPTION_FLAG_PENALTY = 0.15 # per flag
33
+ NON_FACTUAL_SCORE_CAP = 0.65 # claims never grounded can't reach HIGH
34
+
35
+ # ROADMAP_v3 §90 (A85, issue #12). Cross-candidate disagreement is ROUTED,
36
+ # NOT SCORED. A factual claim asserted by at most this share of the
37
+ # candidates is `contested`, and D1 sends it to the 6a/6b loop to be
38
+ # checked; nothing is subtracted for it.
39
+ #
40
+ # It was a penalty before (§10's revisit, E8): 0.15 * (1 - consistency),
41
+ # itself the repair of §10's first shape, a fourth weighted term that made
42
+ # grounding count for less whenever ensemble mode was on. Two things were
43
+ # wrong with the penalty, and the second was measured:
44
+ #
45
+ # 1. It answered the wrong question. Disagreement is the one signal in
46
+ # the pipeline that points at a specific claim and says "the models do
47
+ # not concur here", which is what a verification pass wants as input,
48
+ # and a deduction spends it on a number instead.
49
+ # 2. It could send a claim into the loop that no revision could get out
50
+ # of. A partially grounded claim scores 0.60 (MEDIUM); at 1 of 3 the
51
+ # penalty took it to 0.50 (LOW), so D1 flagged it, 6a was never told
52
+ # why, and 6b re-scored it under the same penalty -- who asserted a
53
+ # claim does not change when it is reworded -- until D2 published it
54
+ # UNVERIFIED.
55
+ #
56
+ # AT MOST HALF, so 1 of 2, 1 of 3 and 2 of 4 are contested and 2 of 3 is
57
+ # not: exactly the shapes where the penalty used to change a tier. The
58
+ # non-ensemble formula is untouched, and the ensemble one is now the same
59
+ # formula, so §10's byte-for-byte regression guard holds for both.
60
+ CONTESTED_SHARE = 0.5
61
+
62
+ # ROADMAP_v2 §45 (SQ3). What a grounding is WORTH.
63
+ #
64
+ # MULTIPLICATIVE, NOT A FOURTH TERM, and that is the whole design. §10
65
+ # shipped a fourth weighted term for consistency and reversed it (see
66
+ # E8 above): redistributing 0.5/0.35 to make room meant grounding counted
67
+ # for less whenever the feature was on, and the new term could only ever
68
+ # subtract. Scaling the grounding component instead says what is actually
69
+ # meant -- how much is this grounding worth -- and leaves the non-scored
70
+ # formula LITERALLY unchanged rather than coincidentally equal, so the
71
+ # byte-for-byte regression guard holds by construction here too.
72
+ #
73
+ # BOTH CONSTANTS BELOW WERE SET BY MEASUREMENT, not by taste, and the
74
+ # first draft of each was wrong in a way only a sweep showed.
75
+ #
76
+ # AUTHORITY_FULL_CREDIT exists because the domain table is a RANKING and
77
+ # this formula needs an EVIDENTIAL scale. Feeding `authority` in raw
78
+ # multiplied two rank-scales together and landed everything in the middle:
79
+ # swept across the whole table, only a `.gov` at similarity 1.0 still
80
+ # reached HIGH, and Nature, arXiv, Reddit and a Pinterest pin were
81
+ # indistinguishable at MEDIUM -- a tier that no longer discriminates,
82
+ # which is the opposite of the point. At or above a peer-reviewed
83
+ # publisher, authority is not what limits a claim's confidence, so that is
84
+ # where full credit starts.
85
+ #
86
+ # SOURCE_QUALITY_FLOOR was 0.4 in the first draft, and 0.4 is exactly the
87
+ # worst available value: 0.5*0.4 + 0.35 == 0.55, which IS the MEDIUM
88
+ # threshold, so the floor guaranteed MEDIUM for every grounded claim
89
+ # however bad its sources were. At 0.25 a claim grounded only in a content
90
+ # farm lands LOW -- flagged, and sent to the 6a/6b loop to find something
91
+ # better, which is the measured Pinterest case getting the treatment it
92
+ # always should have.
93
+ #
94
+ # NOT ZERO, though: a weak-but-real grounding is still better evidenced
95
+ # than one nothing supports, and the forced-UNVERIFIED rule below already
96
+ # handles the latter.
97
+ AUTHORITY_FULL_CREDIT = 0.85
98
+ SOURCE_QUALITY_FLOOR = 0.25
99
+
100
+
101
+ TIER_THRESHOLDS = [
102
+ (0.8, "HIGH"),
103
+ (0.55, "MEDIUM"),
104
+ (0.3, "LOW"),
105
+ ] # anything below the lowest threshold, or an ungrounded factual claim, is UNVERIFIED
106
+
107
+
108
+ # ROADMAP_v2 §45 (SQ6). The constants above are the DEFAULTS; a run may
109
+ # carry its own, resolved once at pipeline start from settings.json by
110
+ # core.config_loader.effective_confidence.
111
+ #
112
+ # THE PARAMETER DEFAULTS TO THESE, which is what keeps every existing
113
+ # caller -- and the byte-for-byte regression tests this file is built
114
+ # around -- scoring exactly what they always did. A retuned run is an
115
+ # explicit act at one call site, never a silent re-reading of global
116
+ # state inside a pure function.
117
+ #
118
+ # Built by a FUNCTION rather than bound at import, so a test or a tool
119
+ # that patches one of the constants above still sees its own value here.
120
+ def default_weights() -> dict:
121
+ return {
122
+ "grounding_weight_factor": GROUNDING_WEIGHT_FACTOR,
123
+ "critic_weight_factor": CRITIC_WEIGHT_FACTOR,
124
+ "assumption_flag_penalty": ASSUMPTION_FLAG_PENALTY,
125
+ "non_factual_score_cap": NON_FACTUAL_SCORE_CAP,
126
+ "source_quality_floor": SOURCE_QUALITY_FLOOR,
127
+ "authority_full_credit": AUTHORITY_FULL_CREDIT,
128
+ "grounding_weights": dict(GROUNDING_WEIGHTS),
129
+ "tier_thresholds": {name: value for value, name in TIER_THRESHOLDS},
130
+ }
131
+
132
+
133
+ def _tier_for(raw_score: float, thresholds: dict) -> str:
134
+ """The tier `raw_score` earns, highest first.
135
+
136
+ SORTED HERE rather than trusting the mapping's order: the resolver
137
+ already refuses a table that does not descend, and sorting again means
138
+ a caller passing one by hand cannot make a tier unreachable by
139
+ writing its keys in the wrong order.
140
+ """
141
+ for name, threshold in sorted(thresholds.items(), key=lambda kv: -kv[1]):
142
+ if raw_score >= threshold:
143
+ return name
144
+ return "UNVERIFIED"
145
+
146
+
147
+ def source_quality(claim: Claim, scored: bool,
148
+ full_credit: float = None):
149
+ """The best source this claim has, as authority x similarity, or None.
150
+
151
+ A PRODUCT, not an average (§45, SQ3). An authoritative source that
152
+ does not discuss the claim and a perfect match on a content farm are
153
+ both worth nothing, and only multiplication says so. MAX across
154
+ sources, because grounding asks whether ANY credible source supports
155
+ the claim -- a noisy-OR that rewarded corroboration would let five
156
+ mediocre blogs outscore one primary source.
157
+
158
+ `scored` IS THE WHOLE SAFETY PROPERTY, and it is passed rather than
159
+ inferred. An empty source list is ambiguous on its own: it means "the
160
+ model claimed this was grounded and cited nothing" on a run that went
161
+ through §45's scoring stage, and it means "this claim was built
162
+ without one" everywhere else -- a test, a §20 correction, a resume.
163
+ Reading 0.0 from the second would re-tier claims nothing has measured,
164
+ which is exactly the silent rescoring this module's own history is
165
+ about. So None (leave the formula alone) is the answer whenever the
166
+ caller has not said scoring ran.
167
+ """
168
+ if not scored:
169
+ return None
170
+ if full_credit is None:
171
+ full_credit = AUTHORITY_FULL_CREDIT
172
+ sources = claim.grounding_sources
173
+ if not isinstance(sources, list):
174
+ return 0.0
175
+ values = []
176
+ for source in sources:
177
+ if not isinstance(source, dict):
178
+ continue
179
+ authority = source.get("authority_score")
180
+ relevance = source.get("similarity_score")
181
+ if any(isinstance(v, bool) or not isinstance(v, (int, float))
182
+ for v in (authority, relevance)):
183
+ continue
184
+ # Authority is normalised against AUTHORITY_FULL_CREDIT before the
185
+ # product: the table ranks source types, and a rank multiplied by
186
+ # a rank is not evidence. Above the reference it is capped rather
187
+ # than allowed to compensate for weak relevance -- a government
188
+ # site that does not discuss the claim is still not evidence for
189
+ # it, which is the property the product exists to express.
190
+ credit = min(1.0, max(0.0, float(authority)) / full_credit)
191
+ values.append(credit * min(1.0, max(0.0, float(relevance))))
192
+ # 0.0 rather than None for a claim with no scorable source at all:
193
+ # scoring DID run, and it found nothing behind a claim the model
194
+ # asserted. source_scoring already says so in the trace; this is the
195
+ # same finding reaching the score.
196
+ return max(values) if values else 0.0
197
+
198
+
199
+ def score_claim(claim: Claim, ensemble_n: int = 0,
200
+ sources_scored: bool = False,
201
+ weights: dict | None = None) -> tuple[ConfidenceTier, dict]:
202
+ """Returns (tier, score_breakdown). Pure function -- no side effects,
203
+ easy to unit test claim-by-claim without running any pass.
204
+
205
+ When ensemble_n > 0 (ROADMAP §10), the share of candidates that
206
+ asserted this claim is recorded, and a factual claim asserted by at
207
+ most CONTESTED_SHARE of them is marked `contested` for D1 to route
208
+ (ROADMAP_v3 §90). It changes no score. ensemble_n is the number of
209
+ candidates that actually produced text, which on a run where a
210
+ candidate's provider failed is fewer than the configured roster.
211
+
212
+ ROADMAP_v2 §30 (B8, B9, B10) hardened all three of this function's
213
+ model-supplied inputs. Every coercion below is a LAST RESORT: the
214
+ payload boundary rejects each of these values at the pass that sent
215
+ them, and re-asks. What reaches here is a value that survived §3's
216
+ corrective retries, or a claim built by some path that never went
217
+ through Pass 2 -- and silently scoring it wrong is the outcome #75
218
+ measured.
219
+ """
220
+ # §45 (SQ6). Resolved weights or this module's own defaults -- and
221
+ # None is the default, so every caller that predates §45 scores
222
+ # exactly what it always did.
223
+ w = default_weights() if weights is None else weights
224
+
225
+ # B9. base.py's Literals are ANNOTATIONS. An unrecognised status used
226
+ # to take GROUNDING_WEIGHTS' 0.0 default AND slip past the
227
+ # forced-UNVERIFIED rule below, which compares == "ungrounded" -- so a
228
+ # claim the model said it could not source was published as merely
229
+ # LOW. Coerced to the CONSERVATIVE end, which is the one that cannot
230
+ # overstate confidence.
231
+ status = claim.grounding_status
232
+ unrecognised_status = None
233
+ if status is not None and status not in GROUNDING_STATUSES:
234
+ unrecognised_status, status = status, "ungrounded"
235
+
236
+ # Same, for the type. Factual is the conservative branch: it has no
237
+ # NON_FACTUAL_SCORE_CAP floor to sit on, and it is the only branch the
238
+ # forced-UNVERIFIED rule can reach. A capitalised `type` used to skip
239
+ # gate D0's routing entirely and then score MEDIUM off the cap.
240
+ claim_type = claim.type
241
+ unrecognised_type = None
242
+ if claim_type not in CLAIM_TYPES:
243
+ unrecognised_type, claim_type = claim_type, "factual"
244
+
245
+ # §45 (SQ3). What the grounding is WORTH, not a fourth axis beside it.
246
+ # None -- no scoring stage ran for this claim -- gives a multiplier of
247
+ # 1.0, which is the pre-§45 arithmetic exactly.
248
+ # ROUNDED ONCE, HERE, then used for both the formula and the
249
+ # breakdown -- E10's rule applied to a second derived value. The
250
+ # unrounded product of two fractions records as
251
+ # `grounding_component: 0.7264705882352941` in an artifact whose other
252
+ # fields are all 4 decimal places, and rounding it on the way OUT
253
+ # instead is exactly how E10's two values came to disagree: the score
254
+ # would be computed from one number and published as another.
255
+ quality = source_quality(claim, sources_scored, w["authority_full_credit"])
256
+ quality_multiplier = (1.0 if quality is None
257
+ else round(max(w["source_quality_floor"], quality), 4))
258
+ grounding_weight = w["grounding_weights"].get(status, 0.0)
259
+ grounding_component = grounding_weight * quality_multiplier
260
+ critic_component = 1.0 - claim.critic_severity
261
+ assumption_penalty = w["assumption_flag_penalty"] * len(claim.assumption_flags)
262
+
263
+ # None when ensemble mode is off -- and (B7) also when ensemble mode is
264
+ # ON and Pass 2 reported nothing for this claim, which is a different
265
+ # fact from "no candidate asserted it". An unreported claim is never
266
+ # contested.
267
+ consistency_score = None
268
+ consistency_reported = None
269
+ if ensemble_n > 0 and claim.asserted_by_candidates:
270
+ # B8. Deduplicated and clamped. `[1, 2, 3, 3]` made consistency
271
+ # 1.3333, which under E8 made the penalty NEGATIVE and scored a
272
+ # MEDIUM claim HIGH; here a duplicate would make one candidate's
273
+ # claim read as two agreeing, and `[1, 1]` of 3 would escape
274
+ # routing. `consistency_reported` keeps the unclamped value when
275
+ # it fired, because a numerator above the denominator is itself
276
+ # evidence that Pass 2 misread the candidate labelling, and
277
+ # clamping silently would hide the evidence along with the bug.
278
+ reported = len(set(claim.asserted_by_candidates)) / ensemble_n
279
+ consistency_score = min(1.0, reported)
280
+ if reported > 1.0:
281
+ consistency_reported = round(reported, 4)
282
+
283
+ if claim_type != "factual":
284
+ # Cap applies to the critic component BEFORE subtracting the
285
+ # assumption penalty -- capping the final score instead would let
286
+ # the cap swallow the penalty whenever the pre-penalty score
287
+ # already exceeded it, making assumption flags invisible on any
288
+ # claim that was already clean under critique. Found this via a
289
+ # direct test case (two speculative claims, one flagged one not,
290
+ # both landing at the same tier) before it shipped.
291
+ capped_critic = min(critic_component, w["non_factual_score_cap"])
292
+ raw_score = capped_critic - assumption_penalty
293
+ formula = "non_factual"
294
+ else:
295
+ capped_critic = None
296
+ formula = "factual"
297
+ raw_score = (
298
+ (w["grounding_weight_factor"] * grounding_component)
299
+ + (w["critic_weight_factor"] * critic_component)
300
+ - assumption_penalty
301
+ )
302
+
303
+ # ROUND ONCE, then use that one value for BOTH the tier comparison and
304
+ # the breakdown. These used to be two values: the tier compared the raw
305
+ # float while the breakdown stored round(raw, 4), so a claim scoring
306
+ # 0.7999999999999999 was recorded as `raw_score: 0.8` and tiered MEDIUM
307
+ # -- against a TIER_THRESHOLDS table that says >= 0.8 is HIGH. Anyone
308
+ # auditing a run artifact read a number that contradicted the published
309
+ # thresholds, and the tier was the one that mattered.
310
+ #
311
+ # Binary floating point made this reachable from ordinary weights, not
312
+ # from an exotic input: 0.5 + 0.35 - 0.15 * (1/3), E8's 2-of-3 penalty,
313
+ # was exactly that value.
314
+ # Sweeping the whole non-ensemble input space, exactly ONE shape changes
315
+ # tier under this fix -- a speculative claim at severity 0.55 with one
316
+ # assumption flag, which records 0.3 and now tiers LOW instead of
317
+ # UNVERIFIED. That is what the threshold table always specified.
318
+ raw_score = round(max(0.0, min(1.0, raw_score)), 4)
319
+
320
+ if claim_type == "factual" and status == "ungrounded":
321
+ tier: ConfidenceTier = "UNVERIFIED"
322
+ else:
323
+ tier = _tier_for(raw_score, w["tier_thresholds"])
324
+
325
+ # ROADMAP_v2 §30 (B10). THE BREAKDOWN RECORDS WHAT WAS APPLIED, AND
326
+ # NAMES THE BRANCH THAT APPLIED IT, so a reader can recompute raw_score
327
+ # from these fields alone. It could not before: a speculative claim
328
+ # asserted by 1 of 3 candidates stored
329
+ # `grounding_component: 0.0, disagreement_penalty: 0.1, raw_score: 0.65`
330
+ # -- and recomputing from those gives 0.25, because E9 subtracts the
331
+ # penalty only in the factual branch and grounding is not in the
332
+ # non-factual formula at all. Both fields were inert and neither said
333
+ # so.
334
+ #
335
+ # Same class as E10 on the same object, which is why it is worth the
336
+ # extra key: "anyone auditing a run artifact read a number that
337
+ # contradicted the published thresholds."
338
+ #
339
+ # factual 0.5*grounding + 0.35*critic - assumption
340
+ # non_factual capped_critic - assumption
341
+ breakdown = {
342
+ "formula": formula,
343
+ "critic_component": critic_component,
344
+ "assumption_penalty": assumption_penalty,
345
+ # Already rounded above -- rounding again here is what let the two
346
+ # values drift apart in the first place.
347
+ "raw_score": raw_score,
348
+ }
349
+ if formula == "factual":
350
+ breakdown["grounding_component"] = grounding_component
351
+ # B10's property, extended: the breakdown alone must let a reader
352
+ # recompute grounding_component. Written ONLY on the factual
353
+ # branch and only when scoring ran, so a field that did nothing is
354
+ # never present -- which is the exact defect B10 records for
355
+ # `grounding_component` and `disagreement_penalty` on a
356
+ # speculative claim.
357
+ if quality is not None:
358
+ breakdown["grounding_weight"] = grounding_weight
359
+ breakdown["source_quality"] = round(quality, 4)
360
+ breakdown["source_quality_multiplier"] = quality_multiplier
361
+ breakdown["source_count"] = len(claim.grounding_sources) \
362
+ if isinstance(claim.grounding_sources, list) else 0
363
+ else:
364
+ breakdown["capped_critic_component"] = capped_critic
365
+ if ensemble_n > 0:
366
+ # E12: the DENOMINATOR, so a stored claim is self-describing on a run
367
+ # where the roster shrank. Without it, "consistency 0.5" is
368
+ # ambiguous between one of two candidates and two of four, and the
369
+ # roster that produced it lives only in the trace.
370
+ breakdown["consistency_denominator"] = ensemble_n
371
+ # B7. None means PASS 2 REPORTED NOTHING, which is not 0.0 -- and
372
+ # the two were indistinguishable, so a claim one candidate raised
373
+ # uncontested took the maximum 0.15 penalty rather than the 0.05 it
374
+ # earned under E8. `contested` needs them apart for the same reason.
375
+ breakdown["consistency_score"] = (
376
+ None if consistency_score is None else round(consistency_score, 4))
377
+ # §90: what D1 routes on, and the record of why a claim that scored
378
+ # HIGH went through 6a/6b anyway. E9 LIVES HERE, and only here:
379
+ # whether candidates declining to speculate says anything about a
380
+ # speculative claim is a judgment §10 never made, so the non-factual
381
+ # branch is never asked and carries no key (B10: a field that did
382
+ # nothing is never present).
383
+ if formula == "factual":
384
+ breakdown["contested"] = (consistency_score is not None
385
+ and consistency_score <= CONTESTED_SHARE)
386
+ if consistency_reported is not None:
387
+ breakdown["consistency_reported"] = consistency_reported
388
+ if unrecognised_status is not None:
389
+ breakdown["unrecognised_grounding_status"] = unrecognised_status
390
+ if unrecognised_type is not None:
391
+ breakdown["unrecognised_type"] = unrecognised_type
392
+ return tier, breakdown
393
+
394
+
395
+ def is_contested(claim: Claim) -> bool:
396
+ """Whether Pass 5 marked this claim contested (ROADMAP_v3 §90).
397
+
398
+ Read off the breakdown rather than recomputed, so D1 routes on exactly
399
+ what the stored claim says and the type coercion above happens once."""
400
+ return claim.score_breakdown.get("contested") is True
401
+
402
+
403
+ def build_coverage_gap_entries(completeness: dict) -> list[dict]:
404
+ """Gaps from Pass 3c become separate coverage-gap entries, not Claim
405
+ objects -- they were never asserted, so there's nothing to re-tier."""
406
+ return [{"gap": gap, "tier": "UNVERIFIED_COVERAGE"} for gap in completeness.get("gaps", [])]
407
+
408
+
409
+ def run_confidence_tiering(
410
+ run: PipelineRun, ensemble_n: int = 0,
411
+ claims: list[Claim] | None = None,
412
+ sources_scored: bool = False,
413
+ weights: dict | None = None,
414
+ ) -> None:
415
+ """The actual Pass 5 entry point the orchestrator calls. Mutates
416
+ the claims in place; zero LLM calls. Pass ensemble_n > 0 to record
417
+ each claim's consistency and mark the contested ones (ROADMAP §10,
418
+ ROADMAP_v3 §90); the formula is the same either way.
419
+
420
+ `claims` narrows WHICH claims are re-tiered, and defaults to all of
421
+ them -- which is Pass 5 itself, where every claim is in play. The
422
+ 6a/6c loop passes its current batch instead: it calls this once per
423
+ round, and re-tiering the whole run there recomputes claims that are
424
+ already finished. A claim D2 had exhausted got its tier restored from
425
+ unchanged score data while the fallback annotation _apply_fallback
426
+ wrote stayed put, so it shipped as (say) LOW beside a note saying it
427
+ could not be verified -- and a claim D1 never flagged could be
428
+ re-tiered by any round that touched its grounding.
429
+
430
+ `run.coverage_gaps` is rebuilt either way: it is a pure function of
431
+ run.completeness, which does not change across the loop, so the
432
+ narrowed call leaves it exactly as the full one would.
433
+ """
434
+ for claim in (run.claims if claims is None else claims):
435
+ tier, breakdown = score_claim(claim, ensemble_n=ensemble_n,
436
+ sources_scored=sources_scored,
437
+ weights=weights)
438
+ claim.confidence_tier = tier
439
+ claim.score_breakdown = breakdown
440
+ _report_coercions(run, claim, breakdown, ensemble_n)
441
+
442
+ run.coverage_gaps = build_coverage_gap_entries(run.completeness)
443
+
444
+
445
+ def _report_coercions(run: PipelineRun, claim: Claim, breakdown: dict,
446
+ ensemble_n: int) -> None:
447
+ """ROADMAP_v2 §30. Pass 5 makes no model call, so a value it had to
448
+ correct has nowhere else to surface.
449
+
450
+ Every line here describes something that should have been impossible:
451
+ the payload boundary rejects each of these at the pass that sent it and
452
+ re-asks. Reaching Pass 5 means the value survived §3's retries, or the
453
+ claim did not come through Pass 2 at all. `run.trace` is the artifact
454
+ base.py calls as trustworthy as the report -- silently scoring around a
455
+ bad value is precisely what #75 measured, so the correction is stated
456
+ even though it is also the safe one.
457
+ """
458
+ if "consistency_reported" in breakdown:
459
+ run.log(
460
+ f"Pass 5: claim {claim.id} was asserted by candidates "
461
+ f"{claim.asserted_by_candidates} against a denominator of "
462
+ f"{ensemble_n} -- consistency clamped to 1.0. A numerator above "
463
+ f"the denominator means Pass 2 misread the candidate labelling."
464
+ )
465
+ if ensemble_n > 0 and breakdown.get("consistency_score") is None:
466
+ run.log(
467
+ f"Pass 5: claim {claim.id} carries no asserted_by_candidates on "
468
+ f"an ensemble run -- not treated as contested, since an "
469
+ f"unreported numerator is not a numerator of zero."
470
+ )
471
+ if "unrecognised_grounding_status" in breakdown:
472
+ run.log(
473
+ f"Pass 5: claim {claim.id} has grounding_status="
474
+ f"{breakdown['unrecognised_grounding_status']!r}, which is not "
475
+ f"one of {', '.join(sorted(GROUNDING_STATUSES))} -- scored as "
476
+ f"ungrounded."
477
+ )
478
+ if "unrecognised_type" in breakdown:
479
+ run.log(
480
+ f"Pass 5: claim {claim.id} has type="
481
+ f"{breakdown['unrecognised_type']!r}, which is not one of "
482
+ f"{', '.join(sorted(CLAIM_TYPES))} -- scored on the factual "
483
+ f"formula."
484
+ )