@yahyahammad/venastine 0.3.8 → 0.3.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/.env.example +19 -19
  2. package/CONFIG_ARCHITECTURE.md +2139 -2002
  3. package/LICENSE +201 -201
  4. package/README.md +1043 -1034
  5. package/agents/builtin/compactor.md +75 -75
  6. package/agents/builtin/explore.md +77 -77
  7. package/agents/builtin/pipeline-reviewer.md +100 -100
  8. package/agents/builtin/plan.md +106 -106
  9. package/agents/builtin/review.md +74 -74
  10. package/agents/manager.py +261 -261
  11. package/agents/subagent_tool.py +472 -472
  12. package/agents/tui_commands.py +238 -146
  13. package/bin/venastine.mjs +584 -528
  14. package/bin/venastine.py +73 -71
  15. package/cli_commands.py +848 -835
  16. package/config.yaml +64 -0
  17. package/config_edit.py +1301 -1301
  18. package/config_schema.py +987 -966
  19. package/config_update.py +749 -749
  20. package/core/agent_activity.py +256 -256
  21. package/core/ansi.py +452 -452
  22. package/core/approval.py +137 -137
  23. package/core/call_view.py +364 -364
  24. package/core/cancel.py +121 -121
  25. package/core/chatgpt_auth.py +747 -747
  26. package/core/claims_view.py +107 -107
  27. package/core/client.py +2756 -2715
  28. package/core/commands.py +464 -454
  29. package/core/compaction.py +1735 -1735
  30. package/core/config_loader.py +1982 -1982
  31. package/core/connect.py +686 -686
  32. package/core/events.py +123 -123
  33. package/core/file_mentions.py +280 -280
  34. package/core/interaction.py +411 -411
  35. package/core/loop.py +2566 -2566
  36. package/core/memory.py +482 -482
  37. package/core/pipeline_models.py +190 -190
  38. package/core/provider_errors.py +274 -234
  39. package/core/reasoning/authorization.py +130 -130
  40. package/core/reasoning/base.py +219 -219
  41. package/core/reasoning/confidence_scoring.py +484 -484
  42. package/core/reasoning/events.py +149 -149
  43. package/core/reasoning/orchestrator.py +1561 -1561
  44. package/core/reasoning/output_writer.py +345 -345
  45. package/core/reasoning/payload_validation.py +524 -524
  46. package/core/reasoning/pipeline_storage.py +427 -427
  47. package/core/reasoning/scholar.py +467 -458
  48. package/core/reasoning/source_corpus.py +433 -433
  49. package/core/reasoning/source_scoring.py +855 -855
  50. package/core/replay.py +315 -315
  51. package/core/shell_sessions.py +1553 -1553
  52. package/credentials.py +307 -307
  53. package/database.py +333 -333
  54. package/docs/PRIVACY.md +78 -78
  55. package/docs/SECURITY.md +114 -108
  56. package/docs/THIRD_PARTY_NOTICES.md +148 -148
  57. package/env_secrets.py +118 -118
  58. package/json_store.py +343 -343
  59. package/logging_setup.py +295 -295
  60. package/main.py +3433 -3433
  61. package/mcp_client/client.py +780 -733
  62. package/mcp_client/config.py +484 -484
  63. package/mcp_client/registration.py +343 -343
  64. package/memories/manager.py +147 -147
  65. package/memories/tui_commands.py +319 -239
  66. package/package.json +303 -303
  67. package/project_init/config_files.py +282 -282
  68. package/project_init/generator.py +728 -728
  69. package/project_init/manifest.py +418 -418
  70. package/project_init/tui_commands.py +147 -147
  71. package/prompts/critic_pass.md +24 -24
  72. package/prompts/revalidate.md +25 -25
  73. package/prompts/revise.md +17 -17
  74. package/prompts/source_grounding.md +66 -66
  75. package/prompts/system_prompts.py +316 -316
  76. package/providers.json.example +94 -94
  77. package/pyproject.toml +149 -149
  78. package/requirements.txt +314 -295
  79. package/safety/policy_enforcement.py +1082 -1082
  80. package/security/askpass.py +419 -419
  81. package/security/capability.py +207 -207
  82. package/security/confine.py +484 -484
  83. package/security/hosted.py +182 -182
  84. package/security/permissions.py +237 -237
  85. package/security/protected_paths.py +501 -501
  86. package/security/sandbox.py +4108 -4108
  87. package/skills/builtin/math/formal-specification.md +67 -67
  88. package/skills/builtin/math/numerical-methods.md +66 -66
  89. package/skills/builtin/research/experiment-design.md +62 -62
  90. package/skills/builtin/research/reproducibility.md +55 -55
  91. package/skills/builtin/research/source-evaluation.md +70 -70
  92. package/skills/builtin/research/statistical-inference.md +63 -63
  93. package/skills/builtin/research/technical-writing.md +59 -59
  94. package/skills/builtin/software/code-review.md +61 -61
  95. package/skills/builtin/software/debugging.md +61 -61
  96. package/skills/builtin/software/test-design.md +67 -67
  97. package/skills/tui_commands.py +110 -110
  98. package/storage.py +1338 -1338
  99. package/tools/base.py +465 -465
  100. package/tools/builtin/_math_common.py +667 -667
  101. package/tools/builtin/_net_common.py +1145 -267
  102. package/tools/builtin/_page_text.py +490 -0
  103. package/tools/builtin/_pdf_text.py +113 -0
  104. package/tools/builtin/arxiv.py +685 -681
  105. package/tools/builtin/fetch_url.py +639 -371
  106. package/tools/builtin/file_ops.py +720 -720
  107. package/tools/builtin/geometry.py +184 -184
  108. package/tools/builtin/load_skill.py +76 -76
  109. package/tools/builtin/pin.py +227 -227
  110. package/tools/builtin/probability_stats.py +224 -224
  111. package/tools/builtin/project_docs.py +464 -464
  112. package/tools/builtin/remember.py +175 -175
  113. package/tools/builtin/shell.py +980 -980
  114. package/tools/builtin/todo.py +149 -149
  115. package/tools/builtin/web_search.py +402 -328
  116. package/tools/context.py +159 -159
  117. package/tools/isolation.py +387 -387
  118. package/tools/registry.py +1252 -1242
  119. package/tui/app.py +8372 -8307
  120. package/tui/app.tcss +754 -739
  121. package/tui/commands.py +26 -26
  122. package/tui/diffs.py +286 -286
  123. package/tui/history.py +128 -128
  124. package/tui/hosted_driver.py +102 -102
  125. package/tui/markdown.py +1329 -1329
  126. package/tui/meters.py +364 -364
  127. package/tui/ravens.py +146 -146
  128. package/tui/screens.py +1255 -1197
  129. package/tui/themes.py +599 -599
  130. package/tui/widgets.py +3861 -3796
@@ -1,484 +1,484 @@
1
- """
2
- core/reasoning/confidence_scoring.py
3
-
4
- Pass 5 -- confidence tiering. ZERO LLM calls, by design: this is a
5
- deterministic, documented weighted formula over already-computed data
6
- (grounding status, critic severity, assumption flags). Kept as its own
7
- module, separate from orchestrator.py, so the weights can be tuned later
8
- without touching any prompt or any sequencing logic.
9
-
10
- Interpretation worth flagging: Pass 3c's "gaps" (things the response
11
- never addressed at all) are NOT claims that exist and failed
12
- verification -- they're an absence. Rather than forcing them into the
13
- Claim model, they're represented separately as coverage_gaps, each
14
- tagged UNVERIFIED_COVERAGE. This is my reading of what that tier is for;
15
- flag it if you intended something different.
16
- """
17
-
18
- from __future__ import annotations
19
-
20
- from core.reasoning.base import (
21
- CLAIM_TYPES,
22
- GROUNDING_STATUSES,
23
- Claim,
24
- ConfidenceTier,
25
- PipelineRun,
26
- )
27
-
28
- # --- Tunable weights -- change these, not the logic below, to retune scoring ---
29
- GROUNDING_WEIGHTS = {"grounded": 1.0, "partial": 0.5, "ungrounded": 0.0, None: 0.0}
30
- GROUNDING_WEIGHT_FACTOR = 0.5
31
- CRITIC_WEIGHT_FACTOR = 0.35
32
- ASSUMPTION_FLAG_PENALTY = 0.15 # per flag
33
- NON_FACTUAL_SCORE_CAP = 0.65 # claims never grounded can't reach HIGH
34
-
35
- # ROADMAP_v3 §90 (A85, issue #12). Cross-candidate disagreement is ROUTED,
36
- # NOT SCORED. A factual claim asserted by at most this share of the
37
- # candidates is `contested`, and D1 sends it to the 6a/6b loop to be
38
- # checked; nothing is subtracted for it.
39
- #
40
- # It was a penalty before (§10's revisit, E8): 0.15 * (1 - consistency),
41
- # itself the repair of §10's first shape, a fourth weighted term that made
42
- # grounding count for less whenever ensemble mode was on. Two things were
43
- # wrong with the penalty, and the second was measured:
44
- #
45
- # 1. It answered the wrong question. Disagreement is the one signal in
46
- # the pipeline that points at a specific claim and says "the models do
47
- # not concur here", which is what a verification pass wants as input,
48
- # and a deduction spends it on a number instead.
49
- # 2. It could send a claim into the loop that no revision could get out
50
- # of. A partially grounded claim scores 0.60 (MEDIUM); at 1 of 3 the
51
- # penalty took it to 0.50 (LOW), so D1 flagged it, 6a was never told
52
- # why, and 6b re-scored it under the same penalty -- who asserted a
53
- # claim does not change when it is reworded -- until D2 published it
54
- # UNVERIFIED.
55
- #
56
- # AT MOST HALF, so 1 of 2, 1 of 3 and 2 of 4 are contested and 2 of 3 is
57
- # not: exactly the shapes where the penalty used to change a tier. The
58
- # non-ensemble formula is untouched, and the ensemble one is now the same
59
- # formula, so §10's byte-for-byte regression guard holds for both.
60
- CONTESTED_SHARE = 0.5
61
-
62
- # ROADMAP_v2 §45 (SQ3). What a grounding is WORTH.
63
- #
64
- # MULTIPLICATIVE, NOT A FOURTH TERM, and that is the whole design. §10
65
- # shipped a fourth weighted term for consistency and reversed it (see
66
- # E8 above): redistributing 0.5/0.35 to make room meant grounding counted
67
- # for less whenever the feature was on, and the new term could only ever
68
- # subtract. Scaling the grounding component instead says what is actually
69
- # meant -- how much is this grounding worth -- and leaves the non-scored
70
- # formula LITERALLY unchanged rather than coincidentally equal, so the
71
- # byte-for-byte regression guard holds by construction here too.
72
- #
73
- # BOTH CONSTANTS BELOW WERE SET BY MEASUREMENT, not by taste, and the
74
- # first draft of each was wrong in a way only a sweep showed.
75
- #
76
- # AUTHORITY_FULL_CREDIT exists because the domain table is a RANKING and
77
- # this formula needs an EVIDENTIAL scale. Feeding `authority` in raw
78
- # multiplied two rank-scales together and landed everything in the middle:
79
- # swept across the whole table, only a `.gov` at similarity 1.0 still
80
- # reached HIGH, and Nature, arXiv, Reddit and a Pinterest pin were
81
- # indistinguishable at MEDIUM -- a tier that no longer discriminates,
82
- # which is the opposite of the point. At or above a peer-reviewed
83
- # publisher, authority is not what limits a claim's confidence, so that is
84
- # where full credit starts.
85
- #
86
- # SOURCE_QUALITY_FLOOR was 0.4 in the first draft, and 0.4 is exactly the
87
- # worst available value: 0.5*0.4 + 0.35 == 0.55, which IS the MEDIUM
88
- # threshold, so the floor guaranteed MEDIUM for every grounded claim
89
- # however bad its sources were. At 0.25 a claim grounded only in a content
90
- # farm lands LOW -- flagged, and sent to the 6a/6b loop to find something
91
- # better, which is the measured Pinterest case getting the treatment it
92
- # always should have.
93
- #
94
- # NOT ZERO, though: a weak-but-real grounding is still better evidenced
95
- # than one nothing supports, and the forced-UNVERIFIED rule below already
96
- # handles the latter.
97
- AUTHORITY_FULL_CREDIT = 0.85
98
- SOURCE_QUALITY_FLOOR = 0.25
99
-
100
-
101
- TIER_THRESHOLDS = [
102
- (0.8, "HIGH"),
103
- (0.55, "MEDIUM"),
104
- (0.3, "LOW"),
105
- ] # anything below the lowest threshold, or an ungrounded factual claim, is UNVERIFIED
106
-
107
-
108
- # ROADMAP_v2 §45 (SQ6). The constants above are the DEFAULTS; a run may
109
- # carry its own, resolved once at pipeline start from settings.json by
110
- # core.config_loader.effective_confidence.
111
- #
112
- # THE PARAMETER DEFAULTS TO THESE, which is what keeps every existing
113
- # caller -- and the byte-for-byte regression tests this file is built
114
- # around -- scoring exactly what they always did. A retuned run is an
115
- # explicit act at one call site, never a silent re-reading of global
116
- # state inside a pure function.
117
- #
118
- # Built by a FUNCTION rather than bound at import, so a test or a tool
119
- # that patches one of the constants above still sees its own value here.
120
- def default_weights() -> dict:
121
- return {
122
- "grounding_weight_factor": GROUNDING_WEIGHT_FACTOR,
123
- "critic_weight_factor": CRITIC_WEIGHT_FACTOR,
124
- "assumption_flag_penalty": ASSUMPTION_FLAG_PENALTY,
125
- "non_factual_score_cap": NON_FACTUAL_SCORE_CAP,
126
- "source_quality_floor": SOURCE_QUALITY_FLOOR,
127
- "authority_full_credit": AUTHORITY_FULL_CREDIT,
128
- "grounding_weights": dict(GROUNDING_WEIGHTS),
129
- "tier_thresholds": {name: value for value, name in TIER_THRESHOLDS},
130
- }
131
-
132
-
133
- def _tier_for(raw_score: float, thresholds: dict) -> str:
134
- """The tier `raw_score` earns, highest first.
135
-
136
- SORTED HERE rather than trusting the mapping's order: the resolver
137
- already refuses a table that does not descend, and sorting again means
138
- a caller passing one by hand cannot make a tier unreachable by
139
- writing its keys in the wrong order.
140
- """
141
- for name, threshold in sorted(thresholds.items(), key=lambda kv: -kv[1]):
142
- if raw_score >= threshold:
143
- return name
144
- return "UNVERIFIED"
145
-
146
-
147
- def source_quality(claim: Claim, scored: bool,
148
- full_credit: float = None):
149
- """The best source this claim has, as authority x similarity, or None.
150
-
151
- A PRODUCT, not an average (§45, SQ3). An authoritative source that
152
- does not discuss the claim and a perfect match on a content farm are
153
- both worth nothing, and only multiplication says so. MAX across
154
- sources, because grounding asks whether ANY credible source supports
155
- the claim -- a noisy-OR that rewarded corroboration would let five
156
- mediocre blogs outscore one primary source.
157
-
158
- `scored` IS THE WHOLE SAFETY PROPERTY, and it is passed rather than
159
- inferred. An empty source list is ambiguous on its own: it means "the
160
- model claimed this was grounded and cited nothing" on a run that went
161
- through §45's scoring stage, and it means "this claim was built
162
- without one" everywhere else -- a test, a §20 correction, a resume.
163
- Reading 0.0 from the second would re-tier claims nothing has measured,
164
- which is exactly the silent rescoring this module's own history is
165
- about. So None (leave the formula alone) is the answer whenever the
166
- caller has not said scoring ran.
167
- """
168
- if not scored:
169
- return None
170
- if full_credit is None:
171
- full_credit = AUTHORITY_FULL_CREDIT
172
- sources = claim.grounding_sources
173
- if not isinstance(sources, list):
174
- return 0.0
175
- values = []
176
- for source in sources:
177
- if not isinstance(source, dict):
178
- continue
179
- authority = source.get("authority_score")
180
- relevance = source.get("similarity_score")
181
- if any(isinstance(v, bool) or not isinstance(v, (int, float))
182
- for v in (authority, relevance)):
183
- continue
184
- # Authority is normalised against AUTHORITY_FULL_CREDIT before the
185
- # product: the table ranks source types, and a rank multiplied by
186
- # a rank is not evidence. Above the reference it is capped rather
187
- # than allowed to compensate for weak relevance -- a government
188
- # site that does not discuss the claim is still not evidence for
189
- # it, which is the property the product exists to express.
190
- credit = min(1.0, max(0.0, float(authority)) / full_credit)
191
- values.append(credit * min(1.0, max(0.0, float(relevance))))
192
- # 0.0 rather than None for a claim with no scorable source at all:
193
- # scoring DID run, and it found nothing behind a claim the model
194
- # asserted. source_scoring already says so in the trace; this is the
195
- # same finding reaching the score.
196
- return max(values) if values else 0.0
197
-
198
-
199
- def score_claim(claim: Claim, ensemble_n: int = 0,
200
- sources_scored: bool = False,
201
- weights: dict | None = None) -> tuple[ConfidenceTier, dict]:
202
- """Returns (tier, score_breakdown). Pure function -- no side effects,
203
- easy to unit test claim-by-claim without running any pass.
204
-
205
- When ensemble_n > 0 (ROADMAP §10), the share of candidates that
206
- asserted this claim is recorded, and a factual claim asserted by at
207
- most CONTESTED_SHARE of them is marked `contested` for D1 to route
208
- (ROADMAP_v3 §90). It changes no score. ensemble_n is the number of
209
- candidates that actually produced text, which on a run where a
210
- candidate's provider failed is fewer than the configured roster.
211
-
212
- ROADMAP_v2 §30 (B8, B9, B10) hardened all three of this function's
213
- model-supplied inputs. Every coercion below is a LAST RESORT: the
214
- payload boundary rejects each of these values at the pass that sent
215
- them, and re-asks. What reaches here is a value that survived §3's
216
- corrective retries, or a claim built by some path that never went
217
- through Pass 2 -- and silently scoring it wrong is the outcome #75
218
- measured.
219
- """
220
- # §45 (SQ6). Resolved weights or this module's own defaults -- and
221
- # None is the default, so every caller that predates §45 scores
222
- # exactly what it always did.
223
- w = default_weights() if weights is None else weights
224
-
225
- # B9. base.py's Literals are ANNOTATIONS. An unrecognised status used
226
- # to take GROUNDING_WEIGHTS' 0.0 default AND slip past the
227
- # forced-UNVERIFIED rule below, which compares == "ungrounded" -- so a
228
- # claim the model said it could not source was published as merely
229
- # LOW. Coerced to the CONSERVATIVE end, which is the one that cannot
230
- # overstate confidence.
231
- status = claim.grounding_status
232
- unrecognised_status = None
233
- if status is not None and status not in GROUNDING_STATUSES:
234
- unrecognised_status, status = status, "ungrounded"
235
-
236
- # Same, for the type. Factual is the conservative branch: it has no
237
- # NON_FACTUAL_SCORE_CAP floor to sit on, and it is the only branch the
238
- # forced-UNVERIFIED rule can reach. A capitalised `type` used to skip
239
- # gate D0's routing entirely and then score MEDIUM off the cap.
240
- claim_type = claim.type
241
- unrecognised_type = None
242
- if claim_type not in CLAIM_TYPES:
243
- unrecognised_type, claim_type = claim_type, "factual"
244
-
245
- # §45 (SQ3). What the grounding is WORTH, not a fourth axis beside it.
246
- # None -- no scoring stage ran for this claim -- gives a multiplier of
247
- # 1.0, which is the pre-§45 arithmetic exactly.
248
- # ROUNDED ONCE, HERE, then used for both the formula and the
249
- # breakdown -- E10's rule applied to a second derived value. The
250
- # unrounded product of two fractions records as
251
- # `grounding_component: 0.7264705882352941` in an artifact whose other
252
- # fields are all 4 decimal places, and rounding it on the way OUT
253
- # instead is exactly how E10's two values came to disagree: the score
254
- # would be computed from one number and published as another.
255
- quality = source_quality(claim, sources_scored, w["authority_full_credit"])
256
- quality_multiplier = (1.0 if quality is None
257
- else round(max(w["source_quality_floor"], quality), 4))
258
- grounding_weight = w["grounding_weights"].get(status, 0.0)
259
- grounding_component = grounding_weight * quality_multiplier
260
- critic_component = 1.0 - claim.critic_severity
261
- assumption_penalty = w["assumption_flag_penalty"] * len(claim.assumption_flags)
262
-
263
- # None when ensemble mode is off -- and (B7) also when ensemble mode is
264
- # ON and Pass 2 reported nothing for this claim, which is a different
265
- # fact from "no candidate asserted it". An unreported claim is never
266
- # contested.
267
- consistency_score = None
268
- consistency_reported = None
269
- if ensemble_n > 0 and claim.asserted_by_candidates:
270
- # B8. Deduplicated and clamped. `[1, 2, 3, 3]` made consistency
271
- # 1.3333, which under E8 made the penalty NEGATIVE and scored a
272
- # MEDIUM claim HIGH; here a duplicate would make one candidate's
273
- # claim read as two agreeing, and `[1, 1]` of 3 would escape
274
- # routing. `consistency_reported` keeps the unclamped value when
275
- # it fired, because a numerator above the denominator is itself
276
- # evidence that Pass 2 misread the candidate labelling, and
277
- # clamping silently would hide the evidence along with the bug.
278
- reported = len(set(claim.asserted_by_candidates)) / ensemble_n
279
- consistency_score = min(1.0, reported)
280
- if reported > 1.0:
281
- consistency_reported = round(reported, 4)
282
-
283
- if claim_type != "factual":
284
- # Cap applies to the critic component BEFORE subtracting the
285
- # assumption penalty -- capping the final score instead would let
286
- # the cap swallow the penalty whenever the pre-penalty score
287
- # already exceeded it, making assumption flags invisible on any
288
- # claim that was already clean under critique. Found this via a
289
- # direct test case (two speculative claims, one flagged one not,
290
- # both landing at the same tier) before it shipped.
291
- capped_critic = min(critic_component, w["non_factual_score_cap"])
292
- raw_score = capped_critic - assumption_penalty
293
- formula = "non_factual"
294
- else:
295
- capped_critic = None
296
- formula = "factual"
297
- raw_score = (
298
- (w["grounding_weight_factor"] * grounding_component)
299
- + (w["critic_weight_factor"] * critic_component)
300
- - assumption_penalty
301
- )
302
-
303
- # ROUND ONCE, then use that one value for BOTH the tier comparison and
304
- # the breakdown. These used to be two values: the tier compared the raw
305
- # float while the breakdown stored round(raw, 4), so a claim scoring
306
- # 0.7999999999999999 was recorded as `raw_score: 0.8` and tiered MEDIUM
307
- # -- against a TIER_THRESHOLDS table that says >= 0.8 is HIGH. Anyone
308
- # auditing a run artifact read a number that contradicted the published
309
- # thresholds, and the tier was the one that mattered.
310
- #
311
- # Binary floating point made this reachable from ordinary weights, not
312
- # from an exotic input: 0.5 + 0.35 - 0.15 * (1/3), E8's 2-of-3 penalty,
313
- # was exactly that value.
314
- # Sweeping the whole non-ensemble input space, exactly ONE shape changes
315
- # tier under this fix -- a speculative claim at severity 0.55 with one
316
- # assumption flag, which records 0.3 and now tiers LOW instead of
317
- # UNVERIFIED. That is what the threshold table always specified.
318
- raw_score = round(max(0.0, min(1.0, raw_score)), 4)
319
-
320
- if claim_type == "factual" and status == "ungrounded":
321
- tier: ConfidenceTier = "UNVERIFIED"
322
- else:
323
- tier = _tier_for(raw_score, w["tier_thresholds"])
324
-
325
- # ROADMAP_v2 §30 (B10). THE BREAKDOWN RECORDS WHAT WAS APPLIED, AND
326
- # NAMES THE BRANCH THAT APPLIED IT, so a reader can recompute raw_score
327
- # from these fields alone. It could not before: a speculative claim
328
- # asserted by 1 of 3 candidates stored
329
- # `grounding_component: 0.0, disagreement_penalty: 0.1, raw_score: 0.65`
330
- # -- and recomputing from those gives 0.25, because E9 subtracts the
331
- # penalty only in the factual branch and grounding is not in the
332
- # non-factual formula at all. Both fields were inert and neither said
333
- # so.
334
- #
335
- # Same class as E10 on the same object, which is why it is worth the
336
- # extra key: "anyone auditing a run artifact read a number that
337
- # contradicted the published thresholds."
338
- #
339
- # factual 0.5*grounding + 0.35*critic - assumption
340
- # non_factual capped_critic - assumption
341
- breakdown = {
342
- "formula": formula,
343
- "critic_component": critic_component,
344
- "assumption_penalty": assumption_penalty,
345
- # Already rounded above -- rounding again here is what let the two
346
- # values drift apart in the first place.
347
- "raw_score": raw_score,
348
- }
349
- if formula == "factual":
350
- breakdown["grounding_component"] = grounding_component
351
- # B10's property, extended: the breakdown alone must let a reader
352
- # recompute grounding_component. Written ONLY on the factual
353
- # branch and only when scoring ran, so a field that did nothing is
354
- # never present -- which is the exact defect B10 records for
355
- # `grounding_component` and `disagreement_penalty` on a
356
- # speculative claim.
357
- if quality is not None:
358
- breakdown["grounding_weight"] = grounding_weight
359
- breakdown["source_quality"] = round(quality, 4)
360
- breakdown["source_quality_multiplier"] = quality_multiplier
361
- breakdown["source_count"] = len(claim.grounding_sources) \
362
- if isinstance(claim.grounding_sources, list) else 0
363
- else:
364
- breakdown["capped_critic_component"] = capped_critic
365
- if ensemble_n > 0:
366
- # E12: the DENOMINATOR, so a stored claim is self-describing on a run
367
- # where the roster shrank. Without it, "consistency 0.5" is
368
- # ambiguous between one of two candidates and two of four, and the
369
- # roster that produced it lives only in the trace.
370
- breakdown["consistency_denominator"] = ensemble_n
371
- # B7. None means PASS 2 REPORTED NOTHING, which is not 0.0 -- and
372
- # the two were indistinguishable, so a claim one candidate raised
373
- # uncontested took the maximum 0.15 penalty rather than the 0.05 it
374
- # earned under E8. `contested` needs them apart for the same reason.
375
- breakdown["consistency_score"] = (
376
- None if consistency_score is None else round(consistency_score, 4))
377
- # §90: what D1 routes on, and the record of why a claim that scored
378
- # HIGH went through 6a/6b anyway. E9 LIVES HERE, and only here:
379
- # whether candidates declining to speculate says anything about a
380
- # speculative claim is a judgment §10 never made, so the non-factual
381
- # branch is never asked and carries no key (B10: a field that did
382
- # nothing is never present).
383
- if formula == "factual":
384
- breakdown["contested"] = (consistency_score is not None
385
- and consistency_score <= CONTESTED_SHARE)
386
- if consistency_reported is not None:
387
- breakdown["consistency_reported"] = consistency_reported
388
- if unrecognised_status is not None:
389
- breakdown["unrecognised_grounding_status"] = unrecognised_status
390
- if unrecognised_type is not None:
391
- breakdown["unrecognised_type"] = unrecognised_type
392
- return tier, breakdown
393
-
394
-
395
- def is_contested(claim: Claim) -> bool:
396
- """Whether Pass 5 marked this claim contested (ROADMAP_v3 §90).
397
-
398
- Read off the breakdown rather than recomputed, so D1 routes on exactly
399
- what the stored claim says and the type coercion above happens once."""
400
- return claim.score_breakdown.get("contested") is True
401
-
402
-
403
- def build_coverage_gap_entries(completeness: dict) -> list[dict]:
404
- """Gaps from Pass 3c become separate coverage-gap entries, not Claim
405
- objects -- they were never asserted, so there's nothing to re-tier."""
406
- return [{"gap": gap, "tier": "UNVERIFIED_COVERAGE"} for gap in completeness.get("gaps", [])]
407
-
408
-
409
- def run_confidence_tiering(
410
- run: PipelineRun, ensemble_n: int = 0,
411
- claims: list[Claim] | None = None,
412
- sources_scored: bool = False,
413
- weights: dict | None = None,
414
- ) -> None:
415
- """The actual Pass 5 entry point the orchestrator calls. Mutates
416
- the claims in place; zero LLM calls. Pass ensemble_n > 0 to record
417
- each claim's consistency and mark the contested ones (ROADMAP §10,
418
- ROADMAP_v3 §90); the formula is the same either way.
419
-
420
- `claims` narrows WHICH claims are re-tiered, and defaults to all of
421
- them -- which is Pass 5 itself, where every claim is in play. The
422
- 6a/6c loop passes its current batch instead: it calls this once per
423
- round, and re-tiering the whole run there recomputes claims that are
424
- already finished. A claim D2 had exhausted got its tier restored from
425
- unchanged score data while the fallback annotation _apply_fallback
426
- wrote stayed put, so it shipped as (say) LOW beside a note saying it
427
- could not be verified -- and a claim D1 never flagged could be
428
- re-tiered by any round that touched its grounding.
429
-
430
- `run.coverage_gaps` is rebuilt either way: it is a pure function of
431
- run.completeness, which does not change across the loop, so the
432
- narrowed call leaves it exactly as the full one would.
433
- """
434
- for claim in (run.claims if claims is None else claims):
435
- tier, breakdown = score_claim(claim, ensemble_n=ensemble_n,
436
- sources_scored=sources_scored,
437
- weights=weights)
438
- claim.confidence_tier = tier
439
- claim.score_breakdown = breakdown
440
- _report_coercions(run, claim, breakdown, ensemble_n)
441
-
442
- run.coverage_gaps = build_coverage_gap_entries(run.completeness)
443
-
444
-
445
- def _report_coercions(run: PipelineRun, claim: Claim, breakdown: dict,
446
- ensemble_n: int) -> None:
447
- """ROADMAP_v2 §30. Pass 5 makes no model call, so a value it had to
448
- correct has nowhere else to surface.
449
-
450
- Every line here describes something that should have been impossible:
451
- the payload boundary rejects each of these at the pass that sent it and
452
- re-asks. Reaching Pass 5 means the value survived §3's retries, or the
453
- claim did not come through Pass 2 at all. `run.trace` is the artifact
454
- base.py calls as trustworthy as the report -- silently scoring around a
455
- bad value is precisely what #75 measured, so the correction is stated
456
- even though it is also the safe one.
457
- """
458
- if "consistency_reported" in breakdown:
459
- run.log(
460
- f"Pass 5: claim {claim.id} was asserted by candidates "
461
- f"{claim.asserted_by_candidates} against a denominator of "
462
- f"{ensemble_n} -- consistency clamped to 1.0. A numerator above "
463
- f"the denominator means Pass 2 misread the candidate labelling."
464
- )
465
- if ensemble_n > 0 and breakdown.get("consistency_score") is None:
466
- run.log(
467
- f"Pass 5: claim {claim.id} carries no asserted_by_candidates on "
468
- f"an ensemble run -- not treated as contested, since an "
469
- f"unreported numerator is not a numerator of zero."
470
- )
471
- if "unrecognised_grounding_status" in breakdown:
472
- run.log(
473
- f"Pass 5: claim {claim.id} has grounding_status="
474
- f"{breakdown['unrecognised_grounding_status']!r}, which is not "
475
- f"one of {', '.join(sorted(GROUNDING_STATUSES))} -- scored as "
476
- f"ungrounded."
477
- )
478
- if "unrecognised_type" in breakdown:
479
- run.log(
480
- f"Pass 5: claim {claim.id} has type="
481
- f"{breakdown['unrecognised_type']!r}, which is not one of "
482
- f"{', '.join(sorted(CLAIM_TYPES))} -- scored on the factual "
483
- f"formula."
484
- )
1
+ """
2
+ core/reasoning/confidence_scoring.py
3
+
4
+ Pass 5 -- confidence tiering. ZERO LLM calls, by design: this is a
5
+ deterministic, documented weighted formula over already-computed data
6
+ (grounding status, critic severity, assumption flags). Kept as its own
7
+ module, separate from orchestrator.py, so the weights can be tuned later
8
+ without touching any prompt or any sequencing logic.
9
+
10
+ Interpretation worth flagging: Pass 3c's "gaps" (things the response
11
+ never addressed at all) are NOT claims that exist and failed
12
+ verification -- they're an absence. Rather than forcing them into the
13
+ Claim model, they're represented separately as coverage_gaps, each
14
+ tagged UNVERIFIED_COVERAGE. This is my reading of what that tier is for;
15
+ flag it if you intended something different.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ from core.reasoning.base import (
21
+ CLAIM_TYPES,
22
+ GROUNDING_STATUSES,
23
+ Claim,
24
+ ConfidenceTier,
25
+ PipelineRun,
26
+ )
27
+
28
+ # --- Tunable weights -- change these, not the logic below, to retune scoring ---
29
+ GROUNDING_WEIGHTS = {"grounded": 1.0, "partial": 0.5, "ungrounded": 0.0, None: 0.0}
30
+ GROUNDING_WEIGHT_FACTOR = 0.5
31
+ CRITIC_WEIGHT_FACTOR = 0.35
32
+ ASSUMPTION_FLAG_PENALTY = 0.15 # per flag
33
+ NON_FACTUAL_SCORE_CAP = 0.65 # claims never grounded can't reach HIGH
34
+
35
+ # ROADMAP_v3 §90 (A85, issue #12). Cross-candidate disagreement is ROUTED,
36
+ # NOT SCORED. A factual claim asserted by at most this share of the
37
+ # candidates is `contested`, and D1 sends it to the 6a/6b loop to be
38
+ # checked; nothing is subtracted for it.
39
+ #
40
+ # It was a penalty before (§10's revisit, E8): 0.15 * (1 - consistency),
41
+ # itself the repair of §10's first shape, a fourth weighted term that made
42
+ # grounding count for less whenever ensemble mode was on. Two things were
43
+ # wrong with the penalty, and the second was measured:
44
+ #
45
+ # 1. It answered the wrong question. Disagreement is the one signal in
46
+ # the pipeline that points at a specific claim and says "the models do
47
+ # not concur here", which is what a verification pass wants as input,
48
+ # and a deduction spends it on a number instead.
49
+ # 2. It could send a claim into the loop that no revision could get out
50
+ # of. A partially grounded claim scores 0.60 (MEDIUM); at 1 of 3 the
51
+ # penalty took it to 0.50 (LOW), so D1 flagged it, 6a was never told
52
+ # why, and 6b re-scored it under the same penalty -- who asserted a
53
+ # claim does not change when it is reworded -- until D2 published it
54
+ # UNVERIFIED.
55
+ #
56
+ # AT MOST HALF, so 1 of 2, 1 of 3 and 2 of 4 are contested and 2 of 3 is
57
+ # not: exactly the shapes where the penalty used to change a tier. The
58
+ # non-ensemble formula is untouched, and the ensemble one is now the same
59
+ # formula, so §10's byte-for-byte regression guard holds for both.
60
+ CONTESTED_SHARE = 0.5
61
+
62
+ # ROADMAP_v2 §45 (SQ3). What a grounding is WORTH.
63
+ #
64
+ # MULTIPLICATIVE, NOT A FOURTH TERM, and that is the whole design. §10
65
+ # shipped a fourth weighted term for consistency and reversed it (see
66
+ # E8 above): redistributing 0.5/0.35 to make room meant grounding counted
67
+ # for less whenever the feature was on, and the new term could only ever
68
+ # subtract. Scaling the grounding component instead says what is actually
69
+ # meant -- how much is this grounding worth -- and leaves the non-scored
70
+ # formula LITERALLY unchanged rather than coincidentally equal, so the
71
+ # byte-for-byte regression guard holds by construction here too.
72
+ #
73
+ # BOTH CONSTANTS BELOW WERE SET BY MEASUREMENT, not by taste, and the
74
+ # first draft of each was wrong in a way only a sweep showed.
75
+ #
76
+ # AUTHORITY_FULL_CREDIT exists because the domain table is a RANKING and
77
+ # this formula needs an EVIDENTIAL scale. Feeding `authority` in raw
78
+ # multiplied two rank-scales together and landed everything in the middle:
79
+ # swept across the whole table, only a `.gov` at similarity 1.0 still
80
+ # reached HIGH, and Nature, arXiv, Reddit and a Pinterest pin were
81
+ # indistinguishable at MEDIUM -- a tier that no longer discriminates,
82
+ # which is the opposite of the point. At or above a peer-reviewed
83
+ # publisher, authority is not what limits a claim's confidence, so that is
84
+ # where full credit starts.
85
+ #
86
+ # SOURCE_QUALITY_FLOOR was 0.4 in the first draft, and 0.4 is exactly the
87
+ # worst available value: 0.5*0.4 + 0.35 == 0.55, which IS the MEDIUM
88
+ # threshold, so the floor guaranteed MEDIUM for every grounded claim
89
+ # however bad its sources were. At 0.25 a claim grounded only in a content
90
+ # farm lands LOW -- flagged, and sent to the 6a/6b loop to find something
91
+ # better, which is the measured Pinterest case getting the treatment it
92
+ # always should have.
93
+ #
94
+ # NOT ZERO, though: a weak-but-real grounding is still better evidenced
95
+ # than one nothing supports, and the forced-UNVERIFIED rule below already
96
+ # handles the latter.
97
+ AUTHORITY_FULL_CREDIT = 0.85
98
+ SOURCE_QUALITY_FLOOR = 0.25
99
+
100
+
101
+ TIER_THRESHOLDS = [
102
+ (0.8, "HIGH"),
103
+ (0.55, "MEDIUM"),
104
+ (0.3, "LOW"),
105
+ ] # anything below the lowest threshold, or an ungrounded factual claim, is UNVERIFIED
106
+
107
+
108
+ # ROADMAP_v2 §45 (SQ6). The constants above are the DEFAULTS; a run may
109
+ # carry its own, resolved once at pipeline start from settings.json by
110
+ # core.config_loader.effective_confidence.
111
+ #
112
+ # THE PARAMETER DEFAULTS TO THESE, which is what keeps every existing
113
+ # caller -- and the byte-for-byte regression tests this file is built
114
+ # around -- scoring exactly what they always did. A retuned run is an
115
+ # explicit act at one call site, never a silent re-reading of global
116
+ # state inside a pure function.
117
+ #
118
+ # Built by a FUNCTION rather than bound at import, so a test or a tool
119
+ # that patches one of the constants above still sees its own value here.
120
+ def default_weights() -> dict:
121
+ return {
122
+ "grounding_weight_factor": GROUNDING_WEIGHT_FACTOR,
123
+ "critic_weight_factor": CRITIC_WEIGHT_FACTOR,
124
+ "assumption_flag_penalty": ASSUMPTION_FLAG_PENALTY,
125
+ "non_factual_score_cap": NON_FACTUAL_SCORE_CAP,
126
+ "source_quality_floor": SOURCE_QUALITY_FLOOR,
127
+ "authority_full_credit": AUTHORITY_FULL_CREDIT,
128
+ "grounding_weights": dict(GROUNDING_WEIGHTS),
129
+ "tier_thresholds": {name: value for value, name in TIER_THRESHOLDS},
130
+ }
131
+
132
+
133
+ def _tier_for(raw_score: float, thresholds: dict) -> str:
134
+ """The tier `raw_score` earns, highest first.
135
+
136
+ SORTED HERE rather than trusting the mapping's order: the resolver
137
+ already refuses a table that does not descend, and sorting again means
138
+ a caller passing one by hand cannot make a tier unreachable by
139
+ writing its keys in the wrong order.
140
+ """
141
+ for name, threshold in sorted(thresholds.items(), key=lambda kv: -kv[1]):
142
+ if raw_score >= threshold:
143
+ return name
144
+ return "UNVERIFIED"
145
+
146
+
147
+ def source_quality(claim: Claim, scored: bool,
148
+ full_credit: float = None):
149
+ """The best source this claim has, as authority x similarity, or None.
150
+
151
+ A PRODUCT, not an average (§45, SQ3). An authoritative source that
152
+ does not discuss the claim and a perfect match on a content farm are
153
+ both worth nothing, and only multiplication says so. MAX across
154
+ sources, because grounding asks whether ANY credible source supports
155
+ the claim -- a noisy-OR that rewarded corroboration would let five
156
+ mediocre blogs outscore one primary source.
157
+
158
+ `scored` IS THE WHOLE SAFETY PROPERTY, and it is passed rather than
159
+ inferred. An empty source list is ambiguous on its own: it means "the
160
+ model claimed this was grounded and cited nothing" on a run that went
161
+ through §45's scoring stage, and it means "this claim was built
162
+ without one" everywhere else -- a test, a §20 correction, a resume.
163
+ Reading 0.0 from the second would re-tier claims nothing has measured,
164
+ which is exactly the silent rescoring this module's own history is
165
+ about. So None (leave the formula alone) is the answer whenever the
166
+ caller has not said scoring ran.
167
+ """
168
+ if not scored:
169
+ return None
170
+ if full_credit is None:
171
+ full_credit = AUTHORITY_FULL_CREDIT
172
+ sources = claim.grounding_sources
173
+ if not isinstance(sources, list):
174
+ return 0.0
175
+ values = []
176
+ for source in sources:
177
+ if not isinstance(source, dict):
178
+ continue
179
+ authority = source.get("authority_score")
180
+ relevance = source.get("similarity_score")
181
+ if any(isinstance(v, bool) or not isinstance(v, (int, float))
182
+ for v in (authority, relevance)):
183
+ continue
184
+ # Authority is normalised against AUTHORITY_FULL_CREDIT before the
185
+ # product: the table ranks source types, and a rank multiplied by
186
+ # a rank is not evidence. Above the reference it is capped rather
187
+ # than allowed to compensate for weak relevance -- a government
188
+ # site that does not discuss the claim is still not evidence for
189
+ # it, which is the property the product exists to express.
190
+ credit = min(1.0, max(0.0, float(authority)) / full_credit)
191
+ values.append(credit * min(1.0, max(0.0, float(relevance))))
192
+ # 0.0 rather than None for a claim with no scorable source at all:
193
+ # scoring DID run, and it found nothing behind a claim the model
194
+ # asserted. source_scoring already says so in the trace; this is the
195
+ # same finding reaching the score.
196
+ return max(values) if values else 0.0
197
+
198
+
199
+ def score_claim(claim: Claim, ensemble_n: int = 0,
200
+ sources_scored: bool = False,
201
+ weights: dict | None = None) -> tuple[ConfidenceTier, dict]:
202
+ """Returns (tier, score_breakdown). Pure function -- no side effects,
203
+ easy to unit test claim-by-claim without running any pass.
204
+
205
+ When ensemble_n > 0 (ROADMAP §10), the share of candidates that
206
+ asserted this claim is recorded, and a factual claim asserted by at
207
+ most CONTESTED_SHARE of them is marked `contested` for D1 to route
208
+ (ROADMAP_v3 §90). It changes no score. ensemble_n is the number of
209
+ candidates that actually produced text, which on a run where a
210
+ candidate's provider failed is fewer than the configured roster.
211
+
212
+ ROADMAP_v2 §30 (B8, B9, B10) hardened all three of this function's
213
+ model-supplied inputs. Every coercion below is a LAST RESORT: the
214
+ payload boundary rejects each of these values at the pass that sent
215
+ them, and re-asks. What reaches here is a value that survived §3's
216
+ corrective retries, or a claim built by some path that never went
217
+ through Pass 2 -- and silently scoring it wrong is the outcome #75
218
+ measured.
219
+ """
220
+ # §45 (SQ6). Resolved weights or this module's own defaults -- and
221
+ # None is the default, so every caller that predates §45 scores
222
+ # exactly what it always did.
223
+ w = default_weights() if weights is None else weights
224
+
225
+ # B9. base.py's Literals are ANNOTATIONS. An unrecognised status used
226
+ # to take GROUNDING_WEIGHTS' 0.0 default AND slip past the
227
+ # forced-UNVERIFIED rule below, which compares == "ungrounded" -- so a
228
+ # claim the model said it could not source was published as merely
229
+ # LOW. Coerced to the CONSERVATIVE end, which is the one that cannot
230
+ # overstate confidence.
231
+ status = claim.grounding_status
232
+ unrecognised_status = None
233
+ if status is not None and status not in GROUNDING_STATUSES:
234
+ unrecognised_status, status = status, "ungrounded"
235
+
236
+ # Same, for the type. Factual is the conservative branch: it has no
237
+ # NON_FACTUAL_SCORE_CAP floor to sit on, and it is the only branch the
238
+ # forced-UNVERIFIED rule can reach. A capitalised `type` used to skip
239
+ # gate D0's routing entirely and then score MEDIUM off the cap.
240
+ claim_type = claim.type
241
+ unrecognised_type = None
242
+ if claim_type not in CLAIM_TYPES:
243
+ unrecognised_type, claim_type = claim_type, "factual"
244
+
245
+ # §45 (SQ3). What the grounding is WORTH, not a fourth axis beside it.
246
+ # None -- no scoring stage ran for this claim -- gives a multiplier of
247
+ # 1.0, which is the pre-§45 arithmetic exactly.
248
+ # ROUNDED ONCE, HERE, then used for both the formula and the
249
+ # breakdown -- E10's rule applied to a second derived value. The
250
+ # unrounded product of two fractions records as
251
+ # `grounding_component: 0.7264705882352941` in an artifact whose other
252
+ # fields are all 4 decimal places, and rounding it on the way OUT
253
+ # instead is exactly how E10's two values came to disagree: the score
254
+ # would be computed from one number and published as another.
255
+ quality = source_quality(claim, sources_scored, w["authority_full_credit"])
256
+ quality_multiplier = (1.0 if quality is None
257
+ else round(max(w["source_quality_floor"], quality), 4))
258
+ grounding_weight = w["grounding_weights"].get(status, 0.0)
259
+ grounding_component = grounding_weight * quality_multiplier
260
+ critic_component = 1.0 - claim.critic_severity
261
+ assumption_penalty = w["assumption_flag_penalty"] * len(claim.assumption_flags)
262
+
263
+ # None when ensemble mode is off -- and (B7) also when ensemble mode is
264
+ # ON and Pass 2 reported nothing for this claim, which is a different
265
+ # fact from "no candidate asserted it". An unreported claim is never
266
+ # contested.
267
+ consistency_score = None
268
+ consistency_reported = None
269
+ if ensemble_n > 0 and claim.asserted_by_candidates:
270
+ # B8. Deduplicated and clamped. `[1, 2, 3, 3]` made consistency
271
+ # 1.3333, which under E8 made the penalty NEGATIVE and scored a
272
+ # MEDIUM claim HIGH; here a duplicate would make one candidate's
273
+ # claim read as two agreeing, and `[1, 1]` of 3 would escape
274
+ # routing. `consistency_reported` keeps the unclamped value when
275
+ # it fired, because a numerator above the denominator is itself
276
+ # evidence that Pass 2 misread the candidate labelling, and
277
+ # clamping silently would hide the evidence along with the bug.
278
+ reported = len(set(claim.asserted_by_candidates)) / ensemble_n
279
+ consistency_score = min(1.0, reported)
280
+ if reported > 1.0:
281
+ consistency_reported = round(reported, 4)
282
+
283
+ if claim_type != "factual":
284
+ # Cap applies to the critic component BEFORE subtracting the
285
+ # assumption penalty -- capping the final score instead would let
286
+ # the cap swallow the penalty whenever the pre-penalty score
287
+ # already exceeded it, making assumption flags invisible on any
288
+ # claim that was already clean under critique. Found this via a
289
+ # direct test case (two speculative claims, one flagged one not,
290
+ # both landing at the same tier) before it shipped.
291
+ capped_critic = min(critic_component, w["non_factual_score_cap"])
292
+ raw_score = capped_critic - assumption_penalty
293
+ formula = "non_factual"
294
+ else:
295
+ capped_critic = None
296
+ formula = "factual"
297
+ raw_score = (
298
+ (w["grounding_weight_factor"] * grounding_component)
299
+ + (w["critic_weight_factor"] * critic_component)
300
+ - assumption_penalty
301
+ )
302
+
303
+ # ROUND ONCE, then use that one value for BOTH the tier comparison and
304
+ # the breakdown. These used to be two values: the tier compared the raw
305
+ # float while the breakdown stored round(raw, 4), so a claim scoring
306
+ # 0.7999999999999999 was recorded as `raw_score: 0.8` and tiered MEDIUM
307
+ # -- against a TIER_THRESHOLDS table that says >= 0.8 is HIGH. Anyone
308
+ # auditing a run artifact read a number that contradicted the published
309
+ # thresholds, and the tier was the one that mattered.
310
+ #
311
+ # Binary floating point made this reachable from ordinary weights, not
312
+ # from an exotic input: 0.5 + 0.35 - 0.15 * (1/3), E8's 2-of-3 penalty,
313
+ # was exactly that value.
314
+ # Sweeping the whole non-ensemble input space, exactly ONE shape changes
315
+ # tier under this fix -- a speculative claim at severity 0.55 with one
316
+ # assumption flag, which records 0.3 and now tiers LOW instead of
317
+ # UNVERIFIED. That is what the threshold table always specified.
318
+ raw_score = round(max(0.0, min(1.0, raw_score)), 4)
319
+
320
+ if claim_type == "factual" and status == "ungrounded":
321
+ tier: ConfidenceTier = "UNVERIFIED"
322
+ else:
323
+ tier = _tier_for(raw_score, w["tier_thresholds"])
324
+
325
+ # ROADMAP_v2 §30 (B10). THE BREAKDOWN RECORDS WHAT WAS APPLIED, AND
326
+ # NAMES THE BRANCH THAT APPLIED IT, so a reader can recompute raw_score
327
+ # from these fields alone. It could not before: a speculative claim
328
+ # asserted by 1 of 3 candidates stored
329
+ # `grounding_component: 0.0, disagreement_penalty: 0.1, raw_score: 0.65`
330
+ # -- and recomputing from those gives 0.25, because E9 subtracts the
331
+ # penalty only in the factual branch and grounding is not in the
332
+ # non-factual formula at all. Both fields were inert and neither said
333
+ # so.
334
+ #
335
+ # Same class as E10 on the same object, which is why it is worth the
336
+ # extra key: "anyone auditing a run artifact read a number that
337
+ # contradicted the published thresholds."
338
+ #
339
+ # factual 0.5*grounding + 0.35*critic - assumption
340
+ # non_factual capped_critic - assumption
341
+ breakdown = {
342
+ "formula": formula,
343
+ "critic_component": critic_component,
344
+ "assumption_penalty": assumption_penalty,
345
+ # Already rounded above -- rounding again here is what let the two
346
+ # values drift apart in the first place.
347
+ "raw_score": raw_score,
348
+ }
349
+ if formula == "factual":
350
+ breakdown["grounding_component"] = grounding_component
351
+ # B10's property, extended: the breakdown alone must let a reader
352
+ # recompute grounding_component. Written ONLY on the factual
353
+ # branch and only when scoring ran, so a field that did nothing is
354
+ # never present -- which is the exact defect B10 records for
355
+ # `grounding_component` and `disagreement_penalty` on a
356
+ # speculative claim.
357
+ if quality is not None:
358
+ breakdown["grounding_weight"] = grounding_weight
359
+ breakdown["source_quality"] = round(quality, 4)
360
+ breakdown["source_quality_multiplier"] = quality_multiplier
361
+ breakdown["source_count"] = len(claim.grounding_sources) \
362
+ if isinstance(claim.grounding_sources, list) else 0
363
+ else:
364
+ breakdown["capped_critic_component"] = capped_critic
365
+ if ensemble_n > 0:
366
+ # E12: the DENOMINATOR, so a stored claim is self-describing on a run
367
+ # where the roster shrank. Without it, "consistency 0.5" is
368
+ # ambiguous between one of two candidates and two of four, and the
369
+ # roster that produced it lives only in the trace.
370
+ breakdown["consistency_denominator"] = ensemble_n
371
+ # B7. None means PASS 2 REPORTED NOTHING, which is not 0.0 -- and
372
+ # the two were indistinguishable, so a claim one candidate raised
373
+ # uncontested took the maximum 0.15 penalty rather than the 0.05 it
374
+ # earned under E8. `contested` needs them apart for the same reason.
375
+ breakdown["consistency_score"] = (
376
+ None if consistency_score is None else round(consistency_score, 4))
377
+ # §90: what D1 routes on, and the record of why a claim that scored
378
+ # HIGH went through 6a/6b anyway. E9 LIVES HERE, and only here:
379
+ # whether candidates declining to speculate says anything about a
380
+ # speculative claim is a judgment §10 never made, so the non-factual
381
+ # branch is never asked and carries no key (B10: a field that did
382
+ # nothing is never present).
383
+ if formula == "factual":
384
+ breakdown["contested"] = (consistency_score is not None
385
+ and consistency_score <= CONTESTED_SHARE)
386
+ if consistency_reported is not None:
387
+ breakdown["consistency_reported"] = consistency_reported
388
+ if unrecognised_status is not None:
389
+ breakdown["unrecognised_grounding_status"] = unrecognised_status
390
+ if unrecognised_type is not None:
391
+ breakdown["unrecognised_type"] = unrecognised_type
392
+ return tier, breakdown
393
+
394
+
395
+ def is_contested(claim: Claim) -> bool:
396
+ """Whether Pass 5 marked this claim contested (ROADMAP_v3 §90).
397
+
398
+ Read off the breakdown rather than recomputed, so D1 routes on exactly
399
+ what the stored claim says and the type coercion above happens once."""
400
+ return claim.score_breakdown.get("contested") is True
401
+
402
+
403
+ def build_coverage_gap_entries(completeness: dict) -> list[dict]:
404
+ """Gaps from Pass 3c become separate coverage-gap entries, not Claim
405
+ objects -- they were never asserted, so there's nothing to re-tier."""
406
+ return [{"gap": gap, "tier": "UNVERIFIED_COVERAGE"} for gap in completeness.get("gaps", [])]
407
+
408
+
409
+ def run_confidence_tiering(
410
+ run: PipelineRun, ensemble_n: int = 0,
411
+ claims: list[Claim] | None = None,
412
+ sources_scored: bool = False,
413
+ weights: dict | None = None,
414
+ ) -> None:
415
+ """The actual Pass 5 entry point the orchestrator calls. Mutates
416
+ the claims in place; zero LLM calls. Pass ensemble_n > 0 to record
417
+ each claim's consistency and mark the contested ones (ROADMAP §10,
418
+ ROADMAP_v3 §90); the formula is the same either way.
419
+
420
+ `claims` narrows WHICH claims are re-tiered, and defaults to all of
421
+ them -- which is Pass 5 itself, where every claim is in play. The
422
+ 6a/6c loop passes its current batch instead: it calls this once per
423
+ round, and re-tiering the whole run there recomputes claims that are
424
+ already finished. A claim D2 had exhausted got its tier restored from
425
+ unchanged score data while the fallback annotation _apply_fallback
426
+ wrote stayed put, so it shipped as (say) LOW beside a note saying it
427
+ could not be verified -- and a claim D1 never flagged could be
428
+ re-tiered by any round that touched its grounding.
429
+
430
+ `run.coverage_gaps` is rebuilt either way: it is a pure function of
431
+ run.completeness, which does not change across the loop, so the
432
+ narrowed call leaves it exactly as the full one would.
433
+ """
434
+ for claim in (run.claims if claims is None else claims):
435
+ tier, breakdown = score_claim(claim, ensemble_n=ensemble_n,
436
+ sources_scored=sources_scored,
437
+ weights=weights)
438
+ claim.confidence_tier = tier
439
+ claim.score_breakdown = breakdown
440
+ _report_coercions(run, claim, breakdown, ensemble_n)
441
+
442
+ run.coverage_gaps = build_coverage_gap_entries(run.completeness)
443
+
444
+
445
+ def _report_coercions(run: PipelineRun, claim: Claim, breakdown: dict,
446
+ ensemble_n: int) -> None:
447
+ """ROADMAP_v2 §30. Pass 5 makes no model call, so a value it had to
448
+ correct has nowhere else to surface.
449
+
450
+ Every line here describes something that should have been impossible:
451
+ the payload boundary rejects each of these at the pass that sent it and
452
+ re-asks. Reaching Pass 5 means the value survived §3's retries, or the
453
+ claim did not come through Pass 2 at all. `run.trace` is the artifact
454
+ base.py calls as trustworthy as the report -- silently scoring around a
455
+ bad value is precisely what #75 measured, so the correction is stated
456
+ even though it is also the safe one.
457
+ """
458
+ if "consistency_reported" in breakdown:
459
+ run.log(
460
+ f"Pass 5: claim {claim.id} was asserted by candidates "
461
+ f"{claim.asserted_by_candidates} against a denominator of "
462
+ f"{ensemble_n} -- consistency clamped to 1.0. A numerator above "
463
+ f"the denominator means Pass 2 misread the candidate labelling."
464
+ )
465
+ if ensemble_n > 0 and breakdown.get("consistency_score") is None:
466
+ run.log(
467
+ f"Pass 5: claim {claim.id} carries no asserted_by_candidates on "
468
+ f"an ensemble run -- not treated as contested, since an "
469
+ f"unreported numerator is not a numerator of zero."
470
+ )
471
+ if "unrecognised_grounding_status" in breakdown:
472
+ run.log(
473
+ f"Pass 5: claim {claim.id} has grounding_status="
474
+ f"{breakdown['unrecognised_grounding_status']!r}, which is not "
475
+ f"one of {', '.join(sorted(GROUNDING_STATUSES))} -- scored as "
476
+ f"ungrounded."
477
+ )
478
+ if "unrecognised_type" in breakdown:
479
+ run.log(
480
+ f"Pass 5: claim {claim.id} has type="
481
+ f"{breakdown['unrecognised_type']!r}, which is not one of "
482
+ f"{', '.join(sorted(CLAIM_TYPES))} -- scored on the factual "
483
+ f"formula."
484
+ )