@yahyahammad/venastine 0.3.8 → 0.3.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -19
- package/CONFIG_ARCHITECTURE.md +2139 -2002
- package/LICENSE +201 -201
- package/README.md +1043 -1034
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -106
- package/agents/builtin/review.md +74 -74
- package/agents/manager.py +261 -261
- package/agents/subagent_tool.py +472 -472
- package/agents/tui_commands.py +238 -146
- package/bin/venastine.mjs +584 -528
- package/bin/venastine.py +73 -71
- package/cli_commands.py +848 -835
- package/config.yaml +64 -0
- package/config_edit.py +1301 -1301
- package/config_schema.py +987 -966
- package/config_update.py +749 -749
- package/core/agent_activity.py +256 -256
- package/core/ansi.py +452 -452
- package/core/approval.py +137 -137
- package/core/call_view.py +364 -364
- package/core/cancel.py +121 -121
- package/core/chatgpt_auth.py +747 -747
- package/core/claims_view.py +107 -107
- package/core/client.py +2756 -2715
- package/core/commands.py +464 -454
- package/core/compaction.py +1735 -1735
- package/core/config_loader.py +1982 -1982
- package/core/connect.py +686 -686
- package/core/events.py +123 -123
- package/core/file_mentions.py +280 -280
- package/core/interaction.py +411 -411
- package/core/loop.py +2566 -2566
- package/core/memory.py +482 -482
- package/core/pipeline_models.py +190 -190
- package/core/provider_errors.py +274 -234
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -484
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/orchestrator.py +1561 -1561
- package/core/reasoning/output_writer.py +345 -345
- package/core/reasoning/payload_validation.py +524 -524
- package/core/reasoning/pipeline_storage.py +427 -427
- package/core/reasoning/scholar.py +467 -458
- package/core/reasoning/source_corpus.py +433 -433
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -315
- package/core/shell_sessions.py +1553 -1553
- package/credentials.py +307 -307
- package/database.py +333 -333
- package/docs/PRIVACY.md +78 -78
- package/docs/SECURITY.md +114 -108
- package/docs/THIRD_PARTY_NOTICES.md +148 -148
- package/env_secrets.py +118 -118
- package/json_store.py +343 -343
- package/logging_setup.py +295 -295
- package/main.py +3433 -3433
- package/mcp_client/client.py +780 -733
- package/mcp_client/config.py +484 -484
- package/mcp_client/registration.py +343 -343
- package/memories/manager.py +147 -147
- package/memories/tui_commands.py +319 -239
- package/package.json +303 -303
- package/project_init/config_files.py +282 -282
- package/project_init/generator.py +728 -728
- package/project_init/manifest.py +418 -418
- package/project_init/tui_commands.py +147 -147
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +17 -17
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -316
- package/providers.json.example +94 -94
- package/pyproject.toml +149 -149
- package/requirements.txt +314 -295
- package/safety/policy_enforcement.py +1082 -1082
- package/security/askpass.py +419 -419
- package/security/capability.py +207 -207
- package/security/confine.py +484 -484
- package/security/hosted.py +182 -182
- package/security/permissions.py +237 -237
- package/security/protected_paths.py +501 -501
- package/security/sandbox.py +4108 -4108
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -110
- package/storage.py +1338 -1338
- package/tools/base.py +465 -465
- package/tools/builtin/_math_common.py +667 -667
- package/tools/builtin/_net_common.py +1145 -267
- package/tools/builtin/_page_text.py +490 -0
- package/tools/builtin/_pdf_text.py +113 -0
- package/tools/builtin/arxiv.py +685 -681
- package/tools/builtin/fetch_url.py +639 -371
- package/tools/builtin/file_ops.py +720 -720
- package/tools/builtin/geometry.py +184 -184
- package/tools/builtin/load_skill.py +76 -76
- package/tools/builtin/pin.py +227 -227
- package/tools/builtin/probability_stats.py +224 -224
- package/tools/builtin/project_docs.py +464 -464
- package/tools/builtin/remember.py +175 -175
- package/tools/builtin/shell.py +980 -980
- package/tools/builtin/todo.py +149 -149
- package/tools/builtin/web_search.py +402 -328
- package/tools/context.py +159 -159
- package/tools/isolation.py +387 -387
- package/tools/registry.py +1252 -1242
- package/tui/app.py +8372 -8307
- package/tui/app.tcss +754 -739
- package/tui/commands.py +26 -26
- package/tui/diffs.py +286 -286
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -102
- package/tui/markdown.py +1329 -1329
- package/tui/meters.py +364 -364
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -1197
- package/tui/themes.py +599 -599
- package/tui/widgets.py +3861 -3796
|
@@ -1,484 +1,484 @@
|
|
|
1
|
-
"""
|
|
2
|
-
core/reasoning/confidence_scoring.py
|
|
3
|
-
|
|
4
|
-
Pass 5 -- confidence tiering. ZERO LLM calls, by design: this is a
|
|
5
|
-
deterministic, documented weighted formula over already-computed data
|
|
6
|
-
(grounding status, critic severity, assumption flags). Kept as its own
|
|
7
|
-
module, separate from orchestrator.py, so the weights can be tuned later
|
|
8
|
-
without touching any prompt or any sequencing logic.
|
|
9
|
-
|
|
10
|
-
Interpretation worth flagging: Pass 3c's "gaps" (things the response
|
|
11
|
-
never addressed at all) are NOT claims that exist and failed
|
|
12
|
-
verification -- they're an absence. Rather than forcing them into the
|
|
13
|
-
Claim model, they're represented separately as coverage_gaps, each
|
|
14
|
-
tagged UNVERIFIED_COVERAGE. This is my reading of what that tier is for;
|
|
15
|
-
flag it if you intended something different.
|
|
16
|
-
"""
|
|
17
|
-
|
|
18
|
-
from __future__ import annotations
|
|
19
|
-
|
|
20
|
-
from core.reasoning.base import (
|
|
21
|
-
CLAIM_TYPES,
|
|
22
|
-
GROUNDING_STATUSES,
|
|
23
|
-
Claim,
|
|
24
|
-
ConfidenceTier,
|
|
25
|
-
PipelineRun,
|
|
26
|
-
)
|
|
27
|
-
|
|
28
|
-
# --- Tunable weights -- change these, not the logic below, to retune scoring ---
|
|
29
|
-
GROUNDING_WEIGHTS = {"grounded": 1.0, "partial": 0.5, "ungrounded": 0.0, None: 0.0}
|
|
30
|
-
GROUNDING_WEIGHT_FACTOR = 0.5
|
|
31
|
-
CRITIC_WEIGHT_FACTOR = 0.35
|
|
32
|
-
ASSUMPTION_FLAG_PENALTY = 0.15 # per flag
|
|
33
|
-
NON_FACTUAL_SCORE_CAP = 0.65 # claims never grounded can't reach HIGH
|
|
34
|
-
|
|
35
|
-
# ROADMAP_v3 §90 (A85, issue #12). Cross-candidate disagreement is ROUTED,
|
|
36
|
-
# NOT SCORED. A factual claim asserted by at most this share of the
|
|
37
|
-
# candidates is `contested`, and D1 sends it to the 6a/6b loop to be
|
|
38
|
-
# checked; nothing is subtracted for it.
|
|
39
|
-
#
|
|
40
|
-
# It was a penalty before (§10's revisit, E8): 0.15 * (1 - consistency),
|
|
41
|
-
# itself the repair of §10's first shape, a fourth weighted term that made
|
|
42
|
-
# grounding count for less whenever ensemble mode was on. Two things were
|
|
43
|
-
# wrong with the penalty, and the second was measured:
|
|
44
|
-
#
|
|
45
|
-
# 1. It answered the wrong question. Disagreement is the one signal in
|
|
46
|
-
# the pipeline that points at a specific claim and says "the models do
|
|
47
|
-
# not concur here", which is what a verification pass wants as input,
|
|
48
|
-
# and a deduction spends it on a number instead.
|
|
49
|
-
# 2. It could send a claim into the loop that no revision could get out
|
|
50
|
-
# of. A partially grounded claim scores 0.60 (MEDIUM); at 1 of 3 the
|
|
51
|
-
# penalty took it to 0.50 (LOW), so D1 flagged it, 6a was never told
|
|
52
|
-
# why, and 6b re-scored it under the same penalty -- who asserted a
|
|
53
|
-
# claim does not change when it is reworded -- until D2 published it
|
|
54
|
-
# UNVERIFIED.
|
|
55
|
-
#
|
|
56
|
-
# AT MOST HALF, so 1 of 2, 1 of 3 and 2 of 4 are contested and 2 of 3 is
|
|
57
|
-
# not: exactly the shapes where the penalty used to change a tier. The
|
|
58
|
-
# non-ensemble formula is untouched, and the ensemble one is now the same
|
|
59
|
-
# formula, so §10's byte-for-byte regression guard holds for both.
|
|
60
|
-
CONTESTED_SHARE = 0.5
|
|
61
|
-
|
|
62
|
-
# ROADMAP_v2 §45 (SQ3). What a grounding is WORTH.
|
|
63
|
-
#
|
|
64
|
-
# MULTIPLICATIVE, NOT A FOURTH TERM, and that is the whole design. §10
|
|
65
|
-
# shipped a fourth weighted term for consistency and reversed it (see
|
|
66
|
-
# E8 above): redistributing 0.5/0.35 to make room meant grounding counted
|
|
67
|
-
# for less whenever the feature was on, and the new term could only ever
|
|
68
|
-
# subtract. Scaling the grounding component instead says what is actually
|
|
69
|
-
# meant -- how much is this grounding worth -- and leaves the non-scored
|
|
70
|
-
# formula LITERALLY unchanged rather than coincidentally equal, so the
|
|
71
|
-
# byte-for-byte regression guard holds by construction here too.
|
|
72
|
-
#
|
|
73
|
-
# BOTH CONSTANTS BELOW WERE SET BY MEASUREMENT, not by taste, and the
|
|
74
|
-
# first draft of each was wrong in a way only a sweep showed.
|
|
75
|
-
#
|
|
76
|
-
# AUTHORITY_FULL_CREDIT exists because the domain table is a RANKING and
|
|
77
|
-
# this formula needs an EVIDENTIAL scale. Feeding `authority` in raw
|
|
78
|
-
# multiplied two rank-scales together and landed everything in the middle:
|
|
79
|
-
# swept across the whole table, only a `.gov` at similarity 1.0 still
|
|
80
|
-
# reached HIGH, and Nature, arXiv, Reddit and a Pinterest pin were
|
|
81
|
-
# indistinguishable at MEDIUM -- a tier that no longer discriminates,
|
|
82
|
-
# which is the opposite of the point. At or above a peer-reviewed
|
|
83
|
-
# publisher, authority is not what limits a claim's confidence, so that is
|
|
84
|
-
# where full credit starts.
|
|
85
|
-
#
|
|
86
|
-
# SOURCE_QUALITY_FLOOR was 0.4 in the first draft, and 0.4 is exactly the
|
|
87
|
-
# worst available value: 0.5*0.4 + 0.35 == 0.55, which IS the MEDIUM
|
|
88
|
-
# threshold, so the floor guaranteed MEDIUM for every grounded claim
|
|
89
|
-
# however bad its sources were. At 0.25 a claim grounded only in a content
|
|
90
|
-
# farm lands LOW -- flagged, and sent to the 6a/6b loop to find something
|
|
91
|
-
# better, which is the measured Pinterest case getting the treatment it
|
|
92
|
-
# always should have.
|
|
93
|
-
#
|
|
94
|
-
# NOT ZERO, though: a weak-but-real grounding is still better evidenced
|
|
95
|
-
# than one nothing supports, and the forced-UNVERIFIED rule below already
|
|
96
|
-
# handles the latter.
|
|
97
|
-
AUTHORITY_FULL_CREDIT = 0.85
|
|
98
|
-
SOURCE_QUALITY_FLOOR = 0.25
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
TIER_THRESHOLDS = [
|
|
102
|
-
(0.8, "HIGH"),
|
|
103
|
-
(0.55, "MEDIUM"),
|
|
104
|
-
(0.3, "LOW"),
|
|
105
|
-
] # anything below the lowest threshold, or an ungrounded factual claim, is UNVERIFIED
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
# ROADMAP_v2 §45 (SQ6). The constants above are the DEFAULTS; a run may
|
|
109
|
-
# carry its own, resolved once at pipeline start from settings.json by
|
|
110
|
-
# core.config_loader.effective_confidence.
|
|
111
|
-
#
|
|
112
|
-
# THE PARAMETER DEFAULTS TO THESE, which is what keeps every existing
|
|
113
|
-
# caller -- and the byte-for-byte regression tests this file is built
|
|
114
|
-
# around -- scoring exactly what they always did. A retuned run is an
|
|
115
|
-
# explicit act at one call site, never a silent re-reading of global
|
|
116
|
-
# state inside a pure function.
|
|
117
|
-
#
|
|
118
|
-
# Built by a FUNCTION rather than bound at import, so a test or a tool
|
|
119
|
-
# that patches one of the constants above still sees its own value here.
|
|
120
|
-
def default_weights() -> dict:
|
|
121
|
-
return {
|
|
122
|
-
"grounding_weight_factor": GROUNDING_WEIGHT_FACTOR,
|
|
123
|
-
"critic_weight_factor": CRITIC_WEIGHT_FACTOR,
|
|
124
|
-
"assumption_flag_penalty": ASSUMPTION_FLAG_PENALTY,
|
|
125
|
-
"non_factual_score_cap": NON_FACTUAL_SCORE_CAP,
|
|
126
|
-
"source_quality_floor": SOURCE_QUALITY_FLOOR,
|
|
127
|
-
"authority_full_credit": AUTHORITY_FULL_CREDIT,
|
|
128
|
-
"grounding_weights": dict(GROUNDING_WEIGHTS),
|
|
129
|
-
"tier_thresholds": {name: value for value, name in TIER_THRESHOLDS},
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
def _tier_for(raw_score: float, thresholds: dict) -> str:
|
|
134
|
-
"""The tier `raw_score` earns, highest first.
|
|
135
|
-
|
|
136
|
-
SORTED HERE rather than trusting the mapping's order: the resolver
|
|
137
|
-
already refuses a table that does not descend, and sorting again means
|
|
138
|
-
a caller passing one by hand cannot make a tier unreachable by
|
|
139
|
-
writing its keys in the wrong order.
|
|
140
|
-
"""
|
|
141
|
-
for name, threshold in sorted(thresholds.items(), key=lambda kv: -kv[1]):
|
|
142
|
-
if raw_score >= threshold:
|
|
143
|
-
return name
|
|
144
|
-
return "UNVERIFIED"
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
def source_quality(claim: Claim, scored: bool,
|
|
148
|
-
full_credit: float = None):
|
|
149
|
-
"""The best source this claim has, as authority x similarity, or None.
|
|
150
|
-
|
|
151
|
-
A PRODUCT, not an average (§45, SQ3). An authoritative source that
|
|
152
|
-
does not discuss the claim and a perfect match on a content farm are
|
|
153
|
-
both worth nothing, and only multiplication says so. MAX across
|
|
154
|
-
sources, because grounding asks whether ANY credible source supports
|
|
155
|
-
the claim -- a noisy-OR that rewarded corroboration would let five
|
|
156
|
-
mediocre blogs outscore one primary source.
|
|
157
|
-
|
|
158
|
-
`scored` IS THE WHOLE SAFETY PROPERTY, and it is passed rather than
|
|
159
|
-
inferred. An empty source list is ambiguous on its own: it means "the
|
|
160
|
-
model claimed this was grounded and cited nothing" on a run that went
|
|
161
|
-
through §45's scoring stage, and it means "this claim was built
|
|
162
|
-
without one" everywhere else -- a test, a §20 correction, a resume.
|
|
163
|
-
Reading 0.0 from the second would re-tier claims nothing has measured,
|
|
164
|
-
which is exactly the silent rescoring this module's own history is
|
|
165
|
-
about. So None (leave the formula alone) is the answer whenever the
|
|
166
|
-
caller has not said scoring ran.
|
|
167
|
-
"""
|
|
168
|
-
if not scored:
|
|
169
|
-
return None
|
|
170
|
-
if full_credit is None:
|
|
171
|
-
full_credit = AUTHORITY_FULL_CREDIT
|
|
172
|
-
sources = claim.grounding_sources
|
|
173
|
-
if not isinstance(sources, list):
|
|
174
|
-
return 0.0
|
|
175
|
-
values = []
|
|
176
|
-
for source in sources:
|
|
177
|
-
if not isinstance(source, dict):
|
|
178
|
-
continue
|
|
179
|
-
authority = source.get("authority_score")
|
|
180
|
-
relevance = source.get("similarity_score")
|
|
181
|
-
if any(isinstance(v, bool) or not isinstance(v, (int, float))
|
|
182
|
-
for v in (authority, relevance)):
|
|
183
|
-
continue
|
|
184
|
-
# Authority is normalised against AUTHORITY_FULL_CREDIT before the
|
|
185
|
-
# product: the table ranks source types, and a rank multiplied by
|
|
186
|
-
# a rank is not evidence. Above the reference it is capped rather
|
|
187
|
-
# than allowed to compensate for weak relevance -- a government
|
|
188
|
-
# site that does not discuss the claim is still not evidence for
|
|
189
|
-
# it, which is the property the product exists to express.
|
|
190
|
-
credit = min(1.0, max(0.0, float(authority)) / full_credit)
|
|
191
|
-
values.append(credit * min(1.0, max(0.0, float(relevance))))
|
|
192
|
-
# 0.0 rather than None for a claim with no scorable source at all:
|
|
193
|
-
# scoring DID run, and it found nothing behind a claim the model
|
|
194
|
-
# asserted. source_scoring already says so in the trace; this is the
|
|
195
|
-
# same finding reaching the score.
|
|
196
|
-
return max(values) if values else 0.0
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
def score_claim(claim: Claim, ensemble_n: int = 0,
|
|
200
|
-
sources_scored: bool = False,
|
|
201
|
-
weights: dict | None = None) -> tuple[ConfidenceTier, dict]:
|
|
202
|
-
"""Returns (tier, score_breakdown). Pure function -- no side effects,
|
|
203
|
-
easy to unit test claim-by-claim without running any pass.
|
|
204
|
-
|
|
205
|
-
When ensemble_n > 0 (ROADMAP §10), the share of candidates that
|
|
206
|
-
asserted this claim is recorded, and a factual claim asserted by at
|
|
207
|
-
most CONTESTED_SHARE of them is marked `contested` for D1 to route
|
|
208
|
-
(ROADMAP_v3 §90). It changes no score. ensemble_n is the number of
|
|
209
|
-
candidates that actually produced text, which on a run where a
|
|
210
|
-
candidate's provider failed is fewer than the configured roster.
|
|
211
|
-
|
|
212
|
-
ROADMAP_v2 §30 (B8, B9, B10) hardened all three of this function's
|
|
213
|
-
model-supplied inputs. Every coercion below is a LAST RESORT: the
|
|
214
|
-
payload boundary rejects each of these values at the pass that sent
|
|
215
|
-
them, and re-asks. What reaches here is a value that survived §3's
|
|
216
|
-
corrective retries, or a claim built by some path that never went
|
|
217
|
-
through Pass 2 -- and silently scoring it wrong is the outcome #75
|
|
218
|
-
measured.
|
|
219
|
-
"""
|
|
220
|
-
# §45 (SQ6). Resolved weights or this module's own defaults -- and
|
|
221
|
-
# None is the default, so every caller that predates §45 scores
|
|
222
|
-
# exactly what it always did.
|
|
223
|
-
w = default_weights() if weights is None else weights
|
|
224
|
-
|
|
225
|
-
# B9. base.py's Literals are ANNOTATIONS. An unrecognised status used
|
|
226
|
-
# to take GROUNDING_WEIGHTS' 0.0 default AND slip past the
|
|
227
|
-
# forced-UNVERIFIED rule below, which compares == "ungrounded" -- so a
|
|
228
|
-
# claim the model said it could not source was published as merely
|
|
229
|
-
# LOW. Coerced to the CONSERVATIVE end, which is the one that cannot
|
|
230
|
-
# overstate confidence.
|
|
231
|
-
status = claim.grounding_status
|
|
232
|
-
unrecognised_status = None
|
|
233
|
-
if status is not None and status not in GROUNDING_STATUSES:
|
|
234
|
-
unrecognised_status, status = status, "ungrounded"
|
|
235
|
-
|
|
236
|
-
# Same, for the type. Factual is the conservative branch: it has no
|
|
237
|
-
# NON_FACTUAL_SCORE_CAP floor to sit on, and it is the only branch the
|
|
238
|
-
# forced-UNVERIFIED rule can reach. A capitalised `type` used to skip
|
|
239
|
-
# gate D0's routing entirely and then score MEDIUM off the cap.
|
|
240
|
-
claim_type = claim.type
|
|
241
|
-
unrecognised_type = None
|
|
242
|
-
if claim_type not in CLAIM_TYPES:
|
|
243
|
-
unrecognised_type, claim_type = claim_type, "factual"
|
|
244
|
-
|
|
245
|
-
# §45 (SQ3). What the grounding is WORTH, not a fourth axis beside it.
|
|
246
|
-
# None -- no scoring stage ran for this claim -- gives a multiplier of
|
|
247
|
-
# 1.0, which is the pre-§45 arithmetic exactly.
|
|
248
|
-
# ROUNDED ONCE, HERE, then used for both the formula and the
|
|
249
|
-
# breakdown -- E10's rule applied to a second derived value. The
|
|
250
|
-
# unrounded product of two fractions records as
|
|
251
|
-
# `grounding_component: 0.7264705882352941` in an artifact whose other
|
|
252
|
-
# fields are all 4 decimal places, and rounding it on the way OUT
|
|
253
|
-
# instead is exactly how E10's two values came to disagree: the score
|
|
254
|
-
# would be computed from one number and published as another.
|
|
255
|
-
quality = source_quality(claim, sources_scored, w["authority_full_credit"])
|
|
256
|
-
quality_multiplier = (1.0 if quality is None
|
|
257
|
-
else round(max(w["source_quality_floor"], quality), 4))
|
|
258
|
-
grounding_weight = w["grounding_weights"].get(status, 0.0)
|
|
259
|
-
grounding_component = grounding_weight * quality_multiplier
|
|
260
|
-
critic_component = 1.0 - claim.critic_severity
|
|
261
|
-
assumption_penalty = w["assumption_flag_penalty"] * len(claim.assumption_flags)
|
|
262
|
-
|
|
263
|
-
# None when ensemble mode is off -- and (B7) also when ensemble mode is
|
|
264
|
-
# ON and Pass 2 reported nothing for this claim, which is a different
|
|
265
|
-
# fact from "no candidate asserted it". An unreported claim is never
|
|
266
|
-
# contested.
|
|
267
|
-
consistency_score = None
|
|
268
|
-
consistency_reported = None
|
|
269
|
-
if ensemble_n > 0 and claim.asserted_by_candidates:
|
|
270
|
-
# B8. Deduplicated and clamped. `[1, 2, 3, 3]` made consistency
|
|
271
|
-
# 1.3333, which under E8 made the penalty NEGATIVE and scored a
|
|
272
|
-
# MEDIUM claim HIGH; here a duplicate would make one candidate's
|
|
273
|
-
# claim read as two agreeing, and `[1, 1]` of 3 would escape
|
|
274
|
-
# routing. `consistency_reported` keeps the unclamped value when
|
|
275
|
-
# it fired, because a numerator above the denominator is itself
|
|
276
|
-
# evidence that Pass 2 misread the candidate labelling, and
|
|
277
|
-
# clamping silently would hide the evidence along with the bug.
|
|
278
|
-
reported = len(set(claim.asserted_by_candidates)) / ensemble_n
|
|
279
|
-
consistency_score = min(1.0, reported)
|
|
280
|
-
if reported > 1.0:
|
|
281
|
-
consistency_reported = round(reported, 4)
|
|
282
|
-
|
|
283
|
-
if claim_type != "factual":
|
|
284
|
-
# Cap applies to the critic component BEFORE subtracting the
|
|
285
|
-
# assumption penalty -- capping the final score instead would let
|
|
286
|
-
# the cap swallow the penalty whenever the pre-penalty score
|
|
287
|
-
# already exceeded it, making assumption flags invisible on any
|
|
288
|
-
# claim that was already clean under critique. Found this via a
|
|
289
|
-
# direct test case (two speculative claims, one flagged one not,
|
|
290
|
-
# both landing at the same tier) before it shipped.
|
|
291
|
-
capped_critic = min(critic_component, w["non_factual_score_cap"])
|
|
292
|
-
raw_score = capped_critic - assumption_penalty
|
|
293
|
-
formula = "non_factual"
|
|
294
|
-
else:
|
|
295
|
-
capped_critic = None
|
|
296
|
-
formula = "factual"
|
|
297
|
-
raw_score = (
|
|
298
|
-
(w["grounding_weight_factor"] * grounding_component)
|
|
299
|
-
+ (w["critic_weight_factor"] * critic_component)
|
|
300
|
-
- assumption_penalty
|
|
301
|
-
)
|
|
302
|
-
|
|
303
|
-
# ROUND ONCE, then use that one value for BOTH the tier comparison and
|
|
304
|
-
# the breakdown. These used to be two values: the tier compared the raw
|
|
305
|
-
# float while the breakdown stored round(raw, 4), so a claim scoring
|
|
306
|
-
# 0.7999999999999999 was recorded as `raw_score: 0.8` and tiered MEDIUM
|
|
307
|
-
# -- against a TIER_THRESHOLDS table that says >= 0.8 is HIGH. Anyone
|
|
308
|
-
# auditing a run artifact read a number that contradicted the published
|
|
309
|
-
# thresholds, and the tier was the one that mattered.
|
|
310
|
-
#
|
|
311
|
-
# Binary floating point made this reachable from ordinary weights, not
|
|
312
|
-
# from an exotic input: 0.5 + 0.35 - 0.15 * (1/3), E8's 2-of-3 penalty,
|
|
313
|
-
# was exactly that value.
|
|
314
|
-
# Sweeping the whole non-ensemble input space, exactly ONE shape changes
|
|
315
|
-
# tier under this fix -- a speculative claim at severity 0.55 with one
|
|
316
|
-
# assumption flag, which records 0.3 and now tiers LOW instead of
|
|
317
|
-
# UNVERIFIED. That is what the threshold table always specified.
|
|
318
|
-
raw_score = round(max(0.0, min(1.0, raw_score)), 4)
|
|
319
|
-
|
|
320
|
-
if claim_type == "factual" and status == "ungrounded":
|
|
321
|
-
tier: ConfidenceTier = "UNVERIFIED"
|
|
322
|
-
else:
|
|
323
|
-
tier = _tier_for(raw_score, w["tier_thresholds"])
|
|
324
|
-
|
|
325
|
-
# ROADMAP_v2 §30 (B10). THE BREAKDOWN RECORDS WHAT WAS APPLIED, AND
|
|
326
|
-
# NAMES THE BRANCH THAT APPLIED IT, so a reader can recompute raw_score
|
|
327
|
-
# from these fields alone. It could not before: a speculative claim
|
|
328
|
-
# asserted by 1 of 3 candidates stored
|
|
329
|
-
# `grounding_component: 0.0, disagreement_penalty: 0.1, raw_score: 0.65`
|
|
330
|
-
# -- and recomputing from those gives 0.25, because E9 subtracts the
|
|
331
|
-
# penalty only in the factual branch and grounding is not in the
|
|
332
|
-
# non-factual formula at all. Both fields were inert and neither said
|
|
333
|
-
# so.
|
|
334
|
-
#
|
|
335
|
-
# Same class as E10 on the same object, which is why it is worth the
|
|
336
|
-
# extra key: "anyone auditing a run artifact read a number that
|
|
337
|
-
# contradicted the published thresholds."
|
|
338
|
-
#
|
|
339
|
-
# factual 0.5*grounding + 0.35*critic - assumption
|
|
340
|
-
# non_factual capped_critic - assumption
|
|
341
|
-
breakdown = {
|
|
342
|
-
"formula": formula,
|
|
343
|
-
"critic_component": critic_component,
|
|
344
|
-
"assumption_penalty": assumption_penalty,
|
|
345
|
-
# Already rounded above -- rounding again here is what let the two
|
|
346
|
-
# values drift apart in the first place.
|
|
347
|
-
"raw_score": raw_score,
|
|
348
|
-
}
|
|
349
|
-
if formula == "factual":
|
|
350
|
-
breakdown["grounding_component"] = grounding_component
|
|
351
|
-
# B10's property, extended: the breakdown alone must let a reader
|
|
352
|
-
# recompute grounding_component. Written ONLY on the factual
|
|
353
|
-
# branch and only when scoring ran, so a field that did nothing is
|
|
354
|
-
# never present -- which is the exact defect B10 records for
|
|
355
|
-
# `grounding_component` and `disagreement_penalty` on a
|
|
356
|
-
# speculative claim.
|
|
357
|
-
if quality is not None:
|
|
358
|
-
breakdown["grounding_weight"] = grounding_weight
|
|
359
|
-
breakdown["source_quality"] = round(quality, 4)
|
|
360
|
-
breakdown["source_quality_multiplier"] = quality_multiplier
|
|
361
|
-
breakdown["source_count"] = len(claim.grounding_sources) \
|
|
362
|
-
if isinstance(claim.grounding_sources, list) else 0
|
|
363
|
-
else:
|
|
364
|
-
breakdown["capped_critic_component"] = capped_critic
|
|
365
|
-
if ensemble_n > 0:
|
|
366
|
-
# E12: the DENOMINATOR, so a stored claim is self-describing on a run
|
|
367
|
-
# where the roster shrank. Without it, "consistency 0.5" is
|
|
368
|
-
# ambiguous between one of two candidates and two of four, and the
|
|
369
|
-
# roster that produced it lives only in the trace.
|
|
370
|
-
breakdown["consistency_denominator"] = ensemble_n
|
|
371
|
-
# B7. None means PASS 2 REPORTED NOTHING, which is not 0.0 -- and
|
|
372
|
-
# the two were indistinguishable, so a claim one candidate raised
|
|
373
|
-
# uncontested took the maximum 0.15 penalty rather than the 0.05 it
|
|
374
|
-
# earned under E8. `contested` needs them apart for the same reason.
|
|
375
|
-
breakdown["consistency_score"] = (
|
|
376
|
-
None if consistency_score is None else round(consistency_score, 4))
|
|
377
|
-
# §90: what D1 routes on, and the record of why a claim that scored
|
|
378
|
-
# HIGH went through 6a/6b anyway. E9 LIVES HERE, and only here:
|
|
379
|
-
# whether candidates declining to speculate says anything about a
|
|
380
|
-
# speculative claim is a judgment §10 never made, so the non-factual
|
|
381
|
-
# branch is never asked and carries no key (B10: a field that did
|
|
382
|
-
# nothing is never present).
|
|
383
|
-
if formula == "factual":
|
|
384
|
-
breakdown["contested"] = (consistency_score is not None
|
|
385
|
-
and consistency_score <= CONTESTED_SHARE)
|
|
386
|
-
if consistency_reported is not None:
|
|
387
|
-
breakdown["consistency_reported"] = consistency_reported
|
|
388
|
-
if unrecognised_status is not None:
|
|
389
|
-
breakdown["unrecognised_grounding_status"] = unrecognised_status
|
|
390
|
-
if unrecognised_type is not None:
|
|
391
|
-
breakdown["unrecognised_type"] = unrecognised_type
|
|
392
|
-
return tier, breakdown
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
def is_contested(claim: Claim) -> bool:
|
|
396
|
-
"""Whether Pass 5 marked this claim contested (ROADMAP_v3 §90).
|
|
397
|
-
|
|
398
|
-
Read off the breakdown rather than recomputed, so D1 routes on exactly
|
|
399
|
-
what the stored claim says and the type coercion above happens once."""
|
|
400
|
-
return claim.score_breakdown.get("contested") is True
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
def build_coverage_gap_entries(completeness: dict) -> list[dict]:
|
|
404
|
-
"""Gaps from Pass 3c become separate coverage-gap entries, not Claim
|
|
405
|
-
objects -- they were never asserted, so there's nothing to re-tier."""
|
|
406
|
-
return [{"gap": gap, "tier": "UNVERIFIED_COVERAGE"} for gap in completeness.get("gaps", [])]
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
def run_confidence_tiering(
|
|
410
|
-
run: PipelineRun, ensemble_n: int = 0,
|
|
411
|
-
claims: list[Claim] | None = None,
|
|
412
|
-
sources_scored: bool = False,
|
|
413
|
-
weights: dict | None = None,
|
|
414
|
-
) -> None:
|
|
415
|
-
"""The actual Pass 5 entry point the orchestrator calls. Mutates
|
|
416
|
-
the claims in place; zero LLM calls. Pass ensemble_n > 0 to record
|
|
417
|
-
each claim's consistency and mark the contested ones (ROADMAP §10,
|
|
418
|
-
ROADMAP_v3 §90); the formula is the same either way.
|
|
419
|
-
|
|
420
|
-
`claims` narrows WHICH claims are re-tiered, and defaults to all of
|
|
421
|
-
them -- which is Pass 5 itself, where every claim is in play. The
|
|
422
|
-
6a/6c loop passes its current batch instead: it calls this once per
|
|
423
|
-
round, and re-tiering the whole run there recomputes claims that are
|
|
424
|
-
already finished. A claim D2 had exhausted got its tier restored from
|
|
425
|
-
unchanged score data while the fallback annotation _apply_fallback
|
|
426
|
-
wrote stayed put, so it shipped as (say) LOW beside a note saying it
|
|
427
|
-
could not be verified -- and a claim D1 never flagged could be
|
|
428
|
-
re-tiered by any round that touched its grounding.
|
|
429
|
-
|
|
430
|
-
`run.coverage_gaps` is rebuilt either way: it is a pure function of
|
|
431
|
-
run.completeness, which does not change across the loop, so the
|
|
432
|
-
narrowed call leaves it exactly as the full one would.
|
|
433
|
-
"""
|
|
434
|
-
for claim in (run.claims if claims is None else claims):
|
|
435
|
-
tier, breakdown = score_claim(claim, ensemble_n=ensemble_n,
|
|
436
|
-
sources_scored=sources_scored,
|
|
437
|
-
weights=weights)
|
|
438
|
-
claim.confidence_tier = tier
|
|
439
|
-
claim.score_breakdown = breakdown
|
|
440
|
-
_report_coercions(run, claim, breakdown, ensemble_n)
|
|
441
|
-
|
|
442
|
-
run.coverage_gaps = build_coverage_gap_entries(run.completeness)
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
def _report_coercions(run: PipelineRun, claim: Claim, breakdown: dict,
|
|
446
|
-
ensemble_n: int) -> None:
|
|
447
|
-
"""ROADMAP_v2 §30. Pass 5 makes no model call, so a value it had to
|
|
448
|
-
correct has nowhere else to surface.
|
|
449
|
-
|
|
450
|
-
Every line here describes something that should have been impossible:
|
|
451
|
-
the payload boundary rejects each of these at the pass that sent it and
|
|
452
|
-
re-asks. Reaching Pass 5 means the value survived §3's retries, or the
|
|
453
|
-
claim did not come through Pass 2 at all. `run.trace` is the artifact
|
|
454
|
-
base.py calls as trustworthy as the report -- silently scoring around a
|
|
455
|
-
bad value is precisely what #75 measured, so the correction is stated
|
|
456
|
-
even though it is also the safe one.
|
|
457
|
-
"""
|
|
458
|
-
if "consistency_reported" in breakdown:
|
|
459
|
-
run.log(
|
|
460
|
-
f"Pass 5: claim {claim.id} was asserted by candidates "
|
|
461
|
-
f"{claim.asserted_by_candidates} against a denominator of "
|
|
462
|
-
f"{ensemble_n} -- consistency clamped to 1.0. A numerator above "
|
|
463
|
-
f"the denominator means Pass 2 misread the candidate labelling."
|
|
464
|
-
)
|
|
465
|
-
if ensemble_n > 0 and breakdown.get("consistency_score") is None:
|
|
466
|
-
run.log(
|
|
467
|
-
f"Pass 5: claim {claim.id} carries no asserted_by_candidates on "
|
|
468
|
-
f"an ensemble run -- not treated as contested, since an "
|
|
469
|
-
f"unreported numerator is not a numerator of zero."
|
|
470
|
-
)
|
|
471
|
-
if "unrecognised_grounding_status" in breakdown:
|
|
472
|
-
run.log(
|
|
473
|
-
f"Pass 5: claim {claim.id} has grounding_status="
|
|
474
|
-
f"{breakdown['unrecognised_grounding_status']!r}, which is not "
|
|
475
|
-
f"one of {', '.join(sorted(GROUNDING_STATUSES))} -- scored as "
|
|
476
|
-
f"ungrounded."
|
|
477
|
-
)
|
|
478
|
-
if "unrecognised_type" in breakdown:
|
|
479
|
-
run.log(
|
|
480
|
-
f"Pass 5: claim {claim.id} has type="
|
|
481
|
-
f"{breakdown['unrecognised_type']!r}, which is not one of "
|
|
482
|
-
f"{', '.join(sorted(CLAIM_TYPES))} -- scored on the factual "
|
|
483
|
-
f"formula."
|
|
484
|
-
)
|
|
1
|
+
"""
|
|
2
|
+
core/reasoning/confidence_scoring.py
|
|
3
|
+
|
|
4
|
+
Pass 5 -- confidence tiering. ZERO LLM calls, by design: this is a
|
|
5
|
+
deterministic, documented weighted formula over already-computed data
|
|
6
|
+
(grounding status, critic severity, assumption flags). Kept as its own
|
|
7
|
+
module, separate from orchestrator.py, so the weights can be tuned later
|
|
8
|
+
without touching any prompt or any sequencing logic.
|
|
9
|
+
|
|
10
|
+
Interpretation worth flagging: Pass 3c's "gaps" (things the response
|
|
11
|
+
never addressed at all) are NOT claims that exist and failed
|
|
12
|
+
verification -- they're an absence. Rather than forcing them into the
|
|
13
|
+
Claim model, they're represented separately as coverage_gaps, each
|
|
14
|
+
tagged UNVERIFIED_COVERAGE. This is my reading of what that tier is for;
|
|
15
|
+
flag it if you intended something different.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from core.reasoning.base import (
|
|
21
|
+
CLAIM_TYPES,
|
|
22
|
+
GROUNDING_STATUSES,
|
|
23
|
+
Claim,
|
|
24
|
+
ConfidenceTier,
|
|
25
|
+
PipelineRun,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
# --- Tunable weights -- change these, not the logic below, to retune scoring ---
|
|
29
|
+
GROUNDING_WEIGHTS = {"grounded": 1.0, "partial": 0.5, "ungrounded": 0.0, None: 0.0}
|
|
30
|
+
GROUNDING_WEIGHT_FACTOR = 0.5
|
|
31
|
+
CRITIC_WEIGHT_FACTOR = 0.35
|
|
32
|
+
ASSUMPTION_FLAG_PENALTY = 0.15 # per flag
|
|
33
|
+
NON_FACTUAL_SCORE_CAP = 0.65 # claims never grounded can't reach HIGH
|
|
34
|
+
|
|
35
|
+
# ROADMAP_v3 §90 (A85, issue #12). Cross-candidate disagreement is ROUTED,
|
|
36
|
+
# NOT SCORED. A factual claim asserted by at most this share of the
|
|
37
|
+
# candidates is `contested`, and D1 sends it to the 6a/6b loop to be
|
|
38
|
+
# checked; nothing is subtracted for it.
|
|
39
|
+
#
|
|
40
|
+
# It was a penalty before (§10's revisit, E8): 0.15 * (1 - consistency),
|
|
41
|
+
# itself the repair of §10's first shape, a fourth weighted term that made
|
|
42
|
+
# grounding count for less whenever ensemble mode was on. Two things were
|
|
43
|
+
# wrong with the penalty, and the second was measured:
|
|
44
|
+
#
|
|
45
|
+
# 1. It answered the wrong question. Disagreement is the one signal in
|
|
46
|
+
# the pipeline that points at a specific claim and says "the models do
|
|
47
|
+
# not concur here", which is what a verification pass wants as input,
|
|
48
|
+
# and a deduction spends it on a number instead.
|
|
49
|
+
# 2. It could send a claim into the loop that no revision could get out
|
|
50
|
+
# of. A partially grounded claim scores 0.60 (MEDIUM); at 1 of 3 the
|
|
51
|
+
# penalty took it to 0.50 (LOW), so D1 flagged it, 6a was never told
|
|
52
|
+
# why, and 6b re-scored it under the same penalty -- who asserted a
|
|
53
|
+
# claim does not change when it is reworded -- until D2 published it
|
|
54
|
+
# UNVERIFIED.
|
|
55
|
+
#
|
|
56
|
+
# AT MOST HALF, so 1 of 2, 1 of 3 and 2 of 4 are contested and 2 of 3 is
|
|
57
|
+
# not: exactly the shapes where the penalty used to change a tier. The
|
|
58
|
+
# non-ensemble formula is untouched, and the ensemble one is now the same
|
|
59
|
+
# formula, so §10's byte-for-byte regression guard holds for both.
|
|
60
|
+
CONTESTED_SHARE = 0.5
|
|
61
|
+
|
|
62
|
+
# ROADMAP_v2 §45 (SQ3). What a grounding is WORTH.
|
|
63
|
+
#
|
|
64
|
+
# MULTIPLICATIVE, NOT A FOURTH TERM, and that is the whole design. §10
|
|
65
|
+
# shipped a fourth weighted term for consistency and reversed it (see
|
|
66
|
+
# E8 above): redistributing 0.5/0.35 to make room meant grounding counted
|
|
67
|
+
# for less whenever the feature was on, and the new term could only ever
|
|
68
|
+
# subtract. Scaling the grounding component instead says what is actually
|
|
69
|
+
# meant -- how much is this grounding worth -- and leaves the non-scored
|
|
70
|
+
# formula LITERALLY unchanged rather than coincidentally equal, so the
|
|
71
|
+
# byte-for-byte regression guard holds by construction here too.
|
|
72
|
+
#
|
|
73
|
+
# BOTH CONSTANTS BELOW WERE SET BY MEASUREMENT, not by taste, and the
|
|
74
|
+
# first draft of each was wrong in a way only a sweep showed.
|
|
75
|
+
#
|
|
76
|
+
# AUTHORITY_FULL_CREDIT exists because the domain table is a RANKING and
|
|
77
|
+
# this formula needs an EVIDENTIAL scale. Feeding `authority` in raw
|
|
78
|
+
# multiplied two rank-scales together and landed everything in the middle:
|
|
79
|
+
# swept across the whole table, only a `.gov` at similarity 1.0 still
|
|
80
|
+
# reached HIGH, and Nature, arXiv, Reddit and a Pinterest pin were
|
|
81
|
+
# indistinguishable at MEDIUM -- a tier that no longer discriminates,
|
|
82
|
+
# which is the opposite of the point. At or above a peer-reviewed
|
|
83
|
+
# publisher, authority is not what limits a claim's confidence, so that is
|
|
84
|
+
# where full credit starts.
|
|
85
|
+
#
|
|
86
|
+
# SOURCE_QUALITY_FLOOR was 0.4 in the first draft, and 0.4 is exactly the
|
|
87
|
+
# worst available value: 0.5*0.4 + 0.35 == 0.55, which IS the MEDIUM
|
|
88
|
+
# threshold, so the floor guaranteed MEDIUM for every grounded claim
|
|
89
|
+
# however bad its sources were. At 0.25 a claim grounded only in a content
|
|
90
|
+
# farm lands LOW -- flagged, and sent to the 6a/6b loop to find something
|
|
91
|
+
# better, which is the measured Pinterest case getting the treatment it
|
|
92
|
+
# always should have.
|
|
93
|
+
#
|
|
94
|
+
# NOT ZERO, though: a weak-but-real grounding is still better evidenced
|
|
95
|
+
# than one nothing supports, and the forced-UNVERIFIED rule below already
|
|
96
|
+
# handles the latter.
|
|
97
|
+
AUTHORITY_FULL_CREDIT = 0.85
|
|
98
|
+
SOURCE_QUALITY_FLOOR = 0.25
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
TIER_THRESHOLDS = [
|
|
102
|
+
(0.8, "HIGH"),
|
|
103
|
+
(0.55, "MEDIUM"),
|
|
104
|
+
(0.3, "LOW"),
|
|
105
|
+
] # anything below the lowest threshold, or an ungrounded factual claim, is UNVERIFIED
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
# ROADMAP_v2 §45 (SQ6). The constants above are the DEFAULTS; a run may
|
|
109
|
+
# carry its own, resolved once at pipeline start from settings.json by
|
|
110
|
+
# core.config_loader.effective_confidence.
|
|
111
|
+
#
|
|
112
|
+
# THE PARAMETER DEFAULTS TO THESE, which is what keeps every existing
|
|
113
|
+
# caller -- and the byte-for-byte regression tests this file is built
|
|
114
|
+
# around -- scoring exactly what they always did. A retuned run is an
|
|
115
|
+
# explicit act at one call site, never a silent re-reading of global
|
|
116
|
+
# state inside a pure function.
|
|
117
|
+
#
|
|
118
|
+
# Built by a FUNCTION rather than bound at import, so a test or a tool
|
|
119
|
+
# that patches one of the constants above still sees its own value here.
|
|
120
|
+
def default_weights() -> dict:
|
|
121
|
+
return {
|
|
122
|
+
"grounding_weight_factor": GROUNDING_WEIGHT_FACTOR,
|
|
123
|
+
"critic_weight_factor": CRITIC_WEIGHT_FACTOR,
|
|
124
|
+
"assumption_flag_penalty": ASSUMPTION_FLAG_PENALTY,
|
|
125
|
+
"non_factual_score_cap": NON_FACTUAL_SCORE_CAP,
|
|
126
|
+
"source_quality_floor": SOURCE_QUALITY_FLOOR,
|
|
127
|
+
"authority_full_credit": AUTHORITY_FULL_CREDIT,
|
|
128
|
+
"grounding_weights": dict(GROUNDING_WEIGHTS),
|
|
129
|
+
"tier_thresholds": {name: value for value, name in TIER_THRESHOLDS},
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _tier_for(raw_score: float, thresholds: dict) -> str:
|
|
134
|
+
"""The tier `raw_score` earns, highest first.
|
|
135
|
+
|
|
136
|
+
SORTED HERE rather than trusting the mapping's order: the resolver
|
|
137
|
+
already refuses a table that does not descend, and sorting again means
|
|
138
|
+
a caller passing one by hand cannot make a tier unreachable by
|
|
139
|
+
writing its keys in the wrong order.
|
|
140
|
+
"""
|
|
141
|
+
for name, threshold in sorted(thresholds.items(), key=lambda kv: -kv[1]):
|
|
142
|
+
if raw_score >= threshold:
|
|
143
|
+
return name
|
|
144
|
+
return "UNVERIFIED"
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def source_quality(claim: Claim, scored: bool,
|
|
148
|
+
full_credit: float = None):
|
|
149
|
+
"""The best source this claim has, as authority x similarity, or None.
|
|
150
|
+
|
|
151
|
+
A PRODUCT, not an average (§45, SQ3). An authoritative source that
|
|
152
|
+
does not discuss the claim and a perfect match on a content farm are
|
|
153
|
+
both worth nothing, and only multiplication says so. MAX across
|
|
154
|
+
sources, because grounding asks whether ANY credible source supports
|
|
155
|
+
the claim -- a noisy-OR that rewarded corroboration would let five
|
|
156
|
+
mediocre blogs outscore one primary source.
|
|
157
|
+
|
|
158
|
+
`scored` IS THE WHOLE SAFETY PROPERTY, and it is passed rather than
|
|
159
|
+
inferred. An empty source list is ambiguous on its own: it means "the
|
|
160
|
+
model claimed this was grounded and cited nothing" on a run that went
|
|
161
|
+
through §45's scoring stage, and it means "this claim was built
|
|
162
|
+
without one" everywhere else -- a test, a §20 correction, a resume.
|
|
163
|
+
Reading 0.0 from the second would re-tier claims nothing has measured,
|
|
164
|
+
which is exactly the silent rescoring this module's own history is
|
|
165
|
+
about. So None (leave the formula alone) is the answer whenever the
|
|
166
|
+
caller has not said scoring ran.
|
|
167
|
+
"""
|
|
168
|
+
if not scored:
|
|
169
|
+
return None
|
|
170
|
+
if full_credit is None:
|
|
171
|
+
full_credit = AUTHORITY_FULL_CREDIT
|
|
172
|
+
sources = claim.grounding_sources
|
|
173
|
+
if not isinstance(sources, list):
|
|
174
|
+
return 0.0
|
|
175
|
+
values = []
|
|
176
|
+
for source in sources:
|
|
177
|
+
if not isinstance(source, dict):
|
|
178
|
+
continue
|
|
179
|
+
authority = source.get("authority_score")
|
|
180
|
+
relevance = source.get("similarity_score")
|
|
181
|
+
if any(isinstance(v, bool) or not isinstance(v, (int, float))
|
|
182
|
+
for v in (authority, relevance)):
|
|
183
|
+
continue
|
|
184
|
+
# Authority is normalised against AUTHORITY_FULL_CREDIT before the
|
|
185
|
+
# product: the table ranks source types, and a rank multiplied by
|
|
186
|
+
# a rank is not evidence. Above the reference it is capped rather
|
|
187
|
+
# than allowed to compensate for weak relevance -- a government
|
|
188
|
+
# site that does not discuss the claim is still not evidence for
|
|
189
|
+
# it, which is the property the product exists to express.
|
|
190
|
+
credit = min(1.0, max(0.0, float(authority)) / full_credit)
|
|
191
|
+
values.append(credit * min(1.0, max(0.0, float(relevance))))
|
|
192
|
+
# 0.0 rather than None for a claim with no scorable source at all:
|
|
193
|
+
# scoring DID run, and it found nothing behind a claim the model
|
|
194
|
+
# asserted. source_scoring already says so in the trace; this is the
|
|
195
|
+
# same finding reaching the score.
|
|
196
|
+
return max(values) if values else 0.0
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def score_claim(claim: Claim, ensemble_n: int = 0,
|
|
200
|
+
sources_scored: bool = False,
|
|
201
|
+
weights: dict | None = None) -> tuple[ConfidenceTier, dict]:
|
|
202
|
+
"""Returns (tier, score_breakdown). Pure function -- no side effects,
|
|
203
|
+
easy to unit test claim-by-claim without running any pass.
|
|
204
|
+
|
|
205
|
+
When ensemble_n > 0 (ROADMAP §10), the share of candidates that
|
|
206
|
+
asserted this claim is recorded, and a factual claim asserted by at
|
|
207
|
+
most CONTESTED_SHARE of them is marked `contested` for D1 to route
|
|
208
|
+
(ROADMAP_v3 §90). It changes no score. ensemble_n is the number of
|
|
209
|
+
candidates that actually produced text, which on a run where a
|
|
210
|
+
candidate's provider failed is fewer than the configured roster.
|
|
211
|
+
|
|
212
|
+
ROADMAP_v2 §30 (B8, B9, B10) hardened all three of this function's
|
|
213
|
+
model-supplied inputs. Every coercion below is a LAST RESORT: the
|
|
214
|
+
payload boundary rejects each of these values at the pass that sent
|
|
215
|
+
them, and re-asks. What reaches here is a value that survived §3's
|
|
216
|
+
corrective retries, or a claim built by some path that never went
|
|
217
|
+
through Pass 2 -- and silently scoring it wrong is the outcome #75
|
|
218
|
+
measured.
|
|
219
|
+
"""
|
|
220
|
+
# §45 (SQ6). Resolved weights or this module's own defaults -- and
|
|
221
|
+
# None is the default, so every caller that predates §45 scores
|
|
222
|
+
# exactly what it always did.
|
|
223
|
+
w = default_weights() if weights is None else weights
|
|
224
|
+
|
|
225
|
+
# B9. base.py's Literals are ANNOTATIONS. An unrecognised status used
|
|
226
|
+
# to take GROUNDING_WEIGHTS' 0.0 default AND slip past the
|
|
227
|
+
# forced-UNVERIFIED rule below, which compares == "ungrounded" -- so a
|
|
228
|
+
# claim the model said it could not source was published as merely
|
|
229
|
+
# LOW. Coerced to the CONSERVATIVE end, which is the one that cannot
|
|
230
|
+
# overstate confidence.
|
|
231
|
+
status = claim.grounding_status
|
|
232
|
+
unrecognised_status = None
|
|
233
|
+
if status is not None and status not in GROUNDING_STATUSES:
|
|
234
|
+
unrecognised_status, status = status, "ungrounded"
|
|
235
|
+
|
|
236
|
+
# Same, for the type. Factual is the conservative branch: it has no
|
|
237
|
+
# NON_FACTUAL_SCORE_CAP floor to sit on, and it is the only branch the
|
|
238
|
+
# forced-UNVERIFIED rule can reach. A capitalised `type` used to skip
|
|
239
|
+
# gate D0's routing entirely and then score MEDIUM off the cap.
|
|
240
|
+
claim_type = claim.type
|
|
241
|
+
unrecognised_type = None
|
|
242
|
+
if claim_type not in CLAIM_TYPES:
|
|
243
|
+
unrecognised_type, claim_type = claim_type, "factual"
|
|
244
|
+
|
|
245
|
+
# §45 (SQ3). What the grounding is WORTH, not a fourth axis beside it.
|
|
246
|
+
# None -- no scoring stage ran for this claim -- gives a multiplier of
|
|
247
|
+
# 1.0, which is the pre-§45 arithmetic exactly.
|
|
248
|
+
# ROUNDED ONCE, HERE, then used for both the formula and the
|
|
249
|
+
# breakdown -- E10's rule applied to a second derived value. The
|
|
250
|
+
# unrounded product of two fractions records as
|
|
251
|
+
# `grounding_component: 0.7264705882352941` in an artifact whose other
|
|
252
|
+
# fields are all 4 decimal places, and rounding it on the way OUT
|
|
253
|
+
# instead is exactly how E10's two values came to disagree: the score
|
|
254
|
+
# would be computed from one number and published as another.
|
|
255
|
+
quality = source_quality(claim, sources_scored, w["authority_full_credit"])
|
|
256
|
+
quality_multiplier = (1.0 if quality is None
|
|
257
|
+
else round(max(w["source_quality_floor"], quality), 4))
|
|
258
|
+
grounding_weight = w["grounding_weights"].get(status, 0.0)
|
|
259
|
+
grounding_component = grounding_weight * quality_multiplier
|
|
260
|
+
critic_component = 1.0 - claim.critic_severity
|
|
261
|
+
assumption_penalty = w["assumption_flag_penalty"] * len(claim.assumption_flags)
|
|
262
|
+
|
|
263
|
+
# None when ensemble mode is off -- and (B7) also when ensemble mode is
|
|
264
|
+
# ON and Pass 2 reported nothing for this claim, which is a different
|
|
265
|
+
# fact from "no candidate asserted it". An unreported claim is never
|
|
266
|
+
# contested.
|
|
267
|
+
consistency_score = None
|
|
268
|
+
consistency_reported = None
|
|
269
|
+
if ensemble_n > 0 and claim.asserted_by_candidates:
|
|
270
|
+
# B8. Deduplicated and clamped. `[1, 2, 3, 3]` made consistency
|
|
271
|
+
# 1.3333, which under E8 made the penalty NEGATIVE and scored a
|
|
272
|
+
# MEDIUM claim HIGH; here a duplicate would make one candidate's
|
|
273
|
+
# claim read as two agreeing, and `[1, 1]` of 3 would escape
|
|
274
|
+
# routing. `consistency_reported` keeps the unclamped value when
|
|
275
|
+
# it fired, because a numerator above the denominator is itself
|
|
276
|
+
# evidence that Pass 2 misread the candidate labelling, and
|
|
277
|
+
# clamping silently would hide the evidence along with the bug.
|
|
278
|
+
reported = len(set(claim.asserted_by_candidates)) / ensemble_n
|
|
279
|
+
consistency_score = min(1.0, reported)
|
|
280
|
+
if reported > 1.0:
|
|
281
|
+
consistency_reported = round(reported, 4)
|
|
282
|
+
|
|
283
|
+
if claim_type != "factual":
|
|
284
|
+
# Cap applies to the critic component BEFORE subtracting the
|
|
285
|
+
# assumption penalty -- capping the final score instead would let
|
|
286
|
+
# the cap swallow the penalty whenever the pre-penalty score
|
|
287
|
+
# already exceeded it, making assumption flags invisible on any
|
|
288
|
+
# claim that was already clean under critique. Found this via a
|
|
289
|
+
# direct test case (two speculative claims, one flagged one not,
|
|
290
|
+
# both landing at the same tier) before it shipped.
|
|
291
|
+
capped_critic = min(critic_component, w["non_factual_score_cap"])
|
|
292
|
+
raw_score = capped_critic - assumption_penalty
|
|
293
|
+
formula = "non_factual"
|
|
294
|
+
else:
|
|
295
|
+
capped_critic = None
|
|
296
|
+
formula = "factual"
|
|
297
|
+
raw_score = (
|
|
298
|
+
(w["grounding_weight_factor"] * grounding_component)
|
|
299
|
+
+ (w["critic_weight_factor"] * critic_component)
|
|
300
|
+
- assumption_penalty
|
|
301
|
+
)
|
|
302
|
+
|
|
303
|
+
# ROUND ONCE, then use that one value for BOTH the tier comparison and
|
|
304
|
+
# the breakdown. These used to be two values: the tier compared the raw
|
|
305
|
+
# float while the breakdown stored round(raw, 4), so a claim scoring
|
|
306
|
+
# 0.7999999999999999 was recorded as `raw_score: 0.8` and tiered MEDIUM
|
|
307
|
+
# -- against a TIER_THRESHOLDS table that says >= 0.8 is HIGH. Anyone
|
|
308
|
+
# auditing a run artifact read a number that contradicted the published
|
|
309
|
+
# thresholds, and the tier was the one that mattered.
|
|
310
|
+
#
|
|
311
|
+
# Binary floating point made this reachable from ordinary weights, not
|
|
312
|
+
# from an exotic input: 0.5 + 0.35 - 0.15 * (1/3), E8's 2-of-3 penalty,
|
|
313
|
+
# was exactly that value.
|
|
314
|
+
# Sweeping the whole non-ensemble input space, exactly ONE shape changes
|
|
315
|
+
# tier under this fix -- a speculative claim at severity 0.55 with one
|
|
316
|
+
# assumption flag, which records 0.3 and now tiers LOW instead of
|
|
317
|
+
# UNVERIFIED. That is what the threshold table always specified.
|
|
318
|
+
raw_score = round(max(0.0, min(1.0, raw_score)), 4)
|
|
319
|
+
|
|
320
|
+
if claim_type == "factual" and status == "ungrounded":
|
|
321
|
+
tier: ConfidenceTier = "UNVERIFIED"
|
|
322
|
+
else:
|
|
323
|
+
tier = _tier_for(raw_score, w["tier_thresholds"])
|
|
324
|
+
|
|
325
|
+
# ROADMAP_v2 §30 (B10). THE BREAKDOWN RECORDS WHAT WAS APPLIED, AND
|
|
326
|
+
# NAMES THE BRANCH THAT APPLIED IT, so a reader can recompute raw_score
|
|
327
|
+
# from these fields alone. It could not before: a speculative claim
|
|
328
|
+
# asserted by 1 of 3 candidates stored
|
|
329
|
+
# `grounding_component: 0.0, disagreement_penalty: 0.1, raw_score: 0.65`
|
|
330
|
+
# -- and recomputing from those gives 0.25, because E9 subtracts the
|
|
331
|
+
# penalty only in the factual branch and grounding is not in the
|
|
332
|
+
# non-factual formula at all. Both fields were inert and neither said
|
|
333
|
+
# so.
|
|
334
|
+
#
|
|
335
|
+
# Same class as E10 on the same object, which is why it is worth the
|
|
336
|
+
# extra key: "anyone auditing a run artifact read a number that
|
|
337
|
+
# contradicted the published thresholds."
|
|
338
|
+
#
|
|
339
|
+
# factual 0.5*grounding + 0.35*critic - assumption
|
|
340
|
+
# non_factual capped_critic - assumption
|
|
341
|
+
breakdown = {
|
|
342
|
+
"formula": formula,
|
|
343
|
+
"critic_component": critic_component,
|
|
344
|
+
"assumption_penalty": assumption_penalty,
|
|
345
|
+
# Already rounded above -- rounding again here is what let the two
|
|
346
|
+
# values drift apart in the first place.
|
|
347
|
+
"raw_score": raw_score,
|
|
348
|
+
}
|
|
349
|
+
if formula == "factual":
|
|
350
|
+
breakdown["grounding_component"] = grounding_component
|
|
351
|
+
# B10's property, extended: the breakdown alone must let a reader
|
|
352
|
+
# recompute grounding_component. Written ONLY on the factual
|
|
353
|
+
# branch and only when scoring ran, so a field that did nothing is
|
|
354
|
+
# never present -- which is the exact defect B10 records for
|
|
355
|
+
# `grounding_component` and `disagreement_penalty` on a
|
|
356
|
+
# speculative claim.
|
|
357
|
+
if quality is not None:
|
|
358
|
+
breakdown["grounding_weight"] = grounding_weight
|
|
359
|
+
breakdown["source_quality"] = round(quality, 4)
|
|
360
|
+
breakdown["source_quality_multiplier"] = quality_multiplier
|
|
361
|
+
breakdown["source_count"] = len(claim.grounding_sources) \
|
|
362
|
+
if isinstance(claim.grounding_sources, list) else 0
|
|
363
|
+
else:
|
|
364
|
+
breakdown["capped_critic_component"] = capped_critic
|
|
365
|
+
if ensemble_n > 0:
|
|
366
|
+
# E12: the DENOMINATOR, so a stored claim is self-describing on a run
|
|
367
|
+
# where the roster shrank. Without it, "consistency 0.5" is
|
|
368
|
+
# ambiguous between one of two candidates and two of four, and the
|
|
369
|
+
# roster that produced it lives only in the trace.
|
|
370
|
+
breakdown["consistency_denominator"] = ensemble_n
|
|
371
|
+
# B7. None means PASS 2 REPORTED NOTHING, which is not 0.0 -- and
|
|
372
|
+
# the two were indistinguishable, so a claim one candidate raised
|
|
373
|
+
# uncontested took the maximum 0.15 penalty rather than the 0.05 it
|
|
374
|
+
# earned under E8. `contested` needs them apart for the same reason.
|
|
375
|
+
breakdown["consistency_score"] = (
|
|
376
|
+
None if consistency_score is None else round(consistency_score, 4))
|
|
377
|
+
# §90: what D1 routes on, and the record of why a claim that scored
|
|
378
|
+
# HIGH went through 6a/6b anyway. E9 LIVES HERE, and only here:
|
|
379
|
+
# whether candidates declining to speculate says anything about a
|
|
380
|
+
# speculative claim is a judgment §10 never made, so the non-factual
|
|
381
|
+
# branch is never asked and carries no key (B10: a field that did
|
|
382
|
+
# nothing is never present).
|
|
383
|
+
if formula == "factual":
|
|
384
|
+
breakdown["contested"] = (consistency_score is not None
|
|
385
|
+
and consistency_score <= CONTESTED_SHARE)
|
|
386
|
+
if consistency_reported is not None:
|
|
387
|
+
breakdown["consistency_reported"] = consistency_reported
|
|
388
|
+
if unrecognised_status is not None:
|
|
389
|
+
breakdown["unrecognised_grounding_status"] = unrecognised_status
|
|
390
|
+
if unrecognised_type is not None:
|
|
391
|
+
breakdown["unrecognised_type"] = unrecognised_type
|
|
392
|
+
return tier, breakdown
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def is_contested(claim: Claim) -> bool:
|
|
396
|
+
"""Whether Pass 5 marked this claim contested (ROADMAP_v3 §90).
|
|
397
|
+
|
|
398
|
+
Read off the breakdown rather than recomputed, so D1 routes on exactly
|
|
399
|
+
what the stored claim says and the type coercion above happens once."""
|
|
400
|
+
return claim.score_breakdown.get("contested") is True
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def build_coverage_gap_entries(completeness: dict) -> list[dict]:
|
|
404
|
+
"""Gaps from Pass 3c become separate coverage-gap entries, not Claim
|
|
405
|
+
objects -- they were never asserted, so there's nothing to re-tier."""
|
|
406
|
+
return [{"gap": gap, "tier": "UNVERIFIED_COVERAGE"} for gap in completeness.get("gaps", [])]
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def run_confidence_tiering(
|
|
410
|
+
run: PipelineRun, ensemble_n: int = 0,
|
|
411
|
+
claims: list[Claim] | None = None,
|
|
412
|
+
sources_scored: bool = False,
|
|
413
|
+
weights: dict | None = None,
|
|
414
|
+
) -> None:
|
|
415
|
+
"""The actual Pass 5 entry point the orchestrator calls. Mutates
|
|
416
|
+
the claims in place; zero LLM calls. Pass ensemble_n > 0 to record
|
|
417
|
+
each claim's consistency and mark the contested ones (ROADMAP §10,
|
|
418
|
+
ROADMAP_v3 §90); the formula is the same either way.
|
|
419
|
+
|
|
420
|
+
`claims` narrows WHICH claims are re-tiered, and defaults to all of
|
|
421
|
+
them -- which is Pass 5 itself, where every claim is in play. The
|
|
422
|
+
6a/6c loop passes its current batch instead: it calls this once per
|
|
423
|
+
round, and re-tiering the whole run there recomputes claims that are
|
|
424
|
+
already finished. A claim D2 had exhausted got its tier restored from
|
|
425
|
+
unchanged score data while the fallback annotation _apply_fallback
|
|
426
|
+
wrote stayed put, so it shipped as (say) LOW beside a note saying it
|
|
427
|
+
could not be verified -- and a claim D1 never flagged could be
|
|
428
|
+
re-tiered by any round that touched its grounding.
|
|
429
|
+
|
|
430
|
+
`run.coverage_gaps` is rebuilt either way: it is a pure function of
|
|
431
|
+
run.completeness, which does not change across the loop, so the
|
|
432
|
+
narrowed call leaves it exactly as the full one would.
|
|
433
|
+
"""
|
|
434
|
+
for claim in (run.claims if claims is None else claims):
|
|
435
|
+
tier, breakdown = score_claim(claim, ensemble_n=ensemble_n,
|
|
436
|
+
sources_scored=sources_scored,
|
|
437
|
+
weights=weights)
|
|
438
|
+
claim.confidence_tier = tier
|
|
439
|
+
claim.score_breakdown = breakdown
|
|
440
|
+
_report_coercions(run, claim, breakdown, ensemble_n)
|
|
441
|
+
|
|
442
|
+
run.coverage_gaps = build_coverage_gap_entries(run.completeness)
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def _report_coercions(run: PipelineRun, claim: Claim, breakdown: dict,
|
|
446
|
+
ensemble_n: int) -> None:
|
|
447
|
+
"""ROADMAP_v2 §30. Pass 5 makes no model call, so a value it had to
|
|
448
|
+
correct has nowhere else to surface.
|
|
449
|
+
|
|
450
|
+
Every line here describes something that should have been impossible:
|
|
451
|
+
the payload boundary rejects each of these at the pass that sent it and
|
|
452
|
+
re-asks. Reaching Pass 5 means the value survived §3's retries, or the
|
|
453
|
+
claim did not come through Pass 2 at all. `run.trace` is the artifact
|
|
454
|
+
base.py calls as trustworthy as the report -- silently scoring around a
|
|
455
|
+
bad value is precisely what #75 measured, so the correction is stated
|
|
456
|
+
even though it is also the safe one.
|
|
457
|
+
"""
|
|
458
|
+
if "consistency_reported" in breakdown:
|
|
459
|
+
run.log(
|
|
460
|
+
f"Pass 5: claim {claim.id} was asserted by candidates "
|
|
461
|
+
f"{claim.asserted_by_candidates} against a denominator of "
|
|
462
|
+
f"{ensemble_n} -- consistency clamped to 1.0. A numerator above "
|
|
463
|
+
f"the denominator means Pass 2 misread the candidate labelling."
|
|
464
|
+
)
|
|
465
|
+
if ensemble_n > 0 and breakdown.get("consistency_score") is None:
|
|
466
|
+
run.log(
|
|
467
|
+
f"Pass 5: claim {claim.id} carries no asserted_by_candidates on "
|
|
468
|
+
f"an ensemble run -- not treated as contested, since an "
|
|
469
|
+
f"unreported numerator is not a numerator of zero."
|
|
470
|
+
)
|
|
471
|
+
if "unrecognised_grounding_status" in breakdown:
|
|
472
|
+
run.log(
|
|
473
|
+
f"Pass 5: claim {claim.id} has grounding_status="
|
|
474
|
+
f"{breakdown['unrecognised_grounding_status']!r}, which is not "
|
|
475
|
+
f"one of {', '.join(sorted(GROUNDING_STATUSES))} -- scored as "
|
|
476
|
+
f"ungrounded."
|
|
477
|
+
)
|
|
478
|
+
if "unrecognised_type" in breakdown:
|
|
479
|
+
run.log(
|
|
480
|
+
f"Pass 5: claim {claim.id} has type="
|
|
481
|
+
f"{breakdown['unrecognised_type']!r}, which is not one of "
|
|
482
|
+
f"{', '.join(sorted(CLAIM_TYPES))} -- scored on the factual "
|
|
483
|
+
f"formula."
|
|
484
|
+
)
|