@yahyahammad/venastine 0.3.8 → 0.3.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -19
- package/CONFIG_ARCHITECTURE.md +2139 -2002
- package/LICENSE +201 -201
- package/README.md +1043 -1034
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -106
- package/agents/builtin/review.md +74 -74
- package/agents/manager.py +261 -261
- package/agents/subagent_tool.py +472 -472
- package/agents/tui_commands.py +238 -146
- package/bin/venastine.mjs +584 -528
- package/bin/venastine.py +73 -71
- package/cli_commands.py +848 -835
- package/config.yaml +64 -0
- package/config_edit.py +1301 -1301
- package/config_schema.py +987 -966
- package/config_update.py +749 -749
- package/core/agent_activity.py +256 -256
- package/core/ansi.py +452 -452
- package/core/approval.py +137 -137
- package/core/call_view.py +364 -364
- package/core/cancel.py +121 -121
- package/core/chatgpt_auth.py +747 -747
- package/core/claims_view.py +107 -107
- package/core/client.py +2756 -2715
- package/core/commands.py +464 -454
- package/core/compaction.py +1735 -1735
- package/core/config_loader.py +1982 -1982
- package/core/connect.py +686 -686
- package/core/events.py +123 -123
- package/core/file_mentions.py +280 -280
- package/core/interaction.py +411 -411
- package/core/loop.py +2566 -2566
- package/core/memory.py +482 -482
- package/core/pipeline_models.py +190 -190
- package/core/provider_errors.py +274 -234
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -484
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/orchestrator.py +1561 -1561
- package/core/reasoning/output_writer.py +345 -345
- package/core/reasoning/payload_validation.py +524 -524
- package/core/reasoning/pipeline_storage.py +427 -427
- package/core/reasoning/scholar.py +467 -458
- package/core/reasoning/source_corpus.py +433 -433
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -315
- package/core/shell_sessions.py +1553 -1553
- package/credentials.py +307 -307
- package/database.py +333 -333
- package/docs/PRIVACY.md +78 -78
- package/docs/SECURITY.md +114 -108
- package/docs/THIRD_PARTY_NOTICES.md +148 -148
- package/env_secrets.py +118 -118
- package/json_store.py +343 -343
- package/logging_setup.py +295 -295
- package/main.py +3433 -3433
- package/mcp_client/client.py +780 -733
- package/mcp_client/config.py +484 -484
- package/mcp_client/registration.py +343 -343
- package/memories/manager.py +147 -147
- package/memories/tui_commands.py +319 -239
- package/package.json +303 -303
- package/project_init/config_files.py +282 -282
- package/project_init/generator.py +728 -728
- package/project_init/manifest.py +418 -418
- package/project_init/tui_commands.py +147 -147
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +17 -17
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -316
- package/providers.json.example +94 -94
- package/pyproject.toml +149 -149
- package/requirements.txt +314 -295
- package/safety/policy_enforcement.py +1082 -1082
- package/security/askpass.py +419 -419
- package/security/capability.py +207 -207
- package/security/confine.py +484 -484
- package/security/hosted.py +182 -182
- package/security/permissions.py +237 -237
- package/security/protected_paths.py +501 -501
- package/security/sandbox.py +4108 -4108
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -110
- package/storage.py +1338 -1338
- package/tools/base.py +465 -465
- package/tools/builtin/_math_common.py +667 -667
- package/tools/builtin/_net_common.py +1145 -267
- package/tools/builtin/_page_text.py +490 -0
- package/tools/builtin/_pdf_text.py +113 -0
- package/tools/builtin/arxiv.py +685 -681
- package/tools/builtin/fetch_url.py +639 -371
- package/tools/builtin/file_ops.py +720 -720
- package/tools/builtin/geometry.py +184 -184
- package/tools/builtin/load_skill.py +76 -76
- package/tools/builtin/pin.py +227 -227
- package/tools/builtin/probability_stats.py +224 -224
- package/tools/builtin/project_docs.py +464 -464
- package/tools/builtin/remember.py +175 -175
- package/tools/builtin/shell.py +980 -980
- package/tools/builtin/todo.py +149 -149
- package/tools/builtin/web_search.py +402 -328
- package/tools/context.py +159 -159
- package/tools/isolation.py +387 -387
- package/tools/registry.py +1252 -1242
- package/tui/app.py +8372 -8307
- package/tui/app.tcss +754 -739
- package/tui/commands.py +26 -26
- package/tui/diffs.py +286 -286
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -102
- package/tui/markdown.py +1329 -1329
- package/tui/meters.py +364 -364
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -1197
- package/tui/themes.py +599 -599
- package/tui/widgets.py +3861 -3796
|
@@ -1,855 +1,855 @@
|
|
|
1
|
-
"""
|
|
2
|
-
core/reasoning/source_scoring.py
|
|
3
|
-
|
|
4
|
-
What a cited source is worth, computed rather than asked for.
|
|
5
|
-
|
|
6
|
-
ROADMAP_v2 §45. Pass 3a used to return `authority_score` and
|
|
7
|
-
`similarity_score` per source, both invented by the model, both unchecked,
|
|
8
|
-
both unread by anything downstream. A run on disk scored a Pinterest pin
|
|
9
|
-
`similarity_score: 1.0` beside the Wikipedia article on the same claim.
|
|
10
|
-
|
|
11
|
-
WHAT MOVED, AND WHAT DID NOT. Authority is now a DOMAIN CLASS computed
|
|
12
|
-
here (SQ4), which the model may correct by at most
|
|
13
|
-
`config.AUTHORITY_ADJUSTMENT_CAP` and only with a stated reason. Relevance
|
|
14
|
-
is still the model's number for now, under a much stricter rubric, and
|
|
15
|
-
becomes a cosine when an embedder is configured (SQ2) -- the field
|
|
16
|
-
`similarity_method` says which, per source, on every run, because a number
|
|
17
|
-
whose provenance is implicit is a number a reader has to guess about.
|
|
18
|
-
|
|
19
|
-
SEPARATE FROM confidence_scoring.py, and for its stated reason. That
|
|
20
|
-
module is Pass 5: a documented weighted formula over already-computed
|
|
21
|
-
data, kept apart from the orchestrator so its weights can be retuned
|
|
22
|
-
without touching sequencing. This is the stage that COMPUTES some of that
|
|
23
|
-
data, it can make network calls (an embedding is a model call), and
|
|
24
|
-
AGENTS.md locks Pass 5 at zero model calls. Merging them would put a
|
|
25
|
-
model call inside the one pass whose defining property is that it has
|
|
26
|
-
none.
|
|
27
|
-
|
|
28
|
-
NOTHING HERE RAISES. Every input is a value a model produced, and every
|
|
29
|
-
malformed one has a conservative reading: a source that cannot be scored
|
|
30
|
-
is dropped, an adjustment with no reason is discarded, an out-of-range
|
|
31
|
-
number is clamped. Each is reported to `run.trace` -- this stage's
|
|
32
|
-
outputs reach a published report, and silently scoring around a bad value
|
|
33
|
-
is precisely the defect §30's B-family measured. What it must never do is
|
|
34
|
-
fail a ten-pass run over one bad float.
|
|
35
|
-
"""
|
|
36
|
-
|
|
37
|
-
from __future__ import annotations
|
|
38
|
-
|
|
39
|
-
import hashlib
|
|
40
|
-
import html
|
|
41
|
-
import logging
|
|
42
|
-
import math
|
|
43
|
-
import re
|
|
44
|
-
|
|
45
|
-
import config
|
|
46
|
-
|
|
47
|
-
logger = logging.getLogger(__name__)
|
|
48
|
-
|
|
49
|
-
# Country-code second-level registries that gate registration the way the
|
|
50
|
-
# bare TLDs do -- `gov.uk`, `ac.uk`, `edu.au`, `mil.br`. DERIVED rather
|
|
51
|
-
# than enumerated in config.py: the list is `{kind}.{cc}` across every
|
|
52
|
-
# ccTLD, which is several hundred rows to write and one rule to state.
|
|
53
|
-
#
|
|
54
|
-
# `ac` is here and NOT in the bare-TLD table on purpose: `.ac` alone is
|
|
55
|
-
# Ascension Island's open commercial ccTLD, so a bare `ac` entry would
|
|
56
|
-
# score any `example.ac` as a university.
|
|
57
|
-
_RESTRICTED_SECOND_LEVEL = frozenset({"gov", "mil", "edu", "ac"})
|
|
58
|
-
|
|
59
|
-
_TAG_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>|<[^>]+>",
|
|
60
|
-
re.IGNORECASE | re.DOTALL)
|
|
61
|
-
_WHITESPACE_RE = re.compile(r"\s+")
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
# ---------------------------------------------------------------------------
|
|
65
|
-
# ---- Text normalisation ---------------------------------------------------
|
|
66
|
-
# ---------------------------------------------------------------------------
|
|
67
|
-
|
|
68
|
-
def strip_markup(text: str) -> str:
|
|
69
|
-
"""Raw HTML to something worth comparing against.
|
|
70
|
-
|
|
71
|
-
`fetch_url` returns the body UNPROCESSED -- no boilerplate removal, no
|
|
72
|
-
readability pass -- so the corpus holds markup. That matters twice:
|
|
73
|
-
a quote of visible text does not literally appear in a string
|
|
74
|
-
containing `<em>`, and an embedding of `<div class="nav-wrapper">`
|
|
75
|
-
measures a page's template rather than its content.
|
|
76
|
-
|
|
77
|
-
`script` and `style` bodies are removed WITH their contents; every
|
|
78
|
-
other tag is removed and its text kept. Entities are unescaped
|
|
79
|
-
afterwards, so `&` compares equal to `&`.
|
|
80
|
-
|
|
81
|
-
NOT a readability extractor. Navigation and footers survive, which
|
|
82
|
-
costs a windowed similarity nothing -- an irrelevant window simply
|
|
83
|
-
does not win the max -- and replacing this with real extraction is
|
|
84
|
-
named in §45's "deliberately not" list.
|
|
85
|
-
"""
|
|
86
|
-
if not text:
|
|
87
|
-
return ""
|
|
88
|
-
return _WHITESPACE_RE.sub(" ", html.unescape(_TAG_RE.sub(" ", text))).strip()
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
def _comparable(text: str) -> str:
|
|
92
|
-
"""Case- and whitespace-insensitive, for the quote containment check."""
|
|
93
|
-
return _WHITESPACE_RE.sub(" ", html.unescape(text or "")).strip().casefold()
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
# ---------------------------------------------------------------------------
|
|
97
|
-
# ---- Domain classification (SQ4) ------------------------------------------
|
|
98
|
-
# ---------------------------------------------------------------------------
|
|
99
|
-
|
|
100
|
-
def _hostname(url: str) -> str:
|
|
101
|
-
from urllib.parse import urlsplit
|
|
102
|
-
try:
|
|
103
|
-
host = urlsplit(url).hostname or ""
|
|
104
|
-
except ValueError:
|
|
105
|
-
return ""
|
|
106
|
-
return host.lower().rstrip(".")
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
def classify_domain(url: str, overrides: dict | None = None) -> tuple[str, float]:
|
|
110
|
-
"""(class name, base authority) for `url`.
|
|
111
|
-
|
|
112
|
-
LONGEST SUFFIX WINS, on label boundaries. `blogs.example.gov` matches
|
|
113
|
-
`gov`; `notagov` does not, because the match requires the suffix to
|
|
114
|
-
start at a label. That boundary is the whole check: a substring test
|
|
115
|
-
would score `evilgov.com` as a government site.
|
|
116
|
-
|
|
117
|
-
`overrides` is the user's `domain_overrides` -- a {suffix: score} map
|
|
118
|
-
merged at the same precedence as the table but winning ties, so adding
|
|
119
|
-
one domain never requires restating the class it belongs to.
|
|
120
|
-
|
|
121
|
-
An unknown host takes `config.DEFAULT_DOMAIN_AUTHORITY` and the class
|
|
122
|
-
name `unknown`, which is the generic-commercial number: a source this
|
|
123
|
-
table has never heard of is ordinary, not suspect.
|
|
124
|
-
"""
|
|
125
|
-
host = _hostname(url)
|
|
126
|
-
if not host:
|
|
127
|
-
return "unknown", config.DEFAULT_DOMAIN_AUTHORITY
|
|
128
|
-
|
|
129
|
-
labels = host.split(".")
|
|
130
|
-
# Every suffix of the host, longest first: a.b.c -> a.b.c, b.c, c
|
|
131
|
-
candidates = [".".join(labels[i:]) for i in range(len(labels))]
|
|
132
|
-
|
|
133
|
-
if overrides:
|
|
134
|
-
for candidate in candidates:
|
|
135
|
-
if candidate in overrides:
|
|
136
|
-
return "override", float(overrides[candidate])
|
|
137
|
-
|
|
138
|
-
for candidate in candidates:
|
|
139
|
-
class_name = config.DOMAIN_AUTHORITY_SUFFIXES.get(candidate)
|
|
140
|
-
if class_name:
|
|
141
|
-
return class_name, config.DOMAIN_AUTHORITY_CLASSES[class_name]
|
|
142
|
-
|
|
143
|
-
# `gov.uk`, `ac.nz`, `edu.au`: a restricted registry under a ccTLD.
|
|
144
|
-
# Checked AFTER the explicit table so a listed host under one of them
|
|
145
|
-
# keeps its own class.
|
|
146
|
-
if len(labels) >= 2 and labels[-2] in _RESTRICTED_SECOND_LEVEL \
|
|
147
|
-
and len(labels[-1]) == 2:
|
|
148
|
-
return ("restricted_registry",
|
|
149
|
-
config.DOMAIN_AUTHORITY_CLASSES["restricted_registry"])
|
|
150
|
-
|
|
151
|
-
return "unknown", config.DEFAULT_DOMAIN_AUTHORITY
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
# ---------------------------------------------------------------------------
|
|
155
|
-
# ---- Per-source scoring ---------------------------------------------------
|
|
156
|
-
# ---------------------------------------------------------------------------
|
|
157
|
-
|
|
158
|
-
def _clamp(value: float, low: float = 0.0, high: float = 1.0) -> float:
|
|
159
|
-
return max(low, min(high, value))
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
def _as_number(value):
|
|
163
|
-
"""`value` as a float, or None. Rejects bools, which are ints in
|
|
164
|
-
Python and would otherwise read `True` as a similarity of 1.0."""
|
|
165
|
-
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
166
|
-
return None
|
|
167
|
-
return float(value)
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
class _Coercions:
|
|
171
|
-
"""What had to be corrected, aggregated by kind.
|
|
172
|
-
|
|
173
|
-
ONE TRACE LINE PER KIND rather than per source. `_report_coercions`
|
|
174
|
-
in confidence_scoring.py names the individual claim, which is right
|
|
175
|
-
there -- Pass 5 touches each claim once. This stage touches every
|
|
176
|
-
SOURCE of every claim, so per-source lines would put sixty entries in
|
|
177
|
-
a trace that currently holds twenty, and a trace nobody finishes
|
|
178
|
-
reading is the same as no trace. The count and the first few examples
|
|
179
|
-
are what a reader acts on.
|
|
180
|
-
"""
|
|
181
|
-
|
|
182
|
-
EXAMPLES = 3
|
|
183
|
-
|
|
184
|
-
def __init__(self) -> None:
|
|
185
|
-
self.kinds: dict = {}
|
|
186
|
-
|
|
187
|
-
def note(self, kind: str, where: str) -> None:
|
|
188
|
-
count, examples = self.kinds.get(kind, (0, []))
|
|
189
|
-
if len(examples) < self.EXAMPLES:
|
|
190
|
-
examples.append(where)
|
|
191
|
-
self.kinds[kind] = (count + 1, examples)
|
|
192
|
-
|
|
193
|
-
def report(self, run) -> None:
|
|
194
|
-
for kind, (count, examples) in sorted(self.kinds.items()):
|
|
195
|
-
run.log(f"Pass 3a scoring: {count} source(s) {kind} "
|
|
196
|
-
f"(e.g. {', '.join(examples)}).")
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
def score_source(source: dict, claim, corpus, coercions: _Coercions,
|
|
200
|
-
overrides: dict | None = None,
|
|
201
|
-
authority_adjustment_cap: float | None = None) -> dict | None:
|
|
202
|
-
"""One cited source, scored. None if it cannot be scored at all.
|
|
203
|
-
|
|
204
|
-
Returns a NEW dict rather than mutating in place: the model's entry is
|
|
205
|
-
the input to this function, and a half-rewritten source -- some fields
|
|
206
|
-
replaced, some left as the model sent them -- is exactly the shape
|
|
207
|
-
that makes an artifact impossible to read. Every key the result
|
|
208
|
-
carries was written here.
|
|
209
|
-
"""
|
|
210
|
-
if not isinstance(source, dict):
|
|
211
|
-
coercions.note("were not objects and were dropped", claim.id)
|
|
212
|
-
return None
|
|
213
|
-
|
|
214
|
-
url = source.get("url")
|
|
215
|
-
if not isinstance(url, str) or not url.strip():
|
|
216
|
-
coercions.note("had no usable url and were dropped", claim.id)
|
|
217
|
-
return None
|
|
218
|
-
url = url.strip()
|
|
219
|
-
|
|
220
|
-
scored = {"url": url}
|
|
221
|
-
_score_relevance(source, scored, corpus, claim, coercions)
|
|
222
|
-
_score_authority(source, scored, claim, coercions, overrides,
|
|
223
|
-
authority_adjustment_cap)
|
|
224
|
-
return scored
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
def _score_relevance(source: dict, scored: dict, corpus, claim,
|
|
228
|
-
coercions: _Coercions) -> None:
|
|
229
|
-
"""The quote, whether it is real, and how close it is.
|
|
230
|
-
|
|
231
|
-
`similarity_score` is the model's for now, clamped, and marked as
|
|
232
|
-
such. The embedder replaces the number and the method; the rubric in
|
|
233
|
-
source_grounding.md is what makes the fallback worth having.
|
|
234
|
-
"""
|
|
235
|
-
quote = source.get("quote")
|
|
236
|
-
if quote is not None and not isinstance(quote, str):
|
|
237
|
-
coercions.note("had a non-string quote", claim.id)
|
|
238
|
-
quote = None
|
|
239
|
-
scored["quote"] = (quote or "").strip()
|
|
240
|
-
|
|
241
|
-
scored["quote_verified"] = _verify_quote(scored["quote"], scored["url"],
|
|
242
|
-
corpus)
|
|
243
|
-
|
|
244
|
-
reported = _as_number(source.get("similarity_score"))
|
|
245
|
-
if reported is None:
|
|
246
|
-
coercions.note("gave no usable similarity_score", claim.id)
|
|
247
|
-
scored["similarity_score_llm"] = None
|
|
248
|
-
scored["similarity_score"] = 0.0
|
|
249
|
-
else:
|
|
250
|
-
if not 0.0 <= reported <= 1.0:
|
|
251
|
-
coercions.note("gave a similarity_score outside 0-1", claim.id)
|
|
252
|
-
scored["similarity_score_llm"] = _clamp(reported)
|
|
253
|
-
scored["similarity_score"] = scored["similarity_score_llm"]
|
|
254
|
-
scored["similarity_method"] = "llm"
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
def _verify_quote(quote: str, url: str, corpus):
|
|
258
|
-
"""True, False, or None -- and None is a third answer, not a default.
|
|
259
|
-
|
|
260
|
-
None means THERE WAS NOTHING TO CHECK AGAINST: no corpus, no captured
|
|
261
|
-
text for this URL (the model cited a search result it never opened, or
|
|
262
|
-
fetched it during a JSON-retry turn, which is not translated), or no
|
|
263
|
-
quote. False means the run retrieved this page and the quoted span is
|
|
264
|
-
not in it, which is the fabricated-citation signal.
|
|
265
|
-
|
|
266
|
-
STRICT CONTAINMENT, after markup stripping, whitespace collapsing,
|
|
267
|
-
entity unescaping and case folding. It still has false negatives -- a
|
|
268
|
-
model that silently fixes a typo, or quotes across an element the
|
|
269
|
-
stripper turned into a space, fails a check it should pass -- so
|
|
270
|
-
`False` is a flag for a person, and §45 slice A deliberately does not
|
|
271
|
-
let it move any score. Loosening it to token overlap would make it
|
|
272
|
-
pass for a fabricated quote about the right topic, which is the one
|
|
273
|
-
thing it exists to catch.
|
|
274
|
-
"""
|
|
275
|
-
if corpus is None or not quote:
|
|
276
|
-
return None
|
|
277
|
-
document = corpus.get(url)
|
|
278
|
-
if document is None:
|
|
279
|
-
return None
|
|
280
|
-
return _comparable(quote) in _comparable(strip_markup(document.text))
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
def _score_authority(source: dict, scored: dict, claim,
|
|
284
|
-
coercions: _Coercions, overrides: dict | None,
|
|
285
|
-
adjustment_cap: float | None = None) -> None:
|
|
286
|
-
class_name, base = classify_domain(scored["url"], overrides)
|
|
287
|
-
scored["authority_class"] = class_name
|
|
288
|
-
scored["authority_base"] = round(base, 4)
|
|
289
|
-
scored["authority_method"] = "domain"
|
|
290
|
-
|
|
291
|
-
cap = (config.AUTHORITY_ADJUSTMENT_CAP if adjustment_cap is None
|
|
292
|
-
else adjustment_cap)
|
|
293
|
-
adjustment = _as_number(source.get("authority_adjustment")) or 0.0
|
|
294
|
-
reason = source.get("authority_reason")
|
|
295
|
-
reason = reason.strip() if isinstance(reason, str) else ""
|
|
296
|
-
|
|
297
|
-
if adjustment and not reason:
|
|
298
|
-
# DISCARDED, not clamped. The reason is what makes the adjustment
|
|
299
|
-
# auditable, and an unexplained nudge is indistinguishable from the
|
|
300
|
-
# invented number this section replaced.
|
|
301
|
-
coercions.note("adjusted authority with no reason, so it was dropped",
|
|
302
|
-
claim.id)
|
|
303
|
-
adjustment = 0.0
|
|
304
|
-
reason = ""
|
|
305
|
-
elif abs(adjustment) > cap:
|
|
306
|
-
coercions.note(f"adjusted authority beyond +/-{cap}", claim.id)
|
|
307
|
-
adjustment = _clamp(adjustment, -cap, cap)
|
|
308
|
-
|
|
309
|
-
scored["authority_adjustment"] = round(adjustment, 4)
|
|
310
|
-
scored["authority_reason"] = reason if adjustment else ""
|
|
311
|
-
scored["authority_score"] = round(_clamp(base + adjustment), 4)
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
# ===========================================================================
|
|
315
|
-
# ---- Similarity: chunking, cosine, calibration (SQ2) ----------------------
|
|
316
|
-
# ===========================================================================
|
|
317
|
-
|
|
318
|
-
# --- Tunable -- change these, not the logic below, to retune similarity ---
|
|
319
|
-
#
|
|
320
|
-
# THE ONE IDEA HERE IS MATCHING GRANULARITY. A claim is one sentence; a
|
|
321
|
-
# fetched page is up to 5000 characters. Embedding the page whole averages
|
|
322
|
-
# twenty topics into one vector, and the claim's topic contributes a few
|
|
323
|
-
# percent of its direction -- that IS the dilution, and it makes every long
|
|
324
|
-
# source look equally irrelevant. A window of roughly a claim's own length
|
|
325
|
-
# asks the question grounding actually asks: does this source contain a
|
|
326
|
-
# passage that states this.
|
|
327
|
-
WINDOW_CHARS = 480 # ~120 tokens, the same order as a claim
|
|
328
|
-
WINDOW_OVERLAP_SENTENCES = 1 # so a claim straddling a boundary is not lost
|
|
329
|
-
MAX_WINDOWS = 24 # more than fetch_url's 5000 chars can produce
|
|
330
|
-
|
|
331
|
-
# Below this, a claim is short enough to be pronoun-bearing ("It was signed
|
|
332
|
-
# in 1947") despite Pass 2 being asked for self-contained ones, so its
|
|
333
|
-
# entities are prepended. Above it, adding them dilutes the query with the
|
|
334
|
-
# very topic words every window shares.
|
|
335
|
-
CLAIM_MIN_CHARS = 80
|
|
336
|
-
|
|
337
|
-
# How many windows the recorded `cosine_top_mean` averages. It is NOT the
|
|
338
|
-
# score -- max is -- and exists so a reader can tell one matching sentence
|
|
339
|
-
# from a document that is wholly on point.
|
|
340
|
-
TOP_K = 3
|
|
341
|
-
|
|
342
|
-
# Inputs per embeddings request. Providers cap both the number of inputs
|
|
343
|
-
# and the tokens per input; this is well under every published limit and
|
|
344
|
-
# keeps one failed request from costing a whole run's vectors.
|
|
345
|
-
EMBED_BATCH_SIZE = 96
|
|
346
|
-
|
|
347
|
-
# Retries per batch. Immediate, with no backoff, following arxiv.py: a
|
|
348
|
-
# transient failure is worth one more try, and sleeping inside a research
|
|
349
|
-
# pass buys little while making the suite slow.
|
|
350
|
-
EMBED_MAX_RETRIES = 2
|
|
351
|
-
|
|
352
|
-
_SENTENCE_RE = re.compile(r"(?<=[.!?])\s+|\n+")
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
def split_sentences(text: str) -> list:
|
|
356
|
-
"""Rough sentence split, deliberately.
|
|
357
|
-
|
|
358
|
-
An abbreviation ("Dr. Smith") splits where it should not, and the cost
|
|
359
|
-
of that is one window boundary in the wrong place -- which a MAX over
|
|
360
|
-
overlapping windows absorbs. A real sentence tokenizer is a dependency,
|
|
361
|
-
and #145's posture is that a dependency earns its place by fixing
|
|
362
|
-
something measured.
|
|
363
|
-
"""
|
|
364
|
-
return [part.strip() for part in _SENTENCE_RE.split(text or "") if part.strip()]
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
def chunk_text(text: str, window_chars: int = WINDOW_CHARS,
|
|
368
|
-
max_windows: int = MAX_WINDOWS,
|
|
369
|
-
overlap: int = WINDOW_OVERLAP_SENTENCES) -> list:
|
|
370
|
-
"""`text` as overlapping windows of about `window_chars`.
|
|
371
|
-
|
|
372
|
-
Sentences are packed greedily and windows overlap by `overlap`
|
|
373
|
-
sentences, so a claim stated across a boundary still lands whole
|
|
374
|
-
inside some window. A single sentence longer than the budget is hard
|
|
375
|
-
split rather than emitted oversized -- one 4000-character
|
|
376
|
-
"sentence" (an unpunctuated navigation blob, which raw HTML produces
|
|
377
|
-
constantly) would otherwise become one diluted window and crowd out
|
|
378
|
-
the rest of the page.
|
|
379
|
-
"""
|
|
380
|
-
cleaned = strip_markup(text)
|
|
381
|
-
if not cleaned:
|
|
382
|
-
return []
|
|
383
|
-
|
|
384
|
-
sentences = []
|
|
385
|
-
for sentence in split_sentences(cleaned):
|
|
386
|
-
while len(sentence) > window_chars:
|
|
387
|
-
sentences.append(sentence[:window_chars])
|
|
388
|
-
sentence = sentence[window_chars:]
|
|
389
|
-
if sentence:
|
|
390
|
-
sentences.append(sentence)
|
|
391
|
-
if not sentences:
|
|
392
|
-
return []
|
|
393
|
-
|
|
394
|
-
windows = []
|
|
395
|
-
start = 0
|
|
396
|
-
while start < len(sentences) and len(windows) < max_windows:
|
|
397
|
-
end, length = start, 0
|
|
398
|
-
while end < len(sentences):
|
|
399
|
-
addition = len(sentences[end]) + (1 if end > start else 0)
|
|
400
|
-
if end > start and length + addition > window_chars:
|
|
401
|
-
break
|
|
402
|
-
length += addition
|
|
403
|
-
end += 1
|
|
404
|
-
windows.append(" ".join(sentences[start:end]))
|
|
405
|
-
if end >= len(sentences):
|
|
406
|
-
break
|
|
407
|
-
# At least one sentence forward, so an overlap wider than the
|
|
408
|
-
# window cannot stall the loop.
|
|
409
|
-
start = max(start + 1, end - overlap)
|
|
410
|
-
return windows
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
def l2_normalize(vector) -> list:
|
|
414
|
-
"""Unit length, so cosine is a dot product.
|
|
415
|
-
|
|
416
|
-
A zero vector is returned unchanged rather than divided by zero. It
|
|
417
|
-
then scores 0.0 against everything, which is the honest reading of a
|
|
418
|
-
vector carrying no direction -- and `embed_texts` already refuses a
|
|
419
|
-
response that is all zeros, so this is the single-vector case.
|
|
420
|
-
"""
|
|
421
|
-
norm = math.sqrt(sum(component * component for component in vector))
|
|
422
|
-
if norm == 0.0:
|
|
423
|
-
return list(vector)
|
|
424
|
-
return [component / norm for component in vector]
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
def cosine(left, right) -> float:
|
|
428
|
-
"""Cosine similarity of two ALREADY-NORMALIZED vectors.
|
|
429
|
-
|
|
430
|
-
Pure Python on purpose: `numpy` is not a dependency, and a few hundred
|
|
431
|
-
1536-dimensional dot products is microseconds. Mismatched lengths
|
|
432
|
-
score 0.0 rather than raising -- `embed_texts` rejects a ragged batch
|
|
433
|
-
at the boundary, so reaching here means two vectors from different
|
|
434
|
-
calls, and failing a run over it would undo SQ10's whole contract.
|
|
435
|
-
"""
|
|
436
|
-
if len(left) != len(right) or not left:
|
|
437
|
-
return 0.0
|
|
438
|
-
return sum(a * b for a, b in zip(left, right))
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
def calibrate(raw: float, floor: float, ceiling: float) -> float:
|
|
442
|
-
"""Raw cosine onto 0-1, using this model's useful band.
|
|
443
|
-
|
|
444
|
-
Unrelated text sits near 0.1 on some embedders and above 0.7 on
|
|
445
|
-
others, so the same passage would read as "irrelevant" on one and
|
|
446
|
-
"supports the claim" on another. An inverted or degenerate band
|
|
447
|
-
(floor >= ceiling) falls back to passing the cosine through clamped,
|
|
448
|
-
because refusing to score is worse than scoring on an uncalibrated
|
|
449
|
-
scale that the recorded raw cosine still explains.
|
|
450
|
-
"""
|
|
451
|
-
if ceiling <= floor:
|
|
452
|
-
return _clamp(raw)
|
|
453
|
-
return _clamp((raw - floor) / (ceiling - floor))
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
def _table_lookup(table: dict, model: str, default):
|
|
457
|
-
"""Longest matching slug substring wins, else `default`."""
|
|
458
|
-
best = None
|
|
459
|
-
for fragment, value in table.items():
|
|
460
|
-
if fragment in model and (best is None or len(fragment) > len(best[0])):
|
|
461
|
-
best = (fragment, value)
|
|
462
|
-
return best[1] if best else default
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
def calibration_for(model: str) -> tuple:
|
|
466
|
-
band = _table_lookup(config.SIMILARITY_CALIBRATION, model,
|
|
467
|
-
config.DEFAULT_SIMILARITY_CALIBRATION)
|
|
468
|
-
return float(band["floor"]), float(band["ceiling"])
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
def prefixes_for(model: str) -> dict:
|
|
472
|
-
return _table_lookup(config.EMBEDDER_PREFIXES, model, {})
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
class EmbeddingScorer:
|
|
476
|
-
"""One run's embedder: batching, caching, and the fallback.
|
|
477
|
-
|
|
478
|
-
STATEFUL AND RUN-SCOPED, because both things it holds are per-run: a
|
|
479
|
-
vector cache keyed by the text itself (a claim is embedded once
|
|
480
|
-
however many sources it has, and a page fetched twice is chunked
|
|
481
|
-
twice into identical windows), and a `failed` flag.
|
|
482
|
-
|
|
483
|
-
ONE FAILURE DISABLES THE SCORER FOR THE REST OF THE RUN, after its
|
|
484
|
-
retries. That is SQ10 taken seriously rather than half-applied: a
|
|
485
|
-
provider that is down stays down for the next forty batches, and
|
|
486
|
-
retrying each of them turns one outage into a run that takes minutes
|
|
487
|
-
to produce the fallback it was always going to produce. Every source
|
|
488
|
-
then carries `similarity_method: "llm"`, and the trace says when the
|
|
489
|
-
switch happened.
|
|
490
|
-
"""
|
|
491
|
-
|
|
492
|
-
def __init__(self, provider_name: str, model: str,
|
|
493
|
-
floor: float | None = None,
|
|
494
|
-
ceiling: float | None = None) -> None:
|
|
495
|
-
self.provider_name = provider_name
|
|
496
|
-
self.model = model
|
|
497
|
-
self.failed_reason = None
|
|
498
|
-
self.input_tokens = 0
|
|
499
|
-
self.calls = 0
|
|
500
|
-
self._cache: dict = {}
|
|
501
|
-
prefixes = prefixes_for(model)
|
|
502
|
-
self._query_prefix = prefixes.get("query", "")
|
|
503
|
-
self._passage_prefix = prefixes.get("passage", "")
|
|
504
|
-
self._query_task = prefixes.get("query_task_type", "")
|
|
505
|
-
self._passage_task = prefixes.get("passage_task_type", "")
|
|
506
|
-
table_floor, table_ceiling = calibration_for(model)
|
|
507
|
-
# A band the user set OUTRANKS the table, model_windows' inversion
|
|
508
|
-
# for model_windows' reason: the table cannot know about a model
|
|
509
|
-
# shipped after this release, and someone who measured their own
|
|
510
|
-
# embedder's floor is stating a fact about the deployment.
|
|
511
|
-
self.floor = table_floor if floor is None else floor
|
|
512
|
-
self.ceiling = table_ceiling if ceiling is None else ceiling
|
|
513
|
-
|
|
514
|
-
@property
|
|
515
|
-
def usable(self) -> bool:
|
|
516
|
-
return self.failed_reason is None
|
|
517
|
-
|
|
518
|
-
def _key(self, text: str, is_query: bool) -> str:
|
|
519
|
-
# The ROLE is part of the key: an asymmetric embedder gives the
|
|
520
|
-
# same string two different vectors depending on which prefix it
|
|
521
|
-
# was sent with, and collapsing them would silently score a claim
|
|
522
|
-
# against a passage-space vector of itself.
|
|
523
|
-
role = "q" if is_query else "p"
|
|
524
|
-
return f"{role}:{hashlib.sha256(text.encode('utf-8')).hexdigest()}"
|
|
525
|
-
|
|
526
|
-
def prime(self, texts, is_query: bool) -> None:
|
|
527
|
-
"""Embed everything not already cached, in batches.
|
|
528
|
-
|
|
529
|
-
Called once per role per stage so the whole run's claims are one
|
|
530
|
-
request and the whole run's windows are a handful -- rather than
|
|
531
|
-
one request per source, which is the shape that makes an embedder
|
|
532
|
-
slower than the pass it is scoring.
|
|
533
|
-
"""
|
|
534
|
-
if not self.usable:
|
|
535
|
-
return
|
|
536
|
-
pending, seen = [], set()
|
|
537
|
-
for text in texts:
|
|
538
|
-
key = self._key(text, is_query)
|
|
539
|
-
if key in self._cache or key in seen:
|
|
540
|
-
continue
|
|
541
|
-
seen.add(key)
|
|
542
|
-
pending.append(text)
|
|
543
|
-
|
|
544
|
-
prefix = self._query_prefix if is_query else self._passage_prefix
|
|
545
|
-
task = self._query_task if is_query else self._passage_task
|
|
546
|
-
for start in range(0, len(pending), EMBED_BATCH_SIZE):
|
|
547
|
-
batch = pending[start:start + EMBED_BATCH_SIZE]
|
|
548
|
-
vectors = self._embed(batch, prefix, task)
|
|
549
|
-
if vectors is None:
|
|
550
|
-
return
|
|
551
|
-
for text, vector in zip(batch, vectors):
|
|
552
|
-
self._cache[self._key(text, is_query)] = l2_normalize(vector)
|
|
553
|
-
|
|
554
|
-
def _embed(self, batch, prefix, task):
|
|
555
|
-
"""The batch, or None once the scorer has given up on this run."""
|
|
556
|
-
from core.client import EmbeddingError, embed_texts
|
|
557
|
-
|
|
558
|
-
last = None
|
|
559
|
-
for _ in range(EMBED_MAX_RETRIES + 1):
|
|
560
|
-
try:
|
|
561
|
-
result = embed_texts(self.provider_name, self.model, batch,
|
|
562
|
-
prefix=prefix, task_type=task)
|
|
563
|
-
except EmbeddingError as exc:
|
|
564
|
-
last = exc
|
|
565
|
-
continue
|
|
566
|
-
self.calls += 1
|
|
567
|
-
self.input_tokens += result.usage.get("input_tokens", 0)
|
|
568
|
-
return result.vectors
|
|
569
|
-
self.failed_reason = str(last)
|
|
570
|
-
return None
|
|
571
|
-
|
|
572
|
-
def vector(self, text: str, is_query: bool):
|
|
573
|
-
return self._cache.get(self._key(text, is_query))
|
|
574
|
-
|
|
575
|
-
def score(self, query: str, passages) -> dict | None:
|
|
576
|
-
"""The best window's calibrated similarity, or None.
|
|
577
|
-
|
|
578
|
-
MAX, not mean. Grounding asks whether the source contains a
|
|
579
|
-
passage that states the claim, which is a max question; a mean
|
|
580
|
-
over a long page answers "is this page broadly about the topic",
|
|
581
|
-
which is a different and weaker question. `cosine_top_mean` is
|
|
582
|
-
recorded beside it so a reader can tell one matching sentence from
|
|
583
|
-
a document that is wholly on point.
|
|
584
|
-
"""
|
|
585
|
-
if not self.usable or not passages:
|
|
586
|
-
return None
|
|
587
|
-
query_vector = self.vector(query, True)
|
|
588
|
-
if query_vector is None:
|
|
589
|
-
return None
|
|
590
|
-
scores = [cosine(query_vector, vector) for vector in
|
|
591
|
-
(self.vector(p, False) for p in passages)
|
|
592
|
-
if vector is not None]
|
|
593
|
-
if not scores:
|
|
594
|
-
return None
|
|
595
|
-
scores.sort(reverse=True)
|
|
596
|
-
top = scores[:TOP_K]
|
|
597
|
-
return {
|
|
598
|
-
"cosine": round(scores[0], 4),
|
|
599
|
-
"cosine_top_mean": round(sum(top) / len(top), 4),
|
|
600
|
-
"windows_scored": len(scores),
|
|
601
|
-
"similarity_score": round(
|
|
602
|
-
calibrate(scores[0], self.floor, self.ceiling), 4),
|
|
603
|
-
}
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
def claim_query_text(claim, min_chars: int = CLAIM_MIN_CHARS) -> str:
|
|
607
|
-
"""What gets embedded for a claim.
|
|
608
|
-
|
|
609
|
-
Its text verbatim -- Pass 2 produces atomic, self-contained claims, so
|
|
610
|
-
there is nothing to add. The exception is a claim short enough to be
|
|
611
|
-
pronoun-bearing anyway ("It was signed in 1947"), where the entities
|
|
612
|
-
Pass 2 already extracted are the missing subject. Prepending them to
|
|
613
|
-
EVERY claim would be worse: the entity words appear in every window of
|
|
614
|
-
the source too, so they raise every score alike and compress the range
|
|
615
|
-
the calibration band depends on.
|
|
616
|
-
|
|
617
|
-
THE REVISION WINS WHERE THERE IS ONE. Pass 6b re-validates the claim
|
|
618
|
-
Pass 6a rewrote, and `revalidate.md` says so in as many words -- but
|
|
619
|
-
`claim.text` still holds the ORIGINAL, because 6c is what finally
|
|
620
|
-
merges a revision into `final_text`. Scoring against `claim.text` on a
|
|
621
|
-
retry round measures a source against a sentence the claim no longer
|
|
622
|
-
asserts, and the vector cache makes it silent: the query is unchanged,
|
|
623
|
-
so it is a cache hit and the round costs nothing and reports nothing.
|
|
624
|
-
Found by running the pipeline end to end and reading the trace.
|
|
625
|
-
"""
|
|
626
|
-
text = (claim.revision_text or claim.text or "").strip()
|
|
627
|
-
if len(text) >= min_chars or not claim.entities:
|
|
628
|
-
return text
|
|
629
|
-
return f"{', '.join(str(e) for e in claim.entities)} — {text}"
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
def source_passages(scored: dict, corpus, window_chars: int = WINDOW_CHARS,
|
|
633
|
-
max_windows: int = MAX_WINDOWS) -> list:
|
|
634
|
-
"""The windows a source is scored over.
|
|
635
|
-
|
|
636
|
-
THE QUOTE IS ITS OWN WINDOW, first. It is the passage the model said
|
|
637
|
-
it relied on, so it is the most specific evidence available -- and
|
|
638
|
-
scoring it directly means a source whose captured text was truncated
|
|
639
|
-
before the relevant paragraph still scores on the part that mattered.
|
|
640
|
-
The document's own windows follow, so a fabricated or badly chosen
|
|
641
|
-
quote cannot LOWER a score the page itself earns.
|
|
642
|
-
"""
|
|
643
|
-
passages = []
|
|
644
|
-
quote = scored.get("quote")
|
|
645
|
-
if quote:
|
|
646
|
-
passages.append(quote)
|
|
647
|
-
document = corpus.get(scored["url"]) if corpus is not None else None
|
|
648
|
-
if document is not None:
|
|
649
|
-
passages.extend(chunk_text(document.text, window_chars, max_windows))
|
|
650
|
-
return passages
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
# ---------------------------------------------------------------------------
|
|
654
|
-
# ---- The stage ------------------------------------------------------------
|
|
655
|
-
# ---------------------------------------------------------------------------
|
|
656
|
-
|
|
657
|
-
def make_scorer(run=None, floor: float | None = None,
|
|
658
|
-
ceiling: float | None = None):
|
|
659
|
-
"""The run's EmbeddingScorer, or None with the warning said once.
|
|
660
|
-
|
|
661
|
-
SQ2's fallback is deliberate and announced. A run with no embedder
|
|
662
|
-
still produces a `similarity_score` for every source -- the grounding
|
|
663
|
-
model's own, under source_grounding.md's anchored bands -- and the
|
|
664
|
-
artifact marks every one of them `similarity_method: "llm"`. What must
|
|
665
|
-
not happen is a reader assuming the number is a measurement. So the
|
|
666
|
-
trace says so at the top of the run rather than leaving it to be
|
|
667
|
-
inferred from a field.
|
|
668
|
-
"""
|
|
669
|
-
from core import pipeline_models
|
|
670
|
-
|
|
671
|
-
chosen = pipeline_models.resolve("embedder")
|
|
672
|
-
if chosen is None:
|
|
673
|
-
if run is not None:
|
|
674
|
-
run.log(
|
|
675
|
-
"Source scoring: no embedder configured, so similarity is "
|
|
676
|
-
"the grounding model's own estimate of sources it chose "
|
|
677
|
-
"(similarity_method=\"llm\" on every source). Set one with "
|
|
678
|
-
"/embedder for a measured cosine.")
|
|
679
|
-
return None
|
|
680
|
-
return EmbeddingScorer(chosen["provider_name"], chosen["model"],
|
|
681
|
-
floor=floor, ceiling=ceiling)
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
def _apply_embeddings(run, claims, corpus, scorer, window_chars, max_windows,
|
|
685
|
-
claim_min_chars) -> None:
|
|
686
|
-
"""Replace every scorable source's similarity with a cosine.
|
|
687
|
-
|
|
688
|
-
TWO PHASES, and that is the whole reason this is not done inside
|
|
689
|
-
`score_source`. Every text the run needs is embedded first, in
|
|
690
|
-
batches, then every pair is scored from the cache -- so a run with
|
|
691
|
-
twenty claims and sixty sources makes a handful of requests instead of
|
|
692
|
-
sixty. Per-source calls would make the embedder slower than the pass
|
|
693
|
-
it is scoring, which is how a good measurement gets turned off.
|
|
694
|
-
"""
|
|
695
|
-
plan = []
|
|
696
|
-
for claim in claims:
|
|
697
|
-
query = claim_query_text(claim, claim_min_chars)
|
|
698
|
-
if not query:
|
|
699
|
-
continue
|
|
700
|
-
for scored in claim.grounding_sources:
|
|
701
|
-
passages = source_passages(scored, corpus, window_chars,
|
|
702
|
-
max_windows)
|
|
703
|
-
if passages:
|
|
704
|
-
plan.append((query, scored, passages))
|
|
705
|
-
if not plan:
|
|
706
|
-
return
|
|
707
|
-
|
|
708
|
-
scorer.prime([query for query, _, _ in plan], is_query=True)
|
|
709
|
-
scorer.prime([p for _, _, passages in plan for p in passages],
|
|
710
|
-
is_query=False)
|
|
711
|
-
|
|
712
|
-
measured = 0
|
|
713
|
-
for query, scored, passages in plan:
|
|
714
|
-
result = scorer.score(query, passages)
|
|
715
|
-
if result is None:
|
|
716
|
-
continue
|
|
717
|
-
scored.update(result)
|
|
718
|
-
scored["similarity_method"] = "embedding"
|
|
719
|
-
scored["embedder"] = f"{scorer.provider_name}|{scorer.model}"
|
|
720
|
-
measured += 1
|
|
721
|
-
|
|
722
|
-
if scorer.usable:
|
|
723
|
-
# THE COUNTERS ARE CUMULATIVE, and the line says so. This stage
|
|
724
|
-
# runs once per grounding -- Pass 3a and then every 6b round -- so
|
|
725
|
-
# a line reading "in 2 calls, 30 tokens" three times invites a
|
|
726
|
-
# reader to total it to six calls and ninety tokens, when the
|
|
727
|
-
# scorer's cache means the later rounds cost nothing. Under-
|
|
728
|
-
# reporting spend is the wrong direction to be ambiguous in.
|
|
729
|
-
run.log(
|
|
730
|
-
f"Source scoring: {measured} source(s) scored by cosine against "
|
|
731
|
-
f"{scorer.model} (run total so far: {scorer.calls} call(s), "
|
|
732
|
-
f"{scorer.input_tokens} embedding token(s)).")
|
|
733
|
-
else:
|
|
734
|
-
# NAMED, and the count says how far it got. A run where the
|
|
735
|
-
# embedder died after twelve sources has twelve measured scores
|
|
736
|
-
# and the rest self-reported, and an artifact whose readers cannot
|
|
737
|
-
# tell which is which is the thing similarity_method exists for.
|
|
738
|
-
run.log(
|
|
739
|
-
f"Source scoring: the embedder failed and the rest of this run "
|
|
740
|
-
f"falls back to the grounding model's own similarity "
|
|
741
|
-
f"({measured} source(s) measured before it stopped). "
|
|
742
|
-
f"Cause: {scorer.failed_reason}")
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
def _apply_scholar(run, claims, scholar) -> None:
|
|
746
|
-
"""Replace a cited paper's domain class with what OpenAlex knows.
|
|
747
|
-
|
|
748
|
-
A THIRD PHASE for `_apply_embeddings`' reason: it makes network calls,
|
|
749
|
-
and the per-source pass has to stay a pure rewrite of what the model
|
|
750
|
-
sent. It runs AFTER that pass so it is handed urls that have already
|
|
751
|
-
been coerced into shape.
|
|
752
|
-
|
|
753
|
-
A source that is not a resolvable paper keeps its domain class and
|
|
754
|
-
gains nothing, which is the right answer -- `authority_method` says
|
|
755
|
-
which of the two produced the number, so no reader has to guess.
|
|
756
|
-
|
|
757
|
-
THE ADJUSTMENT IS REAPPLIED, not discarded. The model's bounded nudge
|
|
758
|
-
is about THIS PAGE -- a retracted-but-uncorrected posting, an author
|
|
759
|
-
manuscript that differs from the published version -- and it is as
|
|
760
|
-
true of a paper as of any other source. Re-derived from the recorded
|
|
761
|
-
adjustment rather than kept, because the base underneath it changed.
|
|
762
|
-
"""
|
|
763
|
-
upgraded = 0
|
|
764
|
-
for claim in claims:
|
|
765
|
-
for scored in claim.grounding_sources:
|
|
766
|
-
if not scholar.usable:
|
|
767
|
-
break
|
|
768
|
-
authority, signals = scholar.authority_for(scored["url"])
|
|
769
|
-
if authority is None:
|
|
770
|
-
continue
|
|
771
|
-
scored["authority_base"] = round(authority, 4)
|
|
772
|
-
scored["authority_method"] = "scholar"
|
|
773
|
-
scored["scholar"] = signals
|
|
774
|
-
scored["authority_score"] = round(
|
|
775
|
-
_clamp(authority + scored.get("authority_adjustment", 0.0)), 4)
|
|
776
|
-
upgraded += 1
|
|
777
|
-
|
|
778
|
-
if upgraded:
|
|
779
|
-
run.log(f"Source scoring: {upgraded} cited paper(s) scored from "
|
|
780
|
-
f"OpenAlex (venue, citation percentile within their own "
|
|
781
|
-
f"field-and-year cohort, author standing) in "
|
|
782
|
-
f"{scholar.calls} lookup(s).")
|
|
783
|
-
if scholar.failed_reason:
|
|
784
|
-
run.log(f"Source scoring: the scholarly lookup stopped after "
|
|
785
|
-
f"{upgraded} paper(s); the rest keep their domain class. "
|
|
786
|
-
f"Cause: {scholar.failed_reason}")
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
def score_grounding_sources(run, claims=None, corpus=None,
|
|
790
|
-
overrides: dict | None = None,
|
|
791
|
-
scorer=None,
|
|
792
|
-
window_chars: int = WINDOW_CHARS,
|
|
793
|
-
max_windows: int = MAX_WINDOWS,
|
|
794
|
-
claim_min_chars: int = CLAIM_MIN_CHARS,
|
|
795
|
-
authority_adjustment_cap: float | None = None,
|
|
796
|
-
scholar=None) -> int:
|
|
797
|
-
"""Rewrite every claim's `grounding_sources` with computed scores.
|
|
798
|
-
|
|
799
|
-
Returns the number of sources scored. Called from
|
|
800
|
-
`orchestrator._ground_and_score`, which is the one seam Pass 3a and
|
|
801
|
-
Pass 6b's re-validation both go through -- so a source re-grounded
|
|
802
|
-
after a revision is re-scored against the revised claim rather than
|
|
803
|
-
keeping a number computed for text that no longer exists.
|
|
804
|
-
|
|
805
|
-
`claims` defaults to every claim in the run. Non-factual claims never
|
|
806
|
-
reach Pass 3a, so their source lists are empty and iterating them
|
|
807
|
-
costs nothing; narrowing the default would be a second rule to keep in
|
|
808
|
-
step with gate D0's routing for no gain.
|
|
809
|
-
"""
|
|
810
|
-
coercions = _Coercions()
|
|
811
|
-
scored_count = 0
|
|
812
|
-
targets = list(run.claims if claims is None else claims)
|
|
813
|
-
|
|
814
|
-
for claim in targets:
|
|
815
|
-
sources = claim.grounding_sources
|
|
816
|
-
if not isinstance(sources, list):
|
|
817
|
-
# The payload boundary type-guards this at the pass that sent
|
|
818
|
-
# it; a value arriving here is one that came from some other
|
|
819
|
-
# path. Emptied rather than iterated, because iterating a
|
|
820
|
-
# string scores one source per character.
|
|
821
|
-
coercions.note("arrived as a non-list source collection",
|
|
822
|
-
claim.id)
|
|
823
|
-
claim.grounding_sources = []
|
|
824
|
-
continue
|
|
825
|
-
|
|
826
|
-
rewritten = []
|
|
827
|
-
for source in sources:
|
|
828
|
-
scored = score_source(source, claim, corpus, coercions,
|
|
829
|
-
overrides, authority_adjustment_cap)
|
|
830
|
-
if scored is not None:
|
|
831
|
-
rewritten.append(scored)
|
|
832
|
-
claim.grounding_sources = rewritten
|
|
833
|
-
scored_count += len(rewritten)
|
|
834
|
-
|
|
835
|
-
if claim.grounding_status in ("grounded", "partial") and not rewritten:
|
|
836
|
-
# Said out loud rather than scored around. §45 SQ3 gives this
|
|
837
|
-
# claim zero source quality once the modulation lands; until
|
|
838
|
-
# then the trace is the only place it is visible at all.
|
|
839
|
-
coercions.note(
|
|
840
|
-
f"were claimed as {claim.grounding_status} with no scorable "
|
|
841
|
-
f"source", claim.id)
|
|
842
|
-
|
|
843
|
-
# BOTH network phases run AFTER the per-source pass, so each is handed
|
|
844
|
-
# sources whose quotes and urls have already been coerced into shape
|
|
845
|
-
# -- and so a payload the first phase rejected entirely never costs a
|
|
846
|
-
# call. Scholar first: it rewrites `authority_base`, and the trace
|
|
847
|
-
# then reads in the order the numbers were produced.
|
|
848
|
-
if scholar is not None and scholar.usable:
|
|
849
|
-
_apply_scholar(run, targets, scholar)
|
|
850
|
-
if scorer is not None and scorer.usable:
|
|
851
|
-
_apply_embeddings(run, targets, corpus, scorer, window_chars,
|
|
852
|
-
max_windows, claim_min_chars)
|
|
853
|
-
|
|
854
|
-
coercions.report(run)
|
|
855
|
-
return scored_count
|
|
1
|
+
"""
|
|
2
|
+
core/reasoning/source_scoring.py
|
|
3
|
+
|
|
4
|
+
What a cited source is worth, computed rather than asked for.
|
|
5
|
+
|
|
6
|
+
ROADMAP_v2 §45. Pass 3a used to return `authority_score` and
|
|
7
|
+
`similarity_score` per source, both invented by the model, both unchecked,
|
|
8
|
+
both unread by anything downstream. A run on disk scored a Pinterest pin
|
|
9
|
+
`similarity_score: 1.0` beside the Wikipedia article on the same claim.
|
|
10
|
+
|
|
11
|
+
WHAT MOVED, AND WHAT DID NOT. Authority is now a DOMAIN CLASS computed
|
|
12
|
+
here (SQ4), which the model may correct by at most
|
|
13
|
+
`config.AUTHORITY_ADJUSTMENT_CAP` and only with a stated reason. Relevance
|
|
14
|
+
is still the model's number for now, under a much stricter rubric, and
|
|
15
|
+
becomes a cosine when an embedder is configured (SQ2) -- the field
|
|
16
|
+
`similarity_method` says which, per source, on every run, because a number
|
|
17
|
+
whose provenance is implicit is a number a reader has to guess about.
|
|
18
|
+
|
|
19
|
+
SEPARATE FROM confidence_scoring.py, and for its stated reason. That
|
|
20
|
+
module is Pass 5: a documented weighted formula over already-computed
|
|
21
|
+
data, kept apart from the orchestrator so its weights can be retuned
|
|
22
|
+
without touching sequencing. This is the stage that COMPUTES some of that
|
|
23
|
+
data, it can make network calls (an embedding is a model call), and
|
|
24
|
+
AGENTS.md locks Pass 5 at zero model calls. Merging them would put a
|
|
25
|
+
model call inside the one pass whose defining property is that it has
|
|
26
|
+
none.
|
|
27
|
+
|
|
28
|
+
NOTHING HERE RAISES. Every input is a value a model produced, and every
|
|
29
|
+
malformed one has a conservative reading: a source that cannot be scored
|
|
30
|
+
is dropped, an adjustment with no reason is discarded, an out-of-range
|
|
31
|
+
number is clamped. Each is reported to `run.trace` -- this stage's
|
|
32
|
+
outputs reach a published report, and silently scoring around a bad value
|
|
33
|
+
is precisely the defect §30's B-family measured. What it must never do is
|
|
34
|
+
fail a ten-pass run over one bad float.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
from __future__ import annotations
|
|
38
|
+
|
|
39
|
+
import hashlib
|
|
40
|
+
import html
|
|
41
|
+
import logging
|
|
42
|
+
import math
|
|
43
|
+
import re
|
|
44
|
+
|
|
45
|
+
import config
|
|
46
|
+
|
|
47
|
+
logger = logging.getLogger(__name__)
|
|
48
|
+
|
|
49
|
+
# Country-code second-level registries that gate registration the way the
|
|
50
|
+
# bare TLDs do -- `gov.uk`, `ac.uk`, `edu.au`, `mil.br`. DERIVED rather
|
|
51
|
+
# than enumerated in config.py: the list is `{kind}.{cc}` across every
|
|
52
|
+
# ccTLD, which is several hundred rows to write and one rule to state.
|
|
53
|
+
#
|
|
54
|
+
# `ac` is here and NOT in the bare-TLD table on purpose: `.ac` alone is
|
|
55
|
+
# Ascension Island's open commercial ccTLD, so a bare `ac` entry would
|
|
56
|
+
# score any `example.ac` as a university.
|
|
57
|
+
_RESTRICTED_SECOND_LEVEL = frozenset({"gov", "mil", "edu", "ac"})
|
|
58
|
+
|
|
59
|
+
_TAG_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>|<[^>]+>",
|
|
60
|
+
re.IGNORECASE | re.DOTALL)
|
|
61
|
+
_WHITESPACE_RE = re.compile(r"\s+")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
# ---------------------------------------------------------------------------
|
|
65
|
+
# ---- Text normalisation ---------------------------------------------------
|
|
66
|
+
# ---------------------------------------------------------------------------
|
|
67
|
+
|
|
68
|
+
def strip_markup(text: str) -> str:
|
|
69
|
+
"""Raw HTML to something worth comparing against.
|
|
70
|
+
|
|
71
|
+
`fetch_url` returns the body UNPROCESSED -- no boilerplate removal, no
|
|
72
|
+
readability pass -- so the corpus holds markup. That matters twice:
|
|
73
|
+
a quote of visible text does not literally appear in a string
|
|
74
|
+
containing `<em>`, and an embedding of `<div class="nav-wrapper">`
|
|
75
|
+
measures a page's template rather than its content.
|
|
76
|
+
|
|
77
|
+
`script` and `style` bodies are removed WITH their contents; every
|
|
78
|
+
other tag is removed and its text kept. Entities are unescaped
|
|
79
|
+
afterwards, so `&` compares equal to `&`.
|
|
80
|
+
|
|
81
|
+
NOT a readability extractor. Navigation and footers survive, which
|
|
82
|
+
costs a windowed similarity nothing -- an irrelevant window simply
|
|
83
|
+
does not win the max -- and replacing this with real extraction is
|
|
84
|
+
named in §45's "deliberately not" list.
|
|
85
|
+
"""
|
|
86
|
+
if not text:
|
|
87
|
+
return ""
|
|
88
|
+
return _WHITESPACE_RE.sub(" ", html.unescape(_TAG_RE.sub(" ", text))).strip()
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _comparable(text: str) -> str:
|
|
92
|
+
"""Case- and whitespace-insensitive, for the quote containment check."""
|
|
93
|
+
return _WHITESPACE_RE.sub(" ", html.unescape(text or "")).strip().casefold()
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# ---------------------------------------------------------------------------
|
|
97
|
+
# ---- Domain classification (SQ4) ------------------------------------------
|
|
98
|
+
# ---------------------------------------------------------------------------
|
|
99
|
+
|
|
100
|
+
def _hostname(url: str) -> str:
|
|
101
|
+
from urllib.parse import urlsplit
|
|
102
|
+
try:
|
|
103
|
+
host = urlsplit(url).hostname or ""
|
|
104
|
+
except ValueError:
|
|
105
|
+
return ""
|
|
106
|
+
return host.lower().rstrip(".")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def classify_domain(url: str, overrides: dict | None = None) -> tuple[str, float]:
|
|
110
|
+
"""(class name, base authority) for `url`.
|
|
111
|
+
|
|
112
|
+
LONGEST SUFFIX WINS, on label boundaries. `blogs.example.gov` matches
|
|
113
|
+
`gov`; `notagov` does not, because the match requires the suffix to
|
|
114
|
+
start at a label. That boundary is the whole check: a substring test
|
|
115
|
+
would score `evilgov.com` as a government site.
|
|
116
|
+
|
|
117
|
+
`overrides` is the user's `domain_overrides` -- a {suffix: score} map
|
|
118
|
+
merged at the same precedence as the table but winning ties, so adding
|
|
119
|
+
one domain never requires restating the class it belongs to.
|
|
120
|
+
|
|
121
|
+
An unknown host takes `config.DEFAULT_DOMAIN_AUTHORITY` and the class
|
|
122
|
+
name `unknown`, which is the generic-commercial number: a source this
|
|
123
|
+
table has never heard of is ordinary, not suspect.
|
|
124
|
+
"""
|
|
125
|
+
host = _hostname(url)
|
|
126
|
+
if not host:
|
|
127
|
+
return "unknown", config.DEFAULT_DOMAIN_AUTHORITY
|
|
128
|
+
|
|
129
|
+
labels = host.split(".")
|
|
130
|
+
# Every suffix of the host, longest first: a.b.c -> a.b.c, b.c, c
|
|
131
|
+
candidates = [".".join(labels[i:]) for i in range(len(labels))]
|
|
132
|
+
|
|
133
|
+
if overrides:
|
|
134
|
+
for candidate in candidates:
|
|
135
|
+
if candidate in overrides:
|
|
136
|
+
return "override", float(overrides[candidate])
|
|
137
|
+
|
|
138
|
+
for candidate in candidates:
|
|
139
|
+
class_name = config.DOMAIN_AUTHORITY_SUFFIXES.get(candidate)
|
|
140
|
+
if class_name:
|
|
141
|
+
return class_name, config.DOMAIN_AUTHORITY_CLASSES[class_name]
|
|
142
|
+
|
|
143
|
+
# `gov.uk`, `ac.nz`, `edu.au`: a restricted registry under a ccTLD.
|
|
144
|
+
# Checked AFTER the explicit table so a listed host under one of them
|
|
145
|
+
# keeps its own class.
|
|
146
|
+
if len(labels) >= 2 and labels[-2] in _RESTRICTED_SECOND_LEVEL \
|
|
147
|
+
and len(labels[-1]) == 2:
|
|
148
|
+
return ("restricted_registry",
|
|
149
|
+
config.DOMAIN_AUTHORITY_CLASSES["restricted_registry"])
|
|
150
|
+
|
|
151
|
+
return "unknown", config.DEFAULT_DOMAIN_AUTHORITY
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
# ---------------------------------------------------------------------------
|
|
155
|
+
# ---- Per-source scoring ---------------------------------------------------
|
|
156
|
+
# ---------------------------------------------------------------------------
|
|
157
|
+
|
|
158
|
+
def _clamp(value: float, low: float = 0.0, high: float = 1.0) -> float:
|
|
159
|
+
return max(low, min(high, value))
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _as_number(value):
|
|
163
|
+
"""`value` as a float, or None. Rejects bools, which are ints in
|
|
164
|
+
Python and would otherwise read `True` as a similarity of 1.0."""
|
|
165
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
166
|
+
return None
|
|
167
|
+
return float(value)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class _Coercions:
|
|
171
|
+
"""What had to be corrected, aggregated by kind.
|
|
172
|
+
|
|
173
|
+
ONE TRACE LINE PER KIND rather than per source. `_report_coercions`
|
|
174
|
+
in confidence_scoring.py names the individual claim, which is right
|
|
175
|
+
there -- Pass 5 touches each claim once. This stage touches every
|
|
176
|
+
SOURCE of every claim, so per-source lines would put sixty entries in
|
|
177
|
+
a trace that currently holds twenty, and a trace nobody finishes
|
|
178
|
+
reading is the same as no trace. The count and the first few examples
|
|
179
|
+
are what a reader acts on.
|
|
180
|
+
"""
|
|
181
|
+
|
|
182
|
+
EXAMPLES = 3
|
|
183
|
+
|
|
184
|
+
def __init__(self) -> None:
|
|
185
|
+
self.kinds: dict = {}
|
|
186
|
+
|
|
187
|
+
def note(self, kind: str, where: str) -> None:
|
|
188
|
+
count, examples = self.kinds.get(kind, (0, []))
|
|
189
|
+
if len(examples) < self.EXAMPLES:
|
|
190
|
+
examples.append(where)
|
|
191
|
+
self.kinds[kind] = (count + 1, examples)
|
|
192
|
+
|
|
193
|
+
def report(self, run) -> None:
|
|
194
|
+
for kind, (count, examples) in sorted(self.kinds.items()):
|
|
195
|
+
run.log(f"Pass 3a scoring: {count} source(s) {kind} "
|
|
196
|
+
f"(e.g. {', '.join(examples)}).")
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def score_source(source: dict, claim, corpus, coercions: _Coercions,
|
|
200
|
+
overrides: dict | None = None,
|
|
201
|
+
authority_adjustment_cap: float | None = None) -> dict | None:
|
|
202
|
+
"""One cited source, scored. None if it cannot be scored at all.
|
|
203
|
+
|
|
204
|
+
Returns a NEW dict rather than mutating in place: the model's entry is
|
|
205
|
+
the input to this function, and a half-rewritten source -- some fields
|
|
206
|
+
replaced, some left as the model sent them -- is exactly the shape
|
|
207
|
+
that makes an artifact impossible to read. Every key the result
|
|
208
|
+
carries was written here.
|
|
209
|
+
"""
|
|
210
|
+
if not isinstance(source, dict):
|
|
211
|
+
coercions.note("were not objects and were dropped", claim.id)
|
|
212
|
+
return None
|
|
213
|
+
|
|
214
|
+
url = source.get("url")
|
|
215
|
+
if not isinstance(url, str) or not url.strip():
|
|
216
|
+
coercions.note("had no usable url and were dropped", claim.id)
|
|
217
|
+
return None
|
|
218
|
+
url = url.strip()
|
|
219
|
+
|
|
220
|
+
scored = {"url": url}
|
|
221
|
+
_score_relevance(source, scored, corpus, claim, coercions)
|
|
222
|
+
_score_authority(source, scored, claim, coercions, overrides,
|
|
223
|
+
authority_adjustment_cap)
|
|
224
|
+
return scored
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _score_relevance(source: dict, scored: dict, corpus, claim,
|
|
228
|
+
coercions: _Coercions) -> None:
|
|
229
|
+
"""The quote, whether it is real, and how close it is.
|
|
230
|
+
|
|
231
|
+
`similarity_score` is the model's for now, clamped, and marked as
|
|
232
|
+
such. The embedder replaces the number and the method; the rubric in
|
|
233
|
+
source_grounding.md is what makes the fallback worth having.
|
|
234
|
+
"""
|
|
235
|
+
quote = source.get("quote")
|
|
236
|
+
if quote is not None and not isinstance(quote, str):
|
|
237
|
+
coercions.note("had a non-string quote", claim.id)
|
|
238
|
+
quote = None
|
|
239
|
+
scored["quote"] = (quote or "").strip()
|
|
240
|
+
|
|
241
|
+
scored["quote_verified"] = _verify_quote(scored["quote"], scored["url"],
|
|
242
|
+
corpus)
|
|
243
|
+
|
|
244
|
+
reported = _as_number(source.get("similarity_score"))
|
|
245
|
+
if reported is None:
|
|
246
|
+
coercions.note("gave no usable similarity_score", claim.id)
|
|
247
|
+
scored["similarity_score_llm"] = None
|
|
248
|
+
scored["similarity_score"] = 0.0
|
|
249
|
+
else:
|
|
250
|
+
if not 0.0 <= reported <= 1.0:
|
|
251
|
+
coercions.note("gave a similarity_score outside 0-1", claim.id)
|
|
252
|
+
scored["similarity_score_llm"] = _clamp(reported)
|
|
253
|
+
scored["similarity_score"] = scored["similarity_score_llm"]
|
|
254
|
+
scored["similarity_method"] = "llm"
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _verify_quote(quote: str, url: str, corpus):
|
|
258
|
+
"""True, False, or None -- and None is a third answer, not a default.
|
|
259
|
+
|
|
260
|
+
None means THERE WAS NOTHING TO CHECK AGAINST: no corpus, no captured
|
|
261
|
+
text for this URL (the model cited a search result it never opened, or
|
|
262
|
+
fetched it during a JSON-retry turn, which is not translated), or no
|
|
263
|
+
quote. False means the run retrieved this page and the quoted span is
|
|
264
|
+
not in it, which is the fabricated-citation signal.
|
|
265
|
+
|
|
266
|
+
STRICT CONTAINMENT, after markup stripping, whitespace collapsing,
|
|
267
|
+
entity unescaping and case folding. It still has false negatives -- a
|
|
268
|
+
model that silently fixes a typo, or quotes across an element the
|
|
269
|
+
stripper turned into a space, fails a check it should pass -- so
|
|
270
|
+
`False` is a flag for a person, and §45 slice A deliberately does not
|
|
271
|
+
let it move any score. Loosening it to token overlap would make it
|
|
272
|
+
pass for a fabricated quote about the right topic, which is the one
|
|
273
|
+
thing it exists to catch.
|
|
274
|
+
"""
|
|
275
|
+
if corpus is None or not quote:
|
|
276
|
+
return None
|
|
277
|
+
document = corpus.get(url)
|
|
278
|
+
if document is None:
|
|
279
|
+
return None
|
|
280
|
+
return _comparable(quote) in _comparable(strip_markup(document.text))
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def _score_authority(source: dict, scored: dict, claim,
|
|
284
|
+
coercions: _Coercions, overrides: dict | None,
|
|
285
|
+
adjustment_cap: float | None = None) -> None:
|
|
286
|
+
class_name, base = classify_domain(scored["url"], overrides)
|
|
287
|
+
scored["authority_class"] = class_name
|
|
288
|
+
scored["authority_base"] = round(base, 4)
|
|
289
|
+
scored["authority_method"] = "domain"
|
|
290
|
+
|
|
291
|
+
cap = (config.AUTHORITY_ADJUSTMENT_CAP if adjustment_cap is None
|
|
292
|
+
else adjustment_cap)
|
|
293
|
+
adjustment = _as_number(source.get("authority_adjustment")) or 0.0
|
|
294
|
+
reason = source.get("authority_reason")
|
|
295
|
+
reason = reason.strip() if isinstance(reason, str) else ""
|
|
296
|
+
|
|
297
|
+
if adjustment and not reason:
|
|
298
|
+
# DISCARDED, not clamped. The reason is what makes the adjustment
|
|
299
|
+
# auditable, and an unexplained nudge is indistinguishable from the
|
|
300
|
+
# invented number this section replaced.
|
|
301
|
+
coercions.note("adjusted authority with no reason, so it was dropped",
|
|
302
|
+
claim.id)
|
|
303
|
+
adjustment = 0.0
|
|
304
|
+
reason = ""
|
|
305
|
+
elif abs(adjustment) > cap:
|
|
306
|
+
coercions.note(f"adjusted authority beyond +/-{cap}", claim.id)
|
|
307
|
+
adjustment = _clamp(adjustment, -cap, cap)
|
|
308
|
+
|
|
309
|
+
scored["authority_adjustment"] = round(adjustment, 4)
|
|
310
|
+
scored["authority_reason"] = reason if adjustment else ""
|
|
311
|
+
scored["authority_score"] = round(_clamp(base + adjustment), 4)
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
# ===========================================================================
|
|
315
|
+
# ---- Similarity: chunking, cosine, calibration (SQ2) ----------------------
|
|
316
|
+
# ===========================================================================
|
|
317
|
+
|
|
318
|
+
# --- Tunable -- change these, not the logic below, to retune similarity ---
|
|
319
|
+
#
|
|
320
|
+
# THE ONE IDEA HERE IS MATCHING GRANULARITY. A claim is one sentence; a
|
|
321
|
+
# fetched page is up to 5000 characters. Embedding the page whole averages
|
|
322
|
+
# twenty topics into one vector, and the claim's topic contributes a few
|
|
323
|
+
# percent of its direction -- that IS the dilution, and it makes every long
|
|
324
|
+
# source look equally irrelevant. A window of roughly a claim's own length
|
|
325
|
+
# asks the question grounding actually asks: does this source contain a
|
|
326
|
+
# passage that states this.
|
|
327
|
+
WINDOW_CHARS = 480 # ~120 tokens, the same order as a claim
|
|
328
|
+
WINDOW_OVERLAP_SENTENCES = 1 # so a claim straddling a boundary is not lost
|
|
329
|
+
MAX_WINDOWS = 24 # more than fetch_url's 5000 chars can produce
|
|
330
|
+
|
|
331
|
+
# Below this, a claim is short enough to be pronoun-bearing ("It was signed
|
|
332
|
+
# in 1947") despite Pass 2 being asked for self-contained ones, so its
|
|
333
|
+
# entities are prepended. Above it, adding them dilutes the query with the
|
|
334
|
+
# very topic words every window shares.
|
|
335
|
+
CLAIM_MIN_CHARS = 80
|
|
336
|
+
|
|
337
|
+
# How many windows the recorded `cosine_top_mean` averages. It is NOT the
|
|
338
|
+
# score -- max is -- and exists so a reader can tell one matching sentence
|
|
339
|
+
# from a document that is wholly on point.
|
|
340
|
+
TOP_K = 3
|
|
341
|
+
|
|
342
|
+
# Inputs per embeddings request. Providers cap both the number of inputs
|
|
343
|
+
# and the tokens per input; this is well under every published limit and
|
|
344
|
+
# keeps one failed request from costing a whole run's vectors.
|
|
345
|
+
EMBED_BATCH_SIZE = 96
|
|
346
|
+
|
|
347
|
+
# Retries per batch. Immediate, with no backoff, following arxiv.py: a
|
|
348
|
+
# transient failure is worth one more try, and sleeping inside a research
|
|
349
|
+
# pass buys little while making the suite slow.
|
|
350
|
+
EMBED_MAX_RETRIES = 2
|
|
351
|
+
|
|
352
|
+
_SENTENCE_RE = re.compile(r"(?<=[.!?])\s+|\n+")
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def split_sentences(text: str) -> list:
|
|
356
|
+
"""Rough sentence split, deliberately.
|
|
357
|
+
|
|
358
|
+
An abbreviation ("Dr. Smith") splits where it should not, and the cost
|
|
359
|
+
of that is one window boundary in the wrong place -- which a MAX over
|
|
360
|
+
overlapping windows absorbs. A real sentence tokenizer is a dependency,
|
|
361
|
+
and #145's posture is that a dependency earns its place by fixing
|
|
362
|
+
something measured.
|
|
363
|
+
"""
|
|
364
|
+
return [part.strip() for part in _SENTENCE_RE.split(text or "") if part.strip()]
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def chunk_text(text: str, window_chars: int = WINDOW_CHARS,
|
|
368
|
+
max_windows: int = MAX_WINDOWS,
|
|
369
|
+
overlap: int = WINDOW_OVERLAP_SENTENCES) -> list:
|
|
370
|
+
"""`text` as overlapping windows of about `window_chars`.
|
|
371
|
+
|
|
372
|
+
Sentences are packed greedily and windows overlap by `overlap`
|
|
373
|
+
sentences, so a claim stated across a boundary still lands whole
|
|
374
|
+
inside some window. A single sentence longer than the budget is hard
|
|
375
|
+
split rather than emitted oversized -- one 4000-character
|
|
376
|
+
"sentence" (an unpunctuated navigation blob, which raw HTML produces
|
|
377
|
+
constantly) would otherwise become one diluted window and crowd out
|
|
378
|
+
the rest of the page.
|
|
379
|
+
"""
|
|
380
|
+
cleaned = strip_markup(text)
|
|
381
|
+
if not cleaned:
|
|
382
|
+
return []
|
|
383
|
+
|
|
384
|
+
sentences = []
|
|
385
|
+
for sentence in split_sentences(cleaned):
|
|
386
|
+
while len(sentence) > window_chars:
|
|
387
|
+
sentences.append(sentence[:window_chars])
|
|
388
|
+
sentence = sentence[window_chars:]
|
|
389
|
+
if sentence:
|
|
390
|
+
sentences.append(sentence)
|
|
391
|
+
if not sentences:
|
|
392
|
+
return []
|
|
393
|
+
|
|
394
|
+
windows = []
|
|
395
|
+
start = 0
|
|
396
|
+
while start < len(sentences) and len(windows) < max_windows:
|
|
397
|
+
end, length = start, 0
|
|
398
|
+
while end < len(sentences):
|
|
399
|
+
addition = len(sentences[end]) + (1 if end > start else 0)
|
|
400
|
+
if end > start and length + addition > window_chars:
|
|
401
|
+
break
|
|
402
|
+
length += addition
|
|
403
|
+
end += 1
|
|
404
|
+
windows.append(" ".join(sentences[start:end]))
|
|
405
|
+
if end >= len(sentences):
|
|
406
|
+
break
|
|
407
|
+
# At least one sentence forward, so an overlap wider than the
|
|
408
|
+
# window cannot stall the loop.
|
|
409
|
+
start = max(start + 1, end - overlap)
|
|
410
|
+
return windows
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def l2_normalize(vector) -> list:
|
|
414
|
+
"""Unit length, so cosine is a dot product.
|
|
415
|
+
|
|
416
|
+
A zero vector is returned unchanged rather than divided by zero. It
|
|
417
|
+
then scores 0.0 against everything, which is the honest reading of a
|
|
418
|
+
vector carrying no direction -- and `embed_texts` already refuses a
|
|
419
|
+
response that is all zeros, so this is the single-vector case.
|
|
420
|
+
"""
|
|
421
|
+
norm = math.sqrt(sum(component * component for component in vector))
|
|
422
|
+
if norm == 0.0:
|
|
423
|
+
return list(vector)
|
|
424
|
+
return [component / norm for component in vector]
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def cosine(left, right) -> float:
|
|
428
|
+
"""Cosine similarity of two ALREADY-NORMALIZED vectors.
|
|
429
|
+
|
|
430
|
+
Pure Python on purpose: `numpy` is not a dependency, and a few hundred
|
|
431
|
+
1536-dimensional dot products is microseconds. Mismatched lengths
|
|
432
|
+
score 0.0 rather than raising -- `embed_texts` rejects a ragged batch
|
|
433
|
+
at the boundary, so reaching here means two vectors from different
|
|
434
|
+
calls, and failing a run over it would undo SQ10's whole contract.
|
|
435
|
+
"""
|
|
436
|
+
if len(left) != len(right) or not left:
|
|
437
|
+
return 0.0
|
|
438
|
+
return sum(a * b for a, b in zip(left, right))
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def calibrate(raw: float, floor: float, ceiling: float) -> float:
|
|
442
|
+
"""Raw cosine onto 0-1, using this model's useful band.
|
|
443
|
+
|
|
444
|
+
Unrelated text sits near 0.1 on some embedders and above 0.7 on
|
|
445
|
+
others, so the same passage would read as "irrelevant" on one and
|
|
446
|
+
"supports the claim" on another. An inverted or degenerate band
|
|
447
|
+
(floor >= ceiling) falls back to passing the cosine through clamped,
|
|
448
|
+
because refusing to score is worse than scoring on an uncalibrated
|
|
449
|
+
scale that the recorded raw cosine still explains.
|
|
450
|
+
"""
|
|
451
|
+
if ceiling <= floor:
|
|
452
|
+
return _clamp(raw)
|
|
453
|
+
return _clamp((raw - floor) / (ceiling - floor))
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def _table_lookup(table: dict, model: str, default):
|
|
457
|
+
"""Longest matching slug substring wins, else `default`."""
|
|
458
|
+
best = None
|
|
459
|
+
for fragment, value in table.items():
|
|
460
|
+
if fragment in model and (best is None or len(fragment) > len(best[0])):
|
|
461
|
+
best = (fragment, value)
|
|
462
|
+
return best[1] if best else default
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def calibration_for(model: str) -> tuple:
|
|
466
|
+
band = _table_lookup(config.SIMILARITY_CALIBRATION, model,
|
|
467
|
+
config.DEFAULT_SIMILARITY_CALIBRATION)
|
|
468
|
+
return float(band["floor"]), float(band["ceiling"])
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def prefixes_for(model: str) -> dict:
|
|
472
|
+
return _table_lookup(config.EMBEDDER_PREFIXES, model, {})
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
class EmbeddingScorer:
|
|
476
|
+
"""One run's embedder: batching, caching, and the fallback.
|
|
477
|
+
|
|
478
|
+
STATEFUL AND RUN-SCOPED, because both things it holds are per-run: a
|
|
479
|
+
vector cache keyed by the text itself (a claim is embedded once
|
|
480
|
+
however many sources it has, and a page fetched twice is chunked
|
|
481
|
+
twice into identical windows), and a `failed` flag.
|
|
482
|
+
|
|
483
|
+
ONE FAILURE DISABLES THE SCORER FOR THE REST OF THE RUN, after its
|
|
484
|
+
retries. That is SQ10 taken seriously rather than half-applied: a
|
|
485
|
+
provider that is down stays down for the next forty batches, and
|
|
486
|
+
retrying each of them turns one outage into a run that takes minutes
|
|
487
|
+
to produce the fallback it was always going to produce. Every source
|
|
488
|
+
then carries `similarity_method: "llm"`, and the trace says when the
|
|
489
|
+
switch happened.
|
|
490
|
+
"""
|
|
491
|
+
|
|
492
|
+
def __init__(self, provider_name: str, model: str,
|
|
493
|
+
floor: float | None = None,
|
|
494
|
+
ceiling: float | None = None) -> None:
|
|
495
|
+
self.provider_name = provider_name
|
|
496
|
+
self.model = model
|
|
497
|
+
self.failed_reason = None
|
|
498
|
+
self.input_tokens = 0
|
|
499
|
+
self.calls = 0
|
|
500
|
+
self._cache: dict = {}
|
|
501
|
+
prefixes = prefixes_for(model)
|
|
502
|
+
self._query_prefix = prefixes.get("query", "")
|
|
503
|
+
self._passage_prefix = prefixes.get("passage", "")
|
|
504
|
+
self._query_task = prefixes.get("query_task_type", "")
|
|
505
|
+
self._passage_task = prefixes.get("passage_task_type", "")
|
|
506
|
+
table_floor, table_ceiling = calibration_for(model)
|
|
507
|
+
# A band the user set OUTRANKS the table, model_windows' inversion
|
|
508
|
+
# for model_windows' reason: the table cannot know about a model
|
|
509
|
+
# shipped after this release, and someone who measured their own
|
|
510
|
+
# embedder's floor is stating a fact about the deployment.
|
|
511
|
+
self.floor = table_floor if floor is None else floor
|
|
512
|
+
self.ceiling = table_ceiling if ceiling is None else ceiling
|
|
513
|
+
|
|
514
|
+
@property
|
|
515
|
+
def usable(self) -> bool:
|
|
516
|
+
return self.failed_reason is None
|
|
517
|
+
|
|
518
|
+
def _key(self, text: str, is_query: bool) -> str:
|
|
519
|
+
# The ROLE is part of the key: an asymmetric embedder gives the
|
|
520
|
+
# same string two different vectors depending on which prefix it
|
|
521
|
+
# was sent with, and collapsing them would silently score a claim
|
|
522
|
+
# against a passage-space vector of itself.
|
|
523
|
+
role = "q" if is_query else "p"
|
|
524
|
+
return f"{role}:{hashlib.sha256(text.encode('utf-8')).hexdigest()}"
|
|
525
|
+
|
|
526
|
+
def prime(self, texts, is_query: bool) -> None:
|
|
527
|
+
"""Embed everything not already cached, in batches.
|
|
528
|
+
|
|
529
|
+
Called once per role per stage so the whole run's claims are one
|
|
530
|
+
request and the whole run's windows are a handful -- rather than
|
|
531
|
+
one request per source, which is the shape that makes an embedder
|
|
532
|
+
slower than the pass it is scoring.
|
|
533
|
+
"""
|
|
534
|
+
if not self.usable:
|
|
535
|
+
return
|
|
536
|
+
pending, seen = [], set()
|
|
537
|
+
for text in texts:
|
|
538
|
+
key = self._key(text, is_query)
|
|
539
|
+
if key in self._cache or key in seen:
|
|
540
|
+
continue
|
|
541
|
+
seen.add(key)
|
|
542
|
+
pending.append(text)
|
|
543
|
+
|
|
544
|
+
prefix = self._query_prefix if is_query else self._passage_prefix
|
|
545
|
+
task = self._query_task if is_query else self._passage_task
|
|
546
|
+
for start in range(0, len(pending), EMBED_BATCH_SIZE):
|
|
547
|
+
batch = pending[start:start + EMBED_BATCH_SIZE]
|
|
548
|
+
vectors = self._embed(batch, prefix, task)
|
|
549
|
+
if vectors is None:
|
|
550
|
+
return
|
|
551
|
+
for text, vector in zip(batch, vectors):
|
|
552
|
+
self._cache[self._key(text, is_query)] = l2_normalize(vector)
|
|
553
|
+
|
|
554
|
+
def _embed(self, batch, prefix, task):
|
|
555
|
+
"""The batch, or None once the scorer has given up on this run."""
|
|
556
|
+
from core.client import EmbeddingError, embed_texts
|
|
557
|
+
|
|
558
|
+
last = None
|
|
559
|
+
for _ in range(EMBED_MAX_RETRIES + 1):
|
|
560
|
+
try:
|
|
561
|
+
result = embed_texts(self.provider_name, self.model, batch,
|
|
562
|
+
prefix=prefix, task_type=task)
|
|
563
|
+
except EmbeddingError as exc:
|
|
564
|
+
last = exc
|
|
565
|
+
continue
|
|
566
|
+
self.calls += 1
|
|
567
|
+
self.input_tokens += result.usage.get("input_tokens", 0)
|
|
568
|
+
return result.vectors
|
|
569
|
+
self.failed_reason = str(last)
|
|
570
|
+
return None
|
|
571
|
+
|
|
572
|
+
def vector(self, text: str, is_query: bool):
|
|
573
|
+
return self._cache.get(self._key(text, is_query))
|
|
574
|
+
|
|
575
|
+
def score(self, query: str, passages) -> dict | None:
|
|
576
|
+
"""The best window's calibrated similarity, or None.
|
|
577
|
+
|
|
578
|
+
MAX, not mean. Grounding asks whether the source contains a
|
|
579
|
+
passage that states the claim, which is a max question; a mean
|
|
580
|
+
over a long page answers "is this page broadly about the topic",
|
|
581
|
+
which is a different and weaker question. `cosine_top_mean` is
|
|
582
|
+
recorded beside it so a reader can tell one matching sentence from
|
|
583
|
+
a document that is wholly on point.
|
|
584
|
+
"""
|
|
585
|
+
if not self.usable or not passages:
|
|
586
|
+
return None
|
|
587
|
+
query_vector = self.vector(query, True)
|
|
588
|
+
if query_vector is None:
|
|
589
|
+
return None
|
|
590
|
+
scores = [cosine(query_vector, vector) for vector in
|
|
591
|
+
(self.vector(p, False) for p in passages)
|
|
592
|
+
if vector is not None]
|
|
593
|
+
if not scores:
|
|
594
|
+
return None
|
|
595
|
+
scores.sort(reverse=True)
|
|
596
|
+
top = scores[:TOP_K]
|
|
597
|
+
return {
|
|
598
|
+
"cosine": round(scores[0], 4),
|
|
599
|
+
"cosine_top_mean": round(sum(top) / len(top), 4),
|
|
600
|
+
"windows_scored": len(scores),
|
|
601
|
+
"similarity_score": round(
|
|
602
|
+
calibrate(scores[0], self.floor, self.ceiling), 4),
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
def claim_query_text(claim, min_chars: int = CLAIM_MIN_CHARS) -> str:
|
|
607
|
+
"""What gets embedded for a claim.
|
|
608
|
+
|
|
609
|
+
Its text verbatim -- Pass 2 produces atomic, self-contained claims, so
|
|
610
|
+
there is nothing to add. The exception is a claim short enough to be
|
|
611
|
+
pronoun-bearing anyway ("It was signed in 1947"), where the entities
|
|
612
|
+
Pass 2 already extracted are the missing subject. Prepending them to
|
|
613
|
+
EVERY claim would be worse: the entity words appear in every window of
|
|
614
|
+
the source too, so they raise every score alike and compress the range
|
|
615
|
+
the calibration band depends on.
|
|
616
|
+
|
|
617
|
+
THE REVISION WINS WHERE THERE IS ONE. Pass 6b re-validates the claim
|
|
618
|
+
Pass 6a rewrote, and `revalidate.md` says so in as many words -- but
|
|
619
|
+
`claim.text` still holds the ORIGINAL, because 6c is what finally
|
|
620
|
+
merges a revision into `final_text`. Scoring against `claim.text` on a
|
|
621
|
+
retry round measures a source against a sentence the claim no longer
|
|
622
|
+
asserts, and the vector cache makes it silent: the query is unchanged,
|
|
623
|
+
so it is a cache hit and the round costs nothing and reports nothing.
|
|
624
|
+
Found by running the pipeline end to end and reading the trace.
|
|
625
|
+
"""
|
|
626
|
+
text = (claim.revision_text or claim.text or "").strip()
|
|
627
|
+
if len(text) >= min_chars or not claim.entities:
|
|
628
|
+
return text
|
|
629
|
+
return f"{', '.join(str(e) for e in claim.entities)} — {text}"
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
def source_passages(scored: dict, corpus, window_chars: int = WINDOW_CHARS,
|
|
633
|
+
max_windows: int = MAX_WINDOWS) -> list:
|
|
634
|
+
"""The windows a source is scored over.
|
|
635
|
+
|
|
636
|
+
THE QUOTE IS ITS OWN WINDOW, first. It is the passage the model said
|
|
637
|
+
it relied on, so it is the most specific evidence available -- and
|
|
638
|
+
scoring it directly means a source whose captured text was truncated
|
|
639
|
+
before the relevant paragraph still scores on the part that mattered.
|
|
640
|
+
The document's own windows follow, so a fabricated or badly chosen
|
|
641
|
+
quote cannot LOWER a score the page itself earns.
|
|
642
|
+
"""
|
|
643
|
+
passages = []
|
|
644
|
+
quote = scored.get("quote")
|
|
645
|
+
if quote:
|
|
646
|
+
passages.append(quote)
|
|
647
|
+
document = corpus.get(scored["url"]) if corpus is not None else None
|
|
648
|
+
if document is not None:
|
|
649
|
+
passages.extend(chunk_text(document.text, window_chars, max_windows))
|
|
650
|
+
return passages
|
|
651
|
+
|
|
652
|
+
|
|
653
|
+
# ---------------------------------------------------------------------------
|
|
654
|
+
# ---- The stage ------------------------------------------------------------
|
|
655
|
+
# ---------------------------------------------------------------------------
|
|
656
|
+
|
|
657
|
+
def make_scorer(run=None, floor: float | None = None,
|
|
658
|
+
ceiling: float | None = None):
|
|
659
|
+
"""The run's EmbeddingScorer, or None with the warning said once.
|
|
660
|
+
|
|
661
|
+
SQ2's fallback is deliberate and announced. A run with no embedder
|
|
662
|
+
still produces a `similarity_score` for every source -- the grounding
|
|
663
|
+
model's own, under source_grounding.md's anchored bands -- and the
|
|
664
|
+
artifact marks every one of them `similarity_method: "llm"`. What must
|
|
665
|
+
not happen is a reader assuming the number is a measurement. So the
|
|
666
|
+
trace says so at the top of the run rather than leaving it to be
|
|
667
|
+
inferred from a field.
|
|
668
|
+
"""
|
|
669
|
+
from core import pipeline_models
|
|
670
|
+
|
|
671
|
+
chosen = pipeline_models.resolve("embedder")
|
|
672
|
+
if chosen is None:
|
|
673
|
+
if run is not None:
|
|
674
|
+
run.log(
|
|
675
|
+
"Source scoring: no embedder configured, so similarity is "
|
|
676
|
+
"the grounding model's own estimate of sources it chose "
|
|
677
|
+
"(similarity_method=\"llm\" on every source). Set one with "
|
|
678
|
+
"/embedder for a measured cosine.")
|
|
679
|
+
return None
|
|
680
|
+
return EmbeddingScorer(chosen["provider_name"], chosen["model"],
|
|
681
|
+
floor=floor, ceiling=ceiling)
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
def _apply_embeddings(run, claims, corpus, scorer, window_chars, max_windows,
|
|
685
|
+
claim_min_chars) -> None:
|
|
686
|
+
"""Replace every scorable source's similarity with a cosine.
|
|
687
|
+
|
|
688
|
+
TWO PHASES, and that is the whole reason this is not done inside
|
|
689
|
+
`score_source`. Every text the run needs is embedded first, in
|
|
690
|
+
batches, then every pair is scored from the cache -- so a run with
|
|
691
|
+
twenty claims and sixty sources makes a handful of requests instead of
|
|
692
|
+
sixty. Per-source calls would make the embedder slower than the pass
|
|
693
|
+
it is scoring, which is how a good measurement gets turned off.
|
|
694
|
+
"""
|
|
695
|
+
plan = []
|
|
696
|
+
for claim in claims:
|
|
697
|
+
query = claim_query_text(claim, claim_min_chars)
|
|
698
|
+
if not query:
|
|
699
|
+
continue
|
|
700
|
+
for scored in claim.grounding_sources:
|
|
701
|
+
passages = source_passages(scored, corpus, window_chars,
|
|
702
|
+
max_windows)
|
|
703
|
+
if passages:
|
|
704
|
+
plan.append((query, scored, passages))
|
|
705
|
+
if not plan:
|
|
706
|
+
return
|
|
707
|
+
|
|
708
|
+
scorer.prime([query for query, _, _ in plan], is_query=True)
|
|
709
|
+
scorer.prime([p for _, _, passages in plan for p in passages],
|
|
710
|
+
is_query=False)
|
|
711
|
+
|
|
712
|
+
measured = 0
|
|
713
|
+
for query, scored, passages in plan:
|
|
714
|
+
result = scorer.score(query, passages)
|
|
715
|
+
if result is None:
|
|
716
|
+
continue
|
|
717
|
+
scored.update(result)
|
|
718
|
+
scored["similarity_method"] = "embedding"
|
|
719
|
+
scored["embedder"] = f"{scorer.provider_name}|{scorer.model}"
|
|
720
|
+
measured += 1
|
|
721
|
+
|
|
722
|
+
if scorer.usable:
|
|
723
|
+
# THE COUNTERS ARE CUMULATIVE, and the line says so. This stage
|
|
724
|
+
# runs once per grounding -- Pass 3a and then every 6b round -- so
|
|
725
|
+
# a line reading "in 2 calls, 30 tokens" three times invites a
|
|
726
|
+
# reader to total it to six calls and ninety tokens, when the
|
|
727
|
+
# scorer's cache means the later rounds cost nothing. Under-
|
|
728
|
+
# reporting spend is the wrong direction to be ambiguous in.
|
|
729
|
+
run.log(
|
|
730
|
+
f"Source scoring: {measured} source(s) scored by cosine against "
|
|
731
|
+
f"{scorer.model} (run total so far: {scorer.calls} call(s), "
|
|
732
|
+
f"{scorer.input_tokens} embedding token(s)).")
|
|
733
|
+
else:
|
|
734
|
+
# NAMED, and the count says how far it got. A run where the
|
|
735
|
+
# embedder died after twelve sources has twelve measured scores
|
|
736
|
+
# and the rest self-reported, and an artifact whose readers cannot
|
|
737
|
+
# tell which is which is the thing similarity_method exists for.
|
|
738
|
+
run.log(
|
|
739
|
+
f"Source scoring: the embedder failed and the rest of this run "
|
|
740
|
+
f"falls back to the grounding model's own similarity "
|
|
741
|
+
f"({measured} source(s) measured before it stopped). "
|
|
742
|
+
f"Cause: {scorer.failed_reason}")
|
|
743
|
+
|
|
744
|
+
|
|
745
|
+
def _apply_scholar(run, claims, scholar) -> None:
|
|
746
|
+
"""Replace a cited paper's domain class with what OpenAlex knows.
|
|
747
|
+
|
|
748
|
+
A THIRD PHASE for `_apply_embeddings`' reason: it makes network calls,
|
|
749
|
+
and the per-source pass has to stay a pure rewrite of what the model
|
|
750
|
+
sent. It runs AFTER that pass so it is handed urls that have already
|
|
751
|
+
been coerced into shape.
|
|
752
|
+
|
|
753
|
+
A source that is not a resolvable paper keeps its domain class and
|
|
754
|
+
gains nothing, which is the right answer -- `authority_method` says
|
|
755
|
+
which of the two produced the number, so no reader has to guess.
|
|
756
|
+
|
|
757
|
+
THE ADJUSTMENT IS REAPPLIED, not discarded. The model's bounded nudge
|
|
758
|
+
is about THIS PAGE -- a retracted-but-uncorrected posting, an author
|
|
759
|
+
manuscript that differs from the published version -- and it is as
|
|
760
|
+
true of a paper as of any other source. Re-derived from the recorded
|
|
761
|
+
adjustment rather than kept, because the base underneath it changed.
|
|
762
|
+
"""
|
|
763
|
+
upgraded = 0
|
|
764
|
+
for claim in claims:
|
|
765
|
+
for scored in claim.grounding_sources:
|
|
766
|
+
if not scholar.usable:
|
|
767
|
+
break
|
|
768
|
+
authority, signals = scholar.authority_for(scored["url"])
|
|
769
|
+
if authority is None:
|
|
770
|
+
continue
|
|
771
|
+
scored["authority_base"] = round(authority, 4)
|
|
772
|
+
scored["authority_method"] = "scholar"
|
|
773
|
+
scored["scholar"] = signals
|
|
774
|
+
scored["authority_score"] = round(
|
|
775
|
+
_clamp(authority + scored.get("authority_adjustment", 0.0)), 4)
|
|
776
|
+
upgraded += 1
|
|
777
|
+
|
|
778
|
+
if upgraded:
|
|
779
|
+
run.log(f"Source scoring: {upgraded} cited paper(s) scored from "
|
|
780
|
+
f"OpenAlex (venue, citation percentile within their own "
|
|
781
|
+
f"field-and-year cohort, author standing) in "
|
|
782
|
+
f"{scholar.calls} lookup(s).")
|
|
783
|
+
if scholar.failed_reason:
|
|
784
|
+
run.log(f"Source scoring: the scholarly lookup stopped after "
|
|
785
|
+
f"{upgraded} paper(s); the rest keep their domain class. "
|
|
786
|
+
f"Cause: {scholar.failed_reason}")
|
|
787
|
+
|
|
788
|
+
|
|
789
|
+
def score_grounding_sources(run, claims=None, corpus=None,
|
|
790
|
+
overrides: dict | None = None,
|
|
791
|
+
scorer=None,
|
|
792
|
+
window_chars: int = WINDOW_CHARS,
|
|
793
|
+
max_windows: int = MAX_WINDOWS,
|
|
794
|
+
claim_min_chars: int = CLAIM_MIN_CHARS,
|
|
795
|
+
authority_adjustment_cap: float | None = None,
|
|
796
|
+
scholar=None) -> int:
|
|
797
|
+
"""Rewrite every claim's `grounding_sources` with computed scores.
|
|
798
|
+
|
|
799
|
+
Returns the number of sources scored. Called from
|
|
800
|
+
`orchestrator._ground_and_score`, which is the one seam Pass 3a and
|
|
801
|
+
Pass 6b's re-validation both go through -- so a source re-grounded
|
|
802
|
+
after a revision is re-scored against the revised claim rather than
|
|
803
|
+
keeping a number computed for text that no longer exists.
|
|
804
|
+
|
|
805
|
+
`claims` defaults to every claim in the run. Non-factual claims never
|
|
806
|
+
reach Pass 3a, so their source lists are empty and iterating them
|
|
807
|
+
costs nothing; narrowing the default would be a second rule to keep in
|
|
808
|
+
step with gate D0's routing for no gain.
|
|
809
|
+
"""
|
|
810
|
+
coercions = _Coercions()
|
|
811
|
+
scored_count = 0
|
|
812
|
+
targets = list(run.claims if claims is None else claims)
|
|
813
|
+
|
|
814
|
+
for claim in targets:
|
|
815
|
+
sources = claim.grounding_sources
|
|
816
|
+
if not isinstance(sources, list):
|
|
817
|
+
# The payload boundary type-guards this at the pass that sent
|
|
818
|
+
# it; a value arriving here is one that came from some other
|
|
819
|
+
# path. Emptied rather than iterated, because iterating a
|
|
820
|
+
# string scores one source per character.
|
|
821
|
+
coercions.note("arrived as a non-list source collection",
|
|
822
|
+
claim.id)
|
|
823
|
+
claim.grounding_sources = []
|
|
824
|
+
continue
|
|
825
|
+
|
|
826
|
+
rewritten = []
|
|
827
|
+
for source in sources:
|
|
828
|
+
scored = score_source(source, claim, corpus, coercions,
|
|
829
|
+
overrides, authority_adjustment_cap)
|
|
830
|
+
if scored is not None:
|
|
831
|
+
rewritten.append(scored)
|
|
832
|
+
claim.grounding_sources = rewritten
|
|
833
|
+
scored_count += len(rewritten)
|
|
834
|
+
|
|
835
|
+
if claim.grounding_status in ("grounded", "partial") and not rewritten:
|
|
836
|
+
# Said out loud rather than scored around. §45 SQ3 gives this
|
|
837
|
+
# claim zero source quality once the modulation lands; until
|
|
838
|
+
# then the trace is the only place it is visible at all.
|
|
839
|
+
coercions.note(
|
|
840
|
+
f"were claimed as {claim.grounding_status} with no scorable "
|
|
841
|
+
f"source", claim.id)
|
|
842
|
+
|
|
843
|
+
# BOTH network phases run AFTER the per-source pass, so each is handed
|
|
844
|
+
# sources whose quotes and urls have already been coerced into shape
|
|
845
|
+
# -- and so a payload the first phase rejected entirely never costs a
|
|
846
|
+
# call. Scholar first: it rewrites `authority_base`, and the trace
|
|
847
|
+
# then reads in the order the numbers were produced.
|
|
848
|
+
if scholar is not None and scholar.usable:
|
|
849
|
+
_apply_scholar(run, targets, scholar)
|
|
850
|
+
if scorer is not None and scorer.usable:
|
|
851
|
+
_apply_embeddings(run, targets, corpus, scorer, window_chars,
|
|
852
|
+
max_windows, claim_min_chars)
|
|
853
|
+
|
|
854
|
+
coercions.report(run)
|
|
855
|
+
return scored_count
|