@yahyahammad/venastine 0.3.8 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -19
- package/CONFIG_ARCHITECTURE.md +2030 -2002
- package/LICENSE +201 -201
- package/README.md +1034 -1034
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -106
- package/agents/builtin/review.md +74 -74
- package/agents/manager.py +261 -261
- package/agents/subagent_tool.py +472 -472
- package/agents/tui_commands.py +238 -146
- package/bin/venastine.mjs +528 -528
- package/bin/venastine.py +71 -71
- package/cli_commands.py +838 -835
- package/config.yaml +13 -0
- package/config_edit.py +1301 -1301
- package/config_schema.py +971 -966
- package/config_update.py +749 -749
- package/core/agent_activity.py +256 -256
- package/core/ansi.py +452 -452
- package/core/approval.py +137 -137
- package/core/call_view.py +364 -364
- package/core/cancel.py +121 -121
- package/core/chatgpt_auth.py +747 -747
- package/core/claims_view.py +107 -107
- package/core/client.py +2715 -2715
- package/core/commands.py +460 -454
- package/core/compaction.py +1735 -1735
- package/core/config_loader.py +1982 -1982
- package/core/connect.py +686 -686
- package/core/events.py +123 -123
- package/core/file_mentions.py +280 -280
- package/core/interaction.py +411 -411
- package/core/loop.py +2566 -2566
- package/core/memory.py +482 -482
- package/core/pipeline_models.py +190 -190
- package/core/provider_errors.py +234 -234
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -484
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/orchestrator.py +1561 -1561
- package/core/reasoning/output_writer.py +345 -345
- package/core/reasoning/payload_validation.py +524 -524
- package/core/reasoning/pipeline_storage.py +427 -427
- package/core/reasoning/scholar.py +458 -458
- package/core/reasoning/source_corpus.py +433 -433
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -315
- package/core/shell_sessions.py +1553 -1553
- package/credentials.py +307 -307
- package/database.py +333 -333
- package/docs/PRIVACY.md +78 -78
- package/docs/SECURITY.md +108 -108
- package/docs/THIRD_PARTY_NOTICES.md +148 -148
- package/env_secrets.py +118 -118
- package/json_store.py +343 -343
- package/logging_setup.py +295 -295
- package/main.py +3433 -3433
- package/mcp_client/client.py +733 -733
- package/mcp_client/config.py +484 -484
- package/mcp_client/registration.py +343 -343
- package/memories/manager.py +147 -147
- package/memories/tui_commands.py +319 -239
- package/package.json +303 -303
- package/project_init/config_files.py +282 -282
- package/project_init/generator.py +728 -728
- package/project_init/manifest.py +418 -418
- package/project_init/tui_commands.py +147 -147
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +17 -17
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -316
- package/providers.json.example +94 -94
- package/pyproject.toml +149 -149
- package/requirements.txt +295 -295
- package/safety/policy_enforcement.py +1082 -1082
- package/security/askpass.py +419 -419
- package/security/capability.py +207 -207
- package/security/confine.py +484 -484
- package/security/hosted.py +182 -182
- package/security/permissions.py +237 -237
- package/security/protected_paths.py +501 -501
- package/security/sandbox.py +4108 -4108
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -110
- package/storage.py +1338 -1338
- package/tools/base.py +465 -465
- package/tools/builtin/_math_common.py +667 -667
- package/tools/builtin/_net_common.py +267 -267
- package/tools/builtin/arxiv.py +681 -681
- package/tools/builtin/fetch_url.py +371 -371
- package/tools/builtin/file_ops.py +720 -720
- package/tools/builtin/geometry.py +184 -184
- package/tools/builtin/load_skill.py +76 -76
- package/tools/builtin/pin.py +227 -227
- package/tools/builtin/probability_stats.py +224 -224
- package/tools/builtin/project_docs.py +464 -464
- package/tools/builtin/remember.py +175 -175
- package/tools/builtin/shell.py +980 -980
- package/tools/builtin/todo.py +149 -149
- package/tools/builtin/web_search.py +328 -328
- package/tools/context.py +159 -159
- package/tools/isolation.py +387 -387
- package/tools/registry.py +1242 -1242
- package/tui/app.py +8318 -8307
- package/tui/app.tcss +747 -739
- package/tui/commands.py +26 -26
- package/tui/diffs.py +286 -286
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -102
- package/tui/markdown.py +1329 -1329
- package/tui/meters.py +364 -364
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -1197
- package/tui/themes.py +599 -599
- package/tui/widgets.py +3813 -3796
|
@@ -1,433 +1,433 @@
|
|
|
1
|
-
"""
|
|
2
|
-
core/reasoning/source_corpus.py
|
|
3
|
-
|
|
4
|
-
What a claim was actually grounded IN, kept for the length of one run.
|
|
5
|
-
|
|
6
|
-
ROADMAP_v2 §45 (SQ2). Pass 3a's `sources` list has always been URLs and two
|
|
7
|
-
model-invented floats. The text those URLs served reached the model, was
|
|
8
|
-
stringified into the pass thread, and was then unreachable: nothing in
|
|
9
|
-
`03_grounding.json` could say what a page had said, so `similarity_score`
|
|
10
|
-
could not be checked against anything and a quote could not be checked at
|
|
11
|
-
all. `fetch_url.py`'s own comment already describes the artifact this
|
|
12
|
-
builds ("output_writer builds each run's sources/ directory from it") --
|
|
13
|
-
that directory has existed and been empty since §12.
|
|
14
|
-
|
|
15
|
-
WHY A SECOND SINK RATHER THAN A CHANGE TO _translate. The orchestrator
|
|
16
|
-
already sees every tool result and deliberately forwards only `ok` plus an
|
|
17
|
-
error string to the UI, because a successful result body is not
|
|
18
|
-
observability. That boundary is unchanged. This is a different consumer
|
|
19
|
-
with a different need, attached beside it -- the corpus is read by the
|
|
20
|
-
scoring stage and written to the run directory, and neither is a
|
|
21
|
-
transcript.
|
|
22
|
-
|
|
23
|
-
REDACTED ON ENTRY, THOUGH IT ARRIVES REDACTED. `registry.dispatch` runs
|
|
24
|
-
every result through `check_output_policy` before the loop emits it, so
|
|
25
|
-
what lands here has already been through `redact_output_text`. It is run
|
|
26
|
-
again anyway, and for `_translate`'s stated reason: one rule owned at one
|
|
27
|
-
boundary -- nothing is stored by this module unredacted -- beats depending
|
|
28
|
-
on a guarantee made two layers away that a later refactor could quietly
|
|
29
|
-
drop. The second pass is a no-op on already-clean text.
|
|
30
|
-
|
|
31
|
-
AND IT IS NOT WHERE §52's SPLIT-CREDENTIAL FIX WENT, deliberately (RB4).
|
|
32
|
-
§51 made this module the one place where two pages of a document become
|
|
33
|
-
adjacent, so a credential straddling a page boundary was rejoined HERE --
|
|
34
|
-
reproduced end to end in batch 106, into `artifact_entries()` and therefore
|
|
35
|
-
into `sources/<sha256>.txt` and the window sent to the embedding provider.
|
|
36
|
-
The register's proposed fix was to re-redact the whole merged text on every
|
|
37
|
-
append. It was declined: `fetch_url` redacts before it cuts now, so the
|
|
38
|
-
halves never exist, and re-scanning up to 20,000 characters per page would
|
|
39
|
-
buy nothing this module can see -- while leaving unfixed the model's own two
|
|
40
|
-
tool results, which the producer-side fix does close.
|
|
41
|
-
|
|
42
|
-
THIS MODULE OWNS URL IDENTITY for the pipeline. Two spellings of one arXiv
|
|
43
|
-
paper are one source, and a model that cites `/abs/2005.14165v3` after
|
|
44
|
-
fetching `/abs/2005.14165` must not be told its own citation has no text.
|
|
45
|
-
`scholar.py` resolves the same URLs and imports `arxiv_id_from_url` from
|
|
46
|
-
here rather than re-deriving it -- a second copy of "what counts as the
|
|
47
|
-
same paper" is how the corpus and the scholarly lookup come to disagree
|
|
48
|
-
about one source in one artifact.
|
|
49
|
-
"""
|
|
50
|
-
|
|
51
|
-
from __future__ import annotations
|
|
52
|
-
|
|
53
|
-
import hashlib
|
|
54
|
-
import logging
|
|
55
|
-
import re
|
|
56
|
-
from dataclasses import dataclass, field
|
|
57
|
-
from datetime import UTC, datetime
|
|
58
|
-
from urllib.parse import urlsplit, urlunsplit
|
|
59
|
-
|
|
60
|
-
from safety.policy_enforcement import redact_output_text
|
|
61
|
-
|
|
62
|
-
logger = logging.getLogger(__name__)
|
|
63
|
-
|
|
64
|
-
# Bounds, not tuning. A research pass calls fetch_url a bounded number of
|
|
65
|
-
# times (§31 budgets), so these are a backstop against a pathological run
|
|
66
|
-
# rather than a knob anyone is expected to turn: the corpus is held in
|
|
67
|
-
# memory for the length of a run and then written to disk.
|
|
68
|
-
#
|
|
69
|
-
# MAX_DOCUMENT_CHARS is deliberately well ABOVE fetch_url's own
|
|
70
|
-
# MAX_CONTENT_CHARS (5000). It bounds this module against a future tool
|
|
71
|
-
# that returns more; it is not a second truncation of the tools that exist,
|
|
72
|
-
# because a cap that bites in normal operation would silently shorten the
|
|
73
|
-
# text a similarity score is computed from.
|
|
74
|
-
MAX_DOCUMENTS = 200
|
|
75
|
-
MAX_DOCUMENT_CHARS = 20_000
|
|
76
|
-
|
|
77
|
-
# Below this, a "document" is a title and a fragment -- a web_search
|
|
78
|
-
# snippet for a result nobody fetched. Kept, because a 300-char snippet is
|
|
79
|
-
# still the only text some cited URLs ever have, and scoring a claim
|
|
80
|
-
# against it is better than reporting no evidence at all. The constant
|
|
81
|
-
# exists to reject the empty and near-empty case, not the short one.
|
|
82
|
-
MIN_DOCUMENT_CHARS = 16
|
|
83
|
-
|
|
84
|
-
_ARXIV_HOSTS = {"arxiv.org", "www.arxiv.org", "export.arxiv.org"}
|
|
85
|
-
# `2005.14165`, `2005.14165v3`, and the pre-2007 `math.GT/0309136` form --
|
|
86
|
-
# whose archive prefix contains a SLASH, which is why the id group cannot
|
|
87
|
-
# simply exclude one. A path with any further segment is not an arXiv id.
|
|
88
|
-
_ARXIV_PATH_RE = re.compile(
|
|
89
|
-
r"^/(?:abs|pdf)/"
|
|
90
|
-
r"(?P<id>(?:[a-z-]+(?:\.[A-Za-z]{2})?/)?[^/?#]+?)"
|
|
91
|
-
r"(?:v\d+)?(?:\.pdf)?$",
|
|
92
|
-
re.IGNORECASE,
|
|
93
|
-
)
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
@dataclass
|
|
97
|
-
class SourceDocument:
|
|
98
|
-
"""One URL's best available text, and where it came from.
|
|
99
|
-
|
|
100
|
-
`tool` and `retrieved_at` are provenance rather than decoration: a
|
|
101
|
-
similarity score computed from a 300-char search snippet and one
|
|
102
|
-
computed from a fetched page are not the same measurement, and a
|
|
103
|
-
reader of the artifact has to be able to tell which they are looking
|
|
104
|
-
at.
|
|
105
|
-
"""
|
|
106
|
-
|
|
107
|
-
url: str
|
|
108
|
-
text: str
|
|
109
|
-
title: str = ""
|
|
110
|
-
tool: str = ""
|
|
111
|
-
retrieved_at: str = ""
|
|
112
|
-
|
|
113
|
-
#: ROADMAP_v3 §51 (PG7). How far into the SOURCE document the held text
|
|
114
|
-
#: reaches, in the source's own character positions -- which is what
|
|
115
|
-
#: `fetch_url` pages by, and not the same as `len(text)`. 0 for a
|
|
116
|
-
#: document no paging tool produced.
|
|
117
|
-
#:
|
|
118
|
-
#: §52 (RB1) removed ONE of the two reasons those numbers differ.
|
|
119
|
-
#: Redaction used to happen here, on each page, so a page's held length
|
|
120
|
-
#: could be shorter than the span it covered; `fetch_url` now redacts
|
|
121
|
-
#: before it cuts, so its pages arrive clean and that gap is gone. What
|
|
122
|
-
#: still separates the two is `.strip()` and `MAX_DOCUMENT_CHARS`, both
|
|
123
|
-
#: of which change how much is HELD and neither of which moves where a
|
|
124
|
-
#: page sat in the document it came from.
|
|
125
|
-
#:
|
|
126
|
-
#: It is what stops a page being absorbed twice. Without it, a model
|
|
127
|
-
#: that re-reads page 2 -- which it will, because re-reading is how a
|
|
128
|
-
#: model checks itself -- would append it again and the corpus would
|
|
129
|
-
#: hold the same paragraph twice, which a similarity score then counts
|
|
130
|
-
#: twice.
|
|
131
|
-
next_offset: int = 0
|
|
132
|
-
|
|
133
|
-
@property
|
|
134
|
-
def sha256(self) -> str:
|
|
135
|
-
return hashlib.sha256(self.text.encode("utf-8")).hexdigest()
|
|
136
|
-
|
|
137
|
-
def as_artifact(self) -> dict:
|
|
138
|
-
return {
|
|
139
|
-
"url": self.url,
|
|
140
|
-
"title": self.title,
|
|
141
|
-
"tool": self.tool,
|
|
142
|
-
"retrieved_at": self.retrieved_at,
|
|
143
|
-
"sha256": self.sha256,
|
|
144
|
-
"chars": len(self.text),
|
|
145
|
-
"text": self.text,
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
def _now() -> str:
|
|
150
|
-
return datetime.now(UTC).isoformat(timespec="seconds")
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
def arxiv_id_from_url(url: str) -> str | None:
|
|
154
|
-
"""The bare arXiv id in `url`, version stripped, or None.
|
|
155
|
-
|
|
156
|
-
Version stripped because a version is not a different paper for any
|
|
157
|
-
purpose this pipeline has: the corpus keys on it, and OpenAlex's DOI
|
|
158
|
-
for `2005.14165v3` is `10.48550/arXiv.2005.14165`.
|
|
159
|
-
"""
|
|
160
|
-
try:
|
|
161
|
-
parts = urlsplit(url)
|
|
162
|
-
except ValueError:
|
|
163
|
-
return None
|
|
164
|
-
if parts.hostname is None or parts.hostname.lower() not in _ARXIV_HOSTS:
|
|
165
|
-
return None
|
|
166
|
-
match = _ARXIV_PATH_RE.match(parts.path)
|
|
167
|
-
return match.group("id") if match else None
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
def normalize_url(url: str) -> str:
|
|
171
|
-
"""The key two spellings of one source have to share.
|
|
172
|
-
|
|
173
|
-
Case-folds the scheme and host, drops the fragment, drops a default
|
|
174
|
-
port, drops a bare trailing slash, and collapses every arXiv `abs`/`pdf`
|
|
175
|
-
spelling onto one canonical `abs` URL. The QUERY IS KEPT: `?id=7` is a
|
|
176
|
-
different page on most sites, and dropping it would merge sources that
|
|
177
|
-
say different things.
|
|
178
|
-
|
|
179
|
-
An unparseable URL is returned stripped rather than raising. This runs
|
|
180
|
-
on model-supplied strings; a malformed one should fail to match a
|
|
181
|
-
document, not fail the run.
|
|
182
|
-
"""
|
|
183
|
-
raw = (url or "").strip()
|
|
184
|
-
if not raw:
|
|
185
|
-
return ""
|
|
186
|
-
|
|
187
|
-
arxiv_id = arxiv_id_from_url(raw)
|
|
188
|
-
if arxiv_id:
|
|
189
|
-
return f"https://arxiv.org/abs/{arxiv_id}"
|
|
190
|
-
|
|
191
|
-
try:
|
|
192
|
-
parts = urlsplit(raw)
|
|
193
|
-
except ValueError:
|
|
194
|
-
return raw
|
|
195
|
-
|
|
196
|
-
host = (parts.hostname or "").lower()
|
|
197
|
-
if not host:
|
|
198
|
-
return raw
|
|
199
|
-
if parts.port and not (
|
|
200
|
-
(parts.scheme == "http" and parts.port == 80)
|
|
201
|
-
or (parts.scheme == "https" and parts.port == 443)
|
|
202
|
-
):
|
|
203
|
-
host = f"{host}:{parts.port}"
|
|
204
|
-
|
|
205
|
-
path = parts.path
|
|
206
|
-
if path == "/":
|
|
207
|
-
path = ""
|
|
208
|
-
|
|
209
|
-
return urlunsplit((parts.scheme.lower(), host, path, parts.query, ""))
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
@dataclass
|
|
213
|
-
class SourceCorpus:
|
|
214
|
-
"""Every source text one run retrieved, keyed by normalized URL.
|
|
215
|
-
|
|
216
|
-
Run-scoped and passed by reference, the way `run.granted_calls` is
|
|
217
|
-
(§25): one object means the corpus is already complete at the moment
|
|
218
|
-
the scoring stage reads it, rather than being assembled at the end by a
|
|
219
|
-
run that reached the end.
|
|
220
|
-
|
|
221
|
-
THE LONGEST TEXT WINS. One URL can arrive three times -- as a 300-char
|
|
222
|
-
web_search snippet, as a 600-char arXiv abstract, and as 5000 chars
|
|
223
|
-
from fetch_url -- and the question every consumer asks is "what is the
|
|
224
|
-
most this run knows about this page". Last-write-wins would make the
|
|
225
|
-
answer depend on the order the model happened to call its tools in.
|
|
226
|
-
"""
|
|
227
|
-
|
|
228
|
-
documents: dict = field(default_factory=dict)
|
|
229
|
-
dropped: int = 0
|
|
230
|
-
|
|
231
|
-
# -- ingest ----------------------------------------------------------
|
|
232
|
-
|
|
233
|
-
def add(self, tool_name: str, result) -> int:
|
|
234
|
-
"""Absorb one tool result. Returns how many documents it added or
|
|
235
|
-
improved.
|
|
236
|
-
|
|
237
|
-
Anything unrecognised is ignored silently: this is attached to
|
|
238
|
-
EVERY tool result a research pass produces, and a `todo_write`
|
|
239
|
-
return value is not a defect worth a log line. A recognised tool
|
|
240
|
-
whose result is the error shape is also ignored -- `arxiv.py` and
|
|
241
|
-
`web_search.py` both return errors rather than raising, so an
|
|
242
|
-
error dict is an ordinary occurrence here.
|
|
243
|
-
"""
|
|
244
|
-
if not isinstance(result, dict) or "error" in result:
|
|
245
|
-
return 0
|
|
246
|
-
if tool_name == "fetch_url":
|
|
247
|
-
return self._add_fetch_url(result)
|
|
248
|
-
if tool_name == "arxiv_search":
|
|
249
|
-
return self._add_arxiv(result)
|
|
250
|
-
if tool_name == "web_search":
|
|
251
|
-
return self._add_web_search(result)
|
|
252
|
-
return 0
|
|
253
|
-
|
|
254
|
-
def _add_fetch_url(self, result: dict) -> int:
|
|
255
|
-
# `url` is the URL that ANSWERED, after redirects -- which is what
|
|
256
|
-
# fetch_url.py:89-95 says the grounding pass attributes the text
|
|
257
|
-
# to, and therefore what the model will cite.
|
|
258
|
-
#
|
|
259
|
-
# `offset` is ROADMAP_v3 §51 (PG7). A result without one is a
|
|
260
|
-
# pre-§51 shape or a hand-built fixture, and 0 is the right reading
|
|
261
|
-
# of both: a whole document starting at the beginning.
|
|
262
|
-
offset = result.get("offset")
|
|
263
|
-
if not isinstance(offset, int) or offset < 0:
|
|
264
|
-
offset = 0
|
|
265
|
-
return self._store(
|
|
266
|
-
url=result.get("url"),
|
|
267
|
-
text=result.get("content"),
|
|
268
|
-
title="",
|
|
269
|
-
tool="fetch_url",
|
|
270
|
-
offset=offset,
|
|
271
|
-
)
|
|
272
|
-
|
|
273
|
-
def _add_arxiv(self, result: dict) -> int:
|
|
274
|
-
added = 0
|
|
275
|
-
for entry in result.get("results") or ():
|
|
276
|
-
if not isinstance(entry, dict):
|
|
277
|
-
continue
|
|
278
|
-
arxiv_id = (entry.get("arxiv_id") or "").strip()
|
|
279
|
-
if not arxiv_id:
|
|
280
|
-
continue
|
|
281
|
-
title = (entry.get("title") or "").strip()
|
|
282
|
-
summary = (entry.get("summary") or "").strip()
|
|
283
|
-
# Title AND abstract, joined. The title carries the paper's
|
|
284
|
-
# subject in the compressed form a claim is usually phrased in,
|
|
285
|
-
# and a 600-char abstract that opens on background prose can
|
|
286
|
-
# otherwise match a claim about the paper's own topic poorly.
|
|
287
|
-
text = "\n\n".join(p for p in (title, summary) if p)
|
|
288
|
-
added += self._store(
|
|
289
|
-
url=f"https://arxiv.org/abs/{arxiv_id}",
|
|
290
|
-
text=text,
|
|
291
|
-
title=title,
|
|
292
|
-
tool="arxiv_search",
|
|
293
|
-
)
|
|
294
|
-
return added
|
|
295
|
-
|
|
296
|
-
def _add_web_search(self, result: dict) -> int:
|
|
297
|
-
added = 0
|
|
298
|
-
for entry in result.get("results") or ():
|
|
299
|
-
if not isinstance(entry, dict):
|
|
300
|
-
continue
|
|
301
|
-
title = (entry.get("title") or "").strip()
|
|
302
|
-
snippet = (entry.get("snippet") or "").strip()
|
|
303
|
-
text = "\n\n".join(p for p in (title, snippet) if p)
|
|
304
|
-
added += self._store(
|
|
305
|
-
url=entry.get("url"),
|
|
306
|
-
text=text,
|
|
307
|
-
title=title,
|
|
308
|
-
tool="web_search",
|
|
309
|
-
)
|
|
310
|
-
return added
|
|
311
|
-
|
|
312
|
-
def _store(self, url, text, title: str, tool: str, offset=None) -> int:
|
|
313
|
-
"""Absorb one document, or one PAGE of one (ROADMAP_v3 §51, PG7).
|
|
314
|
-
|
|
315
|
-
`offset` is the source position this text starts at, or None from a
|
|
316
|
-
tool that does not page. None and 0 are NOT the same thing here:
|
|
317
|
-
None means "this is the whole of what that tool returns", 0 means
|
|
318
|
-
"this is the first page of something that may have more".
|
|
319
|
-
|
|
320
|
-
WHY THE PAGED CASE CANNOT USE LONGEST-WINS. Measured, batch 105,
|
|
321
|
-
before any of this was written: a corpus given page 1 and then page
|
|
322
|
-
2 of one URL returned 1 then **0** -- `len(existing.text) >=
|
|
323
|
-
len(cleaned)` is 5000 >= 5000, so page 2 was DROPPED and the held
|
|
324
|
-
text stayed page 1. Paging would then have made the model better
|
|
325
|
-
informed and the grounding pass no better, and the divergence is
|
|
326
|
-
invisible from both sides: the model reads four pages and the score
|
|
327
|
-
is computed against the first. The near miss is worse than the
|
|
328
|
-
miss -- a page 2 one character longer REPLACED page 1 outright,
|
|
329
|
-
leaving the corpus holding the middle of a document and attributing
|
|
330
|
-
it to the whole URL.
|
|
331
|
-
"""
|
|
332
|
-
if not isinstance(url, str) or not isinstance(text, str):
|
|
333
|
-
return 0
|
|
334
|
-
key = normalize_url(url)
|
|
335
|
-
if not key:
|
|
336
|
-
return 0
|
|
337
|
-
|
|
338
|
-
existing = self.documents.get(key)
|
|
339
|
-
|
|
340
|
-
# A continuation extends what is held; anything else competes on
|
|
341
|
-
# length exactly as it did before §51.
|
|
342
|
-
continuing = (
|
|
343
|
-
offset is not None
|
|
344
|
-
and offset > 0
|
|
345
|
-
and existing is not None
|
|
346
|
-
and existing.tool == tool
|
|
347
|
-
)
|
|
348
|
-
|
|
349
|
-
if continuing:
|
|
350
|
-
# How much of this page is already held. Covers all three
|
|
351
|
-
# shapes with one subtraction: an exact continuation overlaps
|
|
352
|
-
# by nothing, a re-read of the same page overlaps entirely and
|
|
353
|
-
# contributes the empty string, and a page that starts inside
|
|
354
|
-
# what is held contributes only its tail. A page starting
|
|
355
|
-
# BEYOND the held text leaves a gap -- appended anyway, because
|
|
356
|
-
# the text is real and the alternative is losing it, and an
|
|
357
|
-
# incomplete document is what this corpus already holds for
|
|
358
|
-
# every source the byte cap cut.
|
|
359
|
-
#
|
|
360
|
-
# There is deliberately NO `if overlap >= len(text): return 0`
|
|
361
|
-
# above this. One was written, and a mutation pass showed it
|
|
362
|
-
# could be deleted with no test failing anywhere -- because the
|
|
363
|
-
# slice already yields "" in exactly the cases it guarded, and
|
|
364
|
-
# the empty-addition check below catches them. Batch 103's
|
|
365
|
-
# lesson applies to a guard that cannot fire as much as to one
|
|
366
|
-
# that fires in the wrong place: it reads as the thing
|
|
367
|
-
# protecting you.
|
|
368
|
-
overlap = existing.next_offset - offset
|
|
369
|
-
addition = redact_output_text(text[max(0, overlap):])
|
|
370
|
-
if not addition.strip():
|
|
371
|
-
return 0
|
|
372
|
-
merged = (existing.text + addition)[:MAX_DOCUMENT_CHARS]
|
|
373
|
-
if merged == existing.text:
|
|
374
|
-
return 0
|
|
375
|
-
self.documents[key] = SourceDocument(
|
|
376
|
-
url=key, text=merged, title=existing.title,
|
|
377
|
-
tool=tool, retrieved_at=_now(),
|
|
378
|
-
next_offset=max(existing.next_offset, offset + len(text)),
|
|
379
|
-
)
|
|
380
|
-
return 1
|
|
381
|
-
|
|
382
|
-
cleaned = redact_output_text(text).strip()
|
|
383
|
-
if len(cleaned) < MIN_DOCUMENT_CHARS:
|
|
384
|
-
return 0
|
|
385
|
-
cleaned = cleaned[:MAX_DOCUMENT_CHARS]
|
|
386
|
-
|
|
387
|
-
if existing is not None and len(existing.text) >= len(cleaned):
|
|
388
|
-
return 0
|
|
389
|
-
if existing is None and len(self.documents) >= MAX_DOCUMENTS:
|
|
390
|
-
# Counted rather than logged per drop: a run that hits this is
|
|
391
|
-
# hitting it repeatedly, and the count is what the trace line
|
|
392
|
-
# at the end of the stage reports.
|
|
393
|
-
self.dropped += 1
|
|
394
|
-
return 0
|
|
395
|
-
|
|
396
|
-
self.documents[key] = SourceDocument(
|
|
397
|
-
url=key, text=cleaned, title=title or (existing.title if existing else ""),
|
|
398
|
-
tool=tool, retrieved_at=_now(),
|
|
399
|
-
# In the SOURCE's positions, so the next page's offset can be
|
|
400
|
-
# compared against it. `len(text)` and not `len(cleaned)`:
|
|
401
|
-
# redaction and stripping change how much is HELD, never where
|
|
402
|
-
# the page sat in the document it came from.
|
|
403
|
-
next_offset=(offset + len(text)) if offset is not None else 0,
|
|
404
|
-
)
|
|
405
|
-
return 1
|
|
406
|
-
|
|
407
|
-
# -- read ------------------------------------------------------------
|
|
408
|
-
|
|
409
|
-
def get(self, url: str):
|
|
410
|
-
"""The document held for `url`, or None. Normalizes first, so a
|
|
411
|
-
caller passing the model's spelling gets the fetched page."""
|
|
412
|
-
return self.documents.get(normalize_url(url))
|
|
413
|
-
|
|
414
|
-
def text_for(self, url: str) -> str:
|
|
415
|
-
document = self.get(url)
|
|
416
|
-
return document.text if document is not None else ""
|
|
417
|
-
|
|
418
|
-
def __len__(self) -> int:
|
|
419
|
-
return len(self.documents)
|
|
420
|
-
|
|
421
|
-
def __bool__(self) -> bool:
|
|
422
|
-
# Explicit, because __len__ alone would make an empty corpus
|
|
423
|
-
# falsy and every `if corpus:` guard in the orchestrator would
|
|
424
|
-
# then mean "if anything was fetched" rather than "if a corpus is
|
|
425
|
-
# being collected at all". Those are different questions and the
|
|
426
|
-
# first one is never the one being asked.
|
|
427
|
-
return True
|
|
428
|
-
|
|
429
|
-
def artifact_entries(self) -> list[dict]:
|
|
430
|
-
"""Every document, newest spelling of each URL, sorted for a
|
|
431
|
-
stable diff between runs."""
|
|
432
|
-
return [self.documents[key].as_artifact()
|
|
433
|
-
for key in sorted(self.documents)]
|
|
1
|
+
"""
|
|
2
|
+
core/reasoning/source_corpus.py
|
|
3
|
+
|
|
4
|
+
What a claim was actually grounded IN, kept for the length of one run.
|
|
5
|
+
|
|
6
|
+
ROADMAP_v2 §45 (SQ2). Pass 3a's `sources` list has always been URLs and two
|
|
7
|
+
model-invented floats. The text those URLs served reached the model, was
|
|
8
|
+
stringified into the pass thread, and was then unreachable: nothing in
|
|
9
|
+
`03_grounding.json` could say what a page had said, so `similarity_score`
|
|
10
|
+
could not be checked against anything and a quote could not be checked at
|
|
11
|
+
all. `fetch_url.py`'s own comment already describes the artifact this
|
|
12
|
+
builds ("output_writer builds each run's sources/ directory from it") --
|
|
13
|
+
that directory has existed and been empty since §12.
|
|
14
|
+
|
|
15
|
+
WHY A SECOND SINK RATHER THAN A CHANGE TO _translate. The orchestrator
|
|
16
|
+
already sees every tool result and deliberately forwards only `ok` plus an
|
|
17
|
+
error string to the UI, because a successful result body is not
|
|
18
|
+
observability. That boundary is unchanged. This is a different consumer
|
|
19
|
+
with a different need, attached beside it -- the corpus is read by the
|
|
20
|
+
scoring stage and written to the run directory, and neither is a
|
|
21
|
+
transcript.
|
|
22
|
+
|
|
23
|
+
REDACTED ON ENTRY, THOUGH IT ARRIVES REDACTED. `registry.dispatch` runs
|
|
24
|
+
every result through `check_output_policy` before the loop emits it, so
|
|
25
|
+
what lands here has already been through `redact_output_text`. It is run
|
|
26
|
+
again anyway, and for `_translate`'s stated reason: one rule owned at one
|
|
27
|
+
boundary -- nothing is stored by this module unredacted -- beats depending
|
|
28
|
+
on a guarantee made two layers away that a later refactor could quietly
|
|
29
|
+
drop. The second pass is a no-op on already-clean text.
|
|
30
|
+
|
|
31
|
+
AND IT IS NOT WHERE §52's SPLIT-CREDENTIAL FIX WENT, deliberately (RB4).
|
|
32
|
+
§51 made this module the one place where two pages of a document become
|
|
33
|
+
adjacent, so a credential straddling a page boundary was rejoined HERE --
|
|
34
|
+
reproduced end to end in batch 106, into `artifact_entries()` and therefore
|
|
35
|
+
into `sources/<sha256>.txt` and the window sent to the embedding provider.
|
|
36
|
+
The register's proposed fix was to re-redact the whole merged text on every
|
|
37
|
+
append. It was declined: `fetch_url` redacts before it cuts now, so the
|
|
38
|
+
halves never exist, and re-scanning up to 20,000 characters per page would
|
|
39
|
+
buy nothing this module can see -- while leaving unfixed the model's own two
|
|
40
|
+
tool results, which the producer-side fix does close.
|
|
41
|
+
|
|
42
|
+
THIS MODULE OWNS URL IDENTITY for the pipeline. Two spellings of one arXiv
|
|
43
|
+
paper are one source, and a model that cites `/abs/2005.14165v3` after
|
|
44
|
+
fetching `/abs/2005.14165` must not be told its own citation has no text.
|
|
45
|
+
`scholar.py` resolves the same URLs and imports `arxiv_id_from_url` from
|
|
46
|
+
here rather than re-deriving it -- a second copy of "what counts as the
|
|
47
|
+
same paper" is how the corpus and the scholarly lookup come to disagree
|
|
48
|
+
about one source in one artifact.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
from __future__ import annotations
|
|
52
|
+
|
|
53
|
+
import hashlib
|
|
54
|
+
import logging
|
|
55
|
+
import re
|
|
56
|
+
from dataclasses import dataclass, field
|
|
57
|
+
from datetime import UTC, datetime
|
|
58
|
+
from urllib.parse import urlsplit, urlunsplit
|
|
59
|
+
|
|
60
|
+
from safety.policy_enforcement import redact_output_text
|
|
61
|
+
|
|
62
|
+
logger = logging.getLogger(__name__)
|
|
63
|
+
|
|
64
|
+
# Bounds, not tuning. A research pass calls fetch_url a bounded number of
|
|
65
|
+
# times (§31 budgets), so these are a backstop against a pathological run
|
|
66
|
+
# rather than a knob anyone is expected to turn: the corpus is held in
|
|
67
|
+
# memory for the length of a run and then written to disk.
|
|
68
|
+
#
|
|
69
|
+
# MAX_DOCUMENT_CHARS is deliberately well ABOVE fetch_url's own
|
|
70
|
+
# MAX_CONTENT_CHARS (5000). It bounds this module against a future tool
|
|
71
|
+
# that returns more; it is not a second truncation of the tools that exist,
|
|
72
|
+
# because a cap that bites in normal operation would silently shorten the
|
|
73
|
+
# text a similarity score is computed from.
|
|
74
|
+
MAX_DOCUMENTS = 200
|
|
75
|
+
MAX_DOCUMENT_CHARS = 20_000
|
|
76
|
+
|
|
77
|
+
# Below this, a "document" is a title and a fragment -- a web_search
|
|
78
|
+
# snippet for a result nobody fetched. Kept, because a 300-char snippet is
|
|
79
|
+
# still the only text some cited URLs ever have, and scoring a claim
|
|
80
|
+
# against it is better than reporting no evidence at all. The constant
|
|
81
|
+
# exists to reject the empty and near-empty case, not the short one.
|
|
82
|
+
MIN_DOCUMENT_CHARS = 16
|
|
83
|
+
|
|
84
|
+
_ARXIV_HOSTS = {"arxiv.org", "www.arxiv.org", "export.arxiv.org"}
|
|
85
|
+
# `2005.14165`, `2005.14165v3`, and the pre-2007 `math.GT/0309136` form --
|
|
86
|
+
# whose archive prefix contains a SLASH, which is why the id group cannot
|
|
87
|
+
# simply exclude one. A path with any further segment is not an arXiv id.
|
|
88
|
+
_ARXIV_PATH_RE = re.compile(
|
|
89
|
+
r"^/(?:abs|pdf)/"
|
|
90
|
+
r"(?P<id>(?:[a-z-]+(?:\.[A-Za-z]{2})?/)?[^/?#]+?)"
|
|
91
|
+
r"(?:v\d+)?(?:\.pdf)?$",
|
|
92
|
+
re.IGNORECASE,
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass
|
|
97
|
+
class SourceDocument:
|
|
98
|
+
"""One URL's best available text, and where it came from.
|
|
99
|
+
|
|
100
|
+
`tool` and `retrieved_at` are provenance rather than decoration: a
|
|
101
|
+
similarity score computed from a 300-char search snippet and one
|
|
102
|
+
computed from a fetched page are not the same measurement, and a
|
|
103
|
+
reader of the artifact has to be able to tell which they are looking
|
|
104
|
+
at.
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
url: str
|
|
108
|
+
text: str
|
|
109
|
+
title: str = ""
|
|
110
|
+
tool: str = ""
|
|
111
|
+
retrieved_at: str = ""
|
|
112
|
+
|
|
113
|
+
#: ROADMAP_v3 §51 (PG7). How far into the SOURCE document the held text
|
|
114
|
+
#: reaches, in the source's own character positions -- which is what
|
|
115
|
+
#: `fetch_url` pages by, and not the same as `len(text)`. 0 for a
|
|
116
|
+
#: document no paging tool produced.
|
|
117
|
+
#:
|
|
118
|
+
#: §52 (RB1) removed ONE of the two reasons those numbers differ.
|
|
119
|
+
#: Redaction used to happen here, on each page, so a page's held length
|
|
120
|
+
#: could be shorter than the span it covered; `fetch_url` now redacts
|
|
121
|
+
#: before it cuts, so its pages arrive clean and that gap is gone. What
|
|
122
|
+
#: still separates the two is `.strip()` and `MAX_DOCUMENT_CHARS`, both
|
|
123
|
+
#: of which change how much is HELD and neither of which moves where a
|
|
124
|
+
#: page sat in the document it came from.
|
|
125
|
+
#:
|
|
126
|
+
#: It is what stops a page being absorbed twice. Without it, a model
|
|
127
|
+
#: that re-reads page 2 -- which it will, because re-reading is how a
|
|
128
|
+
#: model checks itself -- would append it again and the corpus would
|
|
129
|
+
#: hold the same paragraph twice, which a similarity score then counts
|
|
130
|
+
#: twice.
|
|
131
|
+
next_offset: int = 0
|
|
132
|
+
|
|
133
|
+
@property
|
|
134
|
+
def sha256(self) -> str:
|
|
135
|
+
return hashlib.sha256(self.text.encode("utf-8")).hexdigest()
|
|
136
|
+
|
|
137
|
+
def as_artifact(self) -> dict:
|
|
138
|
+
return {
|
|
139
|
+
"url": self.url,
|
|
140
|
+
"title": self.title,
|
|
141
|
+
"tool": self.tool,
|
|
142
|
+
"retrieved_at": self.retrieved_at,
|
|
143
|
+
"sha256": self.sha256,
|
|
144
|
+
"chars": len(self.text),
|
|
145
|
+
"text": self.text,
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _now() -> str:
|
|
150
|
+
return datetime.now(UTC).isoformat(timespec="seconds")
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def arxiv_id_from_url(url: str) -> str | None:
|
|
154
|
+
"""The bare arXiv id in `url`, version stripped, or None.
|
|
155
|
+
|
|
156
|
+
Version stripped because a version is not a different paper for any
|
|
157
|
+
purpose this pipeline has: the corpus keys on it, and OpenAlex's DOI
|
|
158
|
+
for `2005.14165v3` is `10.48550/arXiv.2005.14165`.
|
|
159
|
+
"""
|
|
160
|
+
try:
|
|
161
|
+
parts = urlsplit(url)
|
|
162
|
+
except ValueError:
|
|
163
|
+
return None
|
|
164
|
+
if parts.hostname is None or parts.hostname.lower() not in _ARXIV_HOSTS:
|
|
165
|
+
return None
|
|
166
|
+
match = _ARXIV_PATH_RE.match(parts.path)
|
|
167
|
+
return match.group("id") if match else None
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def normalize_url(url: str) -> str:
|
|
171
|
+
"""The key two spellings of one source have to share.
|
|
172
|
+
|
|
173
|
+
Case-folds the scheme and host, drops the fragment, drops a default
|
|
174
|
+
port, drops a bare trailing slash, and collapses every arXiv `abs`/`pdf`
|
|
175
|
+
spelling onto one canonical `abs` URL. The QUERY IS KEPT: `?id=7` is a
|
|
176
|
+
different page on most sites, and dropping it would merge sources that
|
|
177
|
+
say different things.
|
|
178
|
+
|
|
179
|
+
An unparseable URL is returned stripped rather than raising. This runs
|
|
180
|
+
on model-supplied strings; a malformed one should fail to match a
|
|
181
|
+
document, not fail the run.
|
|
182
|
+
"""
|
|
183
|
+
raw = (url or "").strip()
|
|
184
|
+
if not raw:
|
|
185
|
+
return ""
|
|
186
|
+
|
|
187
|
+
arxiv_id = arxiv_id_from_url(raw)
|
|
188
|
+
if arxiv_id:
|
|
189
|
+
return f"https://arxiv.org/abs/{arxiv_id}"
|
|
190
|
+
|
|
191
|
+
try:
|
|
192
|
+
parts = urlsplit(raw)
|
|
193
|
+
except ValueError:
|
|
194
|
+
return raw
|
|
195
|
+
|
|
196
|
+
host = (parts.hostname or "").lower()
|
|
197
|
+
if not host:
|
|
198
|
+
return raw
|
|
199
|
+
if parts.port and not (
|
|
200
|
+
(parts.scheme == "http" and parts.port == 80)
|
|
201
|
+
or (parts.scheme == "https" and parts.port == 443)
|
|
202
|
+
):
|
|
203
|
+
host = f"{host}:{parts.port}"
|
|
204
|
+
|
|
205
|
+
path = parts.path
|
|
206
|
+
if path == "/":
|
|
207
|
+
path = ""
|
|
208
|
+
|
|
209
|
+
return urlunsplit((parts.scheme.lower(), host, path, parts.query, ""))
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
@dataclass
|
|
213
|
+
class SourceCorpus:
|
|
214
|
+
"""Every source text one run retrieved, keyed by normalized URL.
|
|
215
|
+
|
|
216
|
+
Run-scoped and passed by reference, the way `run.granted_calls` is
|
|
217
|
+
(§25): one object means the corpus is already complete at the moment
|
|
218
|
+
the scoring stage reads it, rather than being assembled at the end by a
|
|
219
|
+
run that reached the end.
|
|
220
|
+
|
|
221
|
+
THE LONGEST TEXT WINS. One URL can arrive three times -- as a 300-char
|
|
222
|
+
web_search snippet, as a 600-char arXiv abstract, and as 5000 chars
|
|
223
|
+
from fetch_url -- and the question every consumer asks is "what is the
|
|
224
|
+
most this run knows about this page". Last-write-wins would make the
|
|
225
|
+
answer depend on the order the model happened to call its tools in.
|
|
226
|
+
"""
|
|
227
|
+
|
|
228
|
+
documents: dict = field(default_factory=dict)
|
|
229
|
+
dropped: int = 0
|
|
230
|
+
|
|
231
|
+
# -- ingest ----------------------------------------------------------
|
|
232
|
+
|
|
233
|
+
def add(self, tool_name: str, result) -> int:
|
|
234
|
+
"""Absorb one tool result. Returns how many documents it added or
|
|
235
|
+
improved.
|
|
236
|
+
|
|
237
|
+
Anything unrecognised is ignored silently: this is attached to
|
|
238
|
+
EVERY tool result a research pass produces, and a `todo_write`
|
|
239
|
+
return value is not a defect worth a log line. A recognised tool
|
|
240
|
+
whose result is the error shape is also ignored -- `arxiv.py` and
|
|
241
|
+
`web_search.py` both return errors rather than raising, so an
|
|
242
|
+
error dict is an ordinary occurrence here.
|
|
243
|
+
"""
|
|
244
|
+
if not isinstance(result, dict) or "error" in result:
|
|
245
|
+
return 0
|
|
246
|
+
if tool_name == "fetch_url":
|
|
247
|
+
return self._add_fetch_url(result)
|
|
248
|
+
if tool_name == "arxiv_search":
|
|
249
|
+
return self._add_arxiv(result)
|
|
250
|
+
if tool_name == "web_search":
|
|
251
|
+
return self._add_web_search(result)
|
|
252
|
+
return 0
|
|
253
|
+
|
|
254
|
+
def _add_fetch_url(self, result: dict) -> int:
|
|
255
|
+
# `url` is the URL that ANSWERED, after redirects -- which is what
|
|
256
|
+
# fetch_url.py:89-95 says the grounding pass attributes the text
|
|
257
|
+
# to, and therefore what the model will cite.
|
|
258
|
+
#
|
|
259
|
+
# `offset` is ROADMAP_v3 §51 (PG7). A result without one is a
|
|
260
|
+
# pre-§51 shape or a hand-built fixture, and 0 is the right reading
|
|
261
|
+
# of both: a whole document starting at the beginning.
|
|
262
|
+
offset = result.get("offset")
|
|
263
|
+
if not isinstance(offset, int) or offset < 0:
|
|
264
|
+
offset = 0
|
|
265
|
+
return self._store(
|
|
266
|
+
url=result.get("url"),
|
|
267
|
+
text=result.get("content"),
|
|
268
|
+
title="",
|
|
269
|
+
tool="fetch_url",
|
|
270
|
+
offset=offset,
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
def _add_arxiv(self, result: dict) -> int:
|
|
274
|
+
added = 0
|
|
275
|
+
for entry in result.get("results") or ():
|
|
276
|
+
if not isinstance(entry, dict):
|
|
277
|
+
continue
|
|
278
|
+
arxiv_id = (entry.get("arxiv_id") or "").strip()
|
|
279
|
+
if not arxiv_id:
|
|
280
|
+
continue
|
|
281
|
+
title = (entry.get("title") or "").strip()
|
|
282
|
+
summary = (entry.get("summary") or "").strip()
|
|
283
|
+
# Title AND abstract, joined. The title carries the paper's
|
|
284
|
+
# subject in the compressed form a claim is usually phrased in,
|
|
285
|
+
# and a 600-char abstract that opens on background prose can
|
|
286
|
+
# otherwise match a claim about the paper's own topic poorly.
|
|
287
|
+
text = "\n\n".join(p for p in (title, summary) if p)
|
|
288
|
+
added += self._store(
|
|
289
|
+
url=f"https://arxiv.org/abs/{arxiv_id}",
|
|
290
|
+
text=text,
|
|
291
|
+
title=title,
|
|
292
|
+
tool="arxiv_search",
|
|
293
|
+
)
|
|
294
|
+
return added
|
|
295
|
+
|
|
296
|
+
def _add_web_search(self, result: dict) -> int:
|
|
297
|
+
added = 0
|
|
298
|
+
for entry in result.get("results") or ():
|
|
299
|
+
if not isinstance(entry, dict):
|
|
300
|
+
continue
|
|
301
|
+
title = (entry.get("title") or "").strip()
|
|
302
|
+
snippet = (entry.get("snippet") or "").strip()
|
|
303
|
+
text = "\n\n".join(p for p in (title, snippet) if p)
|
|
304
|
+
added += self._store(
|
|
305
|
+
url=entry.get("url"),
|
|
306
|
+
text=text,
|
|
307
|
+
title=title,
|
|
308
|
+
tool="web_search",
|
|
309
|
+
)
|
|
310
|
+
return added
|
|
311
|
+
|
|
312
|
+
def _store(self, url, text, title: str, tool: str, offset=None) -> int:
|
|
313
|
+
"""Absorb one document, or one PAGE of one (ROADMAP_v3 §51, PG7).
|
|
314
|
+
|
|
315
|
+
`offset` is the source position this text starts at, or None from a
|
|
316
|
+
tool that does not page. None and 0 are NOT the same thing here:
|
|
317
|
+
None means "this is the whole of what that tool returns", 0 means
|
|
318
|
+
"this is the first page of something that may have more".
|
|
319
|
+
|
|
320
|
+
WHY THE PAGED CASE CANNOT USE LONGEST-WINS. Measured, batch 105,
|
|
321
|
+
before any of this was written: a corpus given page 1 and then page
|
|
322
|
+
2 of one URL returned 1 then **0** -- `len(existing.text) >=
|
|
323
|
+
len(cleaned)` is 5000 >= 5000, so page 2 was DROPPED and the held
|
|
324
|
+
text stayed page 1. Paging would then have made the model better
|
|
325
|
+
informed and the grounding pass no better, and the divergence is
|
|
326
|
+
invisible from both sides: the model reads four pages and the score
|
|
327
|
+
is computed against the first. The near miss is worse than the
|
|
328
|
+
miss -- a page 2 one character longer REPLACED page 1 outright,
|
|
329
|
+
leaving the corpus holding the middle of a document and attributing
|
|
330
|
+
it to the whole URL.
|
|
331
|
+
"""
|
|
332
|
+
if not isinstance(url, str) or not isinstance(text, str):
|
|
333
|
+
return 0
|
|
334
|
+
key = normalize_url(url)
|
|
335
|
+
if not key:
|
|
336
|
+
return 0
|
|
337
|
+
|
|
338
|
+
existing = self.documents.get(key)
|
|
339
|
+
|
|
340
|
+
# A continuation extends what is held; anything else competes on
|
|
341
|
+
# length exactly as it did before §51.
|
|
342
|
+
continuing = (
|
|
343
|
+
offset is not None
|
|
344
|
+
and offset > 0
|
|
345
|
+
and existing is not None
|
|
346
|
+
and existing.tool == tool
|
|
347
|
+
)
|
|
348
|
+
|
|
349
|
+
if continuing:
|
|
350
|
+
# How much of this page is already held. Covers all three
|
|
351
|
+
# shapes with one subtraction: an exact continuation overlaps
|
|
352
|
+
# by nothing, a re-read of the same page overlaps entirely and
|
|
353
|
+
# contributes the empty string, and a page that starts inside
|
|
354
|
+
# what is held contributes only its tail. A page starting
|
|
355
|
+
# BEYOND the held text leaves a gap -- appended anyway, because
|
|
356
|
+
# the text is real and the alternative is losing it, and an
|
|
357
|
+
# incomplete document is what this corpus already holds for
|
|
358
|
+
# every source the byte cap cut.
|
|
359
|
+
#
|
|
360
|
+
# There is deliberately NO `if overlap >= len(text): return 0`
|
|
361
|
+
# above this. One was written, and a mutation pass showed it
|
|
362
|
+
# could be deleted with no test failing anywhere -- because the
|
|
363
|
+
# slice already yields "" in exactly the cases it guarded, and
|
|
364
|
+
# the empty-addition check below catches them. Batch 103's
|
|
365
|
+
# lesson applies to a guard that cannot fire as much as to one
|
|
366
|
+
# that fires in the wrong place: it reads as the thing
|
|
367
|
+
# protecting you.
|
|
368
|
+
overlap = existing.next_offset - offset
|
|
369
|
+
addition = redact_output_text(text[max(0, overlap):])
|
|
370
|
+
if not addition.strip():
|
|
371
|
+
return 0
|
|
372
|
+
merged = (existing.text + addition)[:MAX_DOCUMENT_CHARS]
|
|
373
|
+
if merged == existing.text:
|
|
374
|
+
return 0
|
|
375
|
+
self.documents[key] = SourceDocument(
|
|
376
|
+
url=key, text=merged, title=existing.title,
|
|
377
|
+
tool=tool, retrieved_at=_now(),
|
|
378
|
+
next_offset=max(existing.next_offset, offset + len(text)),
|
|
379
|
+
)
|
|
380
|
+
return 1
|
|
381
|
+
|
|
382
|
+
cleaned = redact_output_text(text).strip()
|
|
383
|
+
if len(cleaned) < MIN_DOCUMENT_CHARS:
|
|
384
|
+
return 0
|
|
385
|
+
cleaned = cleaned[:MAX_DOCUMENT_CHARS]
|
|
386
|
+
|
|
387
|
+
if existing is not None and len(existing.text) >= len(cleaned):
|
|
388
|
+
return 0
|
|
389
|
+
if existing is None and len(self.documents) >= MAX_DOCUMENTS:
|
|
390
|
+
# Counted rather than logged per drop: a run that hits this is
|
|
391
|
+
# hitting it repeatedly, and the count is what the trace line
|
|
392
|
+
# at the end of the stage reports.
|
|
393
|
+
self.dropped += 1
|
|
394
|
+
return 0
|
|
395
|
+
|
|
396
|
+
self.documents[key] = SourceDocument(
|
|
397
|
+
url=key, text=cleaned, title=title or (existing.title if existing else ""),
|
|
398
|
+
tool=tool, retrieved_at=_now(),
|
|
399
|
+
# In the SOURCE's positions, so the next page's offset can be
|
|
400
|
+
# compared against it. `len(text)` and not `len(cleaned)`:
|
|
401
|
+
# redaction and stripping change how much is HELD, never where
|
|
402
|
+
# the page sat in the document it came from.
|
|
403
|
+
next_offset=(offset + len(text)) if offset is not None else 0,
|
|
404
|
+
)
|
|
405
|
+
return 1
|
|
406
|
+
|
|
407
|
+
# -- read ------------------------------------------------------------
|
|
408
|
+
|
|
409
|
+
def get(self, url: str):
|
|
410
|
+
"""The document held for `url`, or None. Normalizes first, so a
|
|
411
|
+
caller passing the model's spelling gets the fetched page."""
|
|
412
|
+
return self.documents.get(normalize_url(url))
|
|
413
|
+
|
|
414
|
+
def text_for(self, url: str) -> str:
|
|
415
|
+
document = self.get(url)
|
|
416
|
+
return document.text if document is not None else ""
|
|
417
|
+
|
|
418
|
+
def __len__(self) -> int:
|
|
419
|
+
return len(self.documents)
|
|
420
|
+
|
|
421
|
+
def __bool__(self) -> bool:
|
|
422
|
+
# Explicit, because __len__ alone would make an empty corpus
|
|
423
|
+
# falsy and every `if corpus:` guard in the orchestrator would
|
|
424
|
+
# then mean "if anything was fetched" rather than "if a corpus is
|
|
425
|
+
# being collected at all". Those are different questions and the
|
|
426
|
+
# first one is never the one being asked.
|
|
427
|
+
return True
|
|
428
|
+
|
|
429
|
+
def artifact_entries(self) -> list[dict]:
|
|
430
|
+
"""Every document, newest spelling of each URL, sorted for a
|
|
431
|
+
stable diff between runs."""
|
|
432
|
+
return [self.documents[key].as_artifact()
|
|
433
|
+
for key in sorted(self.documents)]
|