@yahyahammad/venastine 0.2.0 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/.env.example +9 -4
  2. package/CONFIG_ARCHITECTURE.md +2030 -0
  3. package/LICENSE +201 -201
  4. package/NOTICE +1 -1
  5. package/README.md +1034 -895
  6. package/agents/builtin/build.md +73 -0
  7. package/agents/builtin/compactor.md +75 -75
  8. package/agents/builtin/explore.md +77 -77
  9. package/agents/builtin/general.md +80 -0
  10. package/agents/builtin/pipeline-reviewer.md +100 -100
  11. package/agents/builtin/plan.md +106 -88
  12. package/agents/builtin/review.md +74 -74
  13. package/agents/builtin/test.md +65 -0
  14. package/agents/builtin/writer.md +67 -0
  15. package/agents/manager.py +68 -2
  16. package/agents/subagent_tool.py +472 -238
  17. package/agents/tui_commands.py +238 -151
  18. package/bin/venastine.mjs +150 -4
  19. package/bin/venastine.py +71 -0
  20. package/cli_commands.py +838 -0
  21. package/config.py +95 -1198
  22. package/config.yaml +1103 -0
  23. package/config_edit.py +1301 -0
  24. package/config_schema.py +971 -0
  25. package/config_update.py +749 -0
  26. package/core/agent_activity.py +256 -118
  27. package/core/ansi.py +452 -0
  28. package/core/approval.py +137 -115
  29. package/core/call_view.py +364 -0
  30. package/core/cancel.py +121 -0
  31. package/core/chatgpt_auth.py +747 -0
  32. package/core/claims_view.py +107 -0
  33. package/core/client.py +2715 -1559
  34. package/core/commands.py +460 -0
  35. package/core/compaction.py +1735 -971
  36. package/core/config_loader.py +1982 -1563
  37. package/core/connect.py +686 -0
  38. package/core/events.py +123 -61
  39. package/core/file_mentions.py +280 -0
  40. package/core/interaction.py +411 -328
  41. package/core/line_pattern.py +128 -0
  42. package/core/loop.py +2566 -1490
  43. package/core/memory.py +482 -370
  44. package/core/pipeline_models.py +190 -176
  45. package/core/provider_errors.py +234 -0
  46. package/core/reasoning/authorization.py +130 -130
  47. package/core/reasoning/base.py +219 -219
  48. package/core/reasoning/confidence_scoring.py +484 -468
  49. package/core/reasoning/events.py +149 -149
  50. package/core/reasoning/json_retry.py +19 -1
  51. package/core/reasoning/orchestrator.py +1561 -1382
  52. package/core/reasoning/output_writer.py +345 -286
  53. package/core/reasoning/payload_validation.py +524 -398
  54. package/core/reasoning/pipeline_storage.py +88 -22
  55. package/core/reasoning/review.py +659 -615
  56. package/core/reasoning/scholar.py +458 -434
  57. package/core/reasoning/source_corpus.py +433 -327
  58. package/core/reasoning/source_scoring.py +855 -855
  59. package/core/replay.py +315 -183
  60. package/core/session.py +36 -12
  61. package/core/session_view.py +259 -0
  62. package/core/session_wake.py +259 -0
  63. package/core/shell_sessions.py +1553 -0
  64. package/core/workspace_trust.py +1 -1
  65. package/credentials.py +148 -20
  66. package/database.py +333 -252
  67. package/{PRIVACY.md → docs/PRIVACY.md} +78 -66
  68. package/docs/SECURITY.md +108 -0
  69. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  70. package/env_secrets.py +50 -9
  71. package/json_store.py +343 -123
  72. package/logging_setup.py +295 -185
  73. package/main.py +3433 -1961
  74. package/mcp_client/client.py +733 -499
  75. package/mcp_client/config.py +126 -21
  76. package/mcp_client/registration.py +343 -182
  77. package/memories/manager.py +147 -132
  78. package/memories/tui_commands.py +319 -244
  79. package/package.json +303 -73
  80. package/project_init/config_files.py +282 -278
  81. package/project_init/generator.py +728 -564
  82. package/project_init/manifest.py +18 -6
  83. package/project_init/tui_commands.py +147 -144
  84. package/prompts/critic_pass.md +24 -24
  85. package/prompts/revalidate.md +25 -25
  86. package/prompts/revise.md +2 -0
  87. package/prompts/source_grounding.md +66 -66
  88. package/prompts/system_prompts.py +316 -262
  89. package/providers.json.example +4 -2
  90. package/pyproject.toml +149 -100
  91. package/requirements.txt +295 -72
  92. package/safety/policy_enforcement.py +1082 -746
  93. package/security/askpass.py +419 -0
  94. package/security/capability.py +207 -177
  95. package/security/confine.py +484 -0
  96. package/security/hosted.py +182 -0
  97. package/security/permissions.py +237 -190
  98. package/security/posture.py +124 -12
  99. package/security/protected_paths.py +501 -309
  100. package/security/sandbox.py +4108 -985
  101. package/security/secrets.py +855 -0
  102. package/skills/builtin/math/formal-specification.md +67 -67
  103. package/skills/builtin/math/numerical-methods.md +66 -66
  104. package/skills/builtin/research/experiment-design.md +62 -62
  105. package/skills/builtin/research/reproducibility.md +55 -55
  106. package/skills/builtin/research/source-evaluation.md +70 -70
  107. package/skills/builtin/research/statistical-inference.md +63 -63
  108. package/skills/builtin/research/technical-writing.md +59 -59
  109. package/skills/builtin/software/code-review.md +61 -61
  110. package/skills/builtin/software/debugging.md +61 -61
  111. package/skills/builtin/software/test-design.md +67 -67
  112. package/skills/tui_commands.py +110 -111
  113. package/storage.py +1338 -1005
  114. package/tools/base.py +161 -1
  115. package/tools/builtin/_math_common.py +667 -377
  116. package/tools/builtin/_net_common.py +225 -19
  117. package/tools/builtin/arxiv.py +497 -39
  118. package/tools/builtin/ask_user.py +231 -213
  119. package/tools/builtin/discrete_math.py +35 -4
  120. package/tools/builtin/fetch_url.py +371 -153
  121. package/tools/builtin/file_ops.py +720 -412
  122. package/tools/builtin/geometry.py +48 -12
  123. package/tools/builtin/get_time.py +2 -2
  124. package/tools/builtin/linear_algebra.py +66 -23
  125. package/tools/builtin/load_skill.py +76 -66
  126. package/tools/builtin/logic.py +171 -12
  127. package/tools/builtin/pin.py +227 -193
  128. package/tools/builtin/probability_stats.py +12 -16
  129. package/tools/builtin/project_docs.py +464 -405
  130. package/tools/builtin/remember.py +175 -158
  131. package/tools/builtin/shell.py +980 -461
  132. package/tools/builtin/shell_sessions.py +613 -0
  133. package/tools/builtin/symbolic_math.py +80 -10
  134. package/tools/builtin/todo.py +149 -147
  135. package/tools/builtin/web_search.py +191 -13
  136. package/tools/context.py +159 -136
  137. package/tools/isolation.py +107 -34
  138. package/tools/registry.py +1242 -925
  139. package/tui/app.py +8318 -4100
  140. package/tui/app.tcss +747 -556
  141. package/tui/commands.py +26 -170
  142. package/tui/diffs.py +34 -13
  143. package/tui/history.py +128 -128
  144. package/tui/hosted_driver.py +102 -0
  145. package/tui/markdown.py +1329 -1279
  146. package/tui/meters.py +364 -268
  147. package/tui/ravens.py +146 -146
  148. package/tui/screens.py +1255 -943
  149. package/tui/themes.py +599 -559
  150. package/tui/widgets.py +3813 -2332
  151. package/SECURITY.md +0 -88
  152. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -1,327 +1,433 @@
1
- """
2
- core/reasoning/source_corpus.py
3
-
4
- What a claim was actually grounded IN, kept for the length of one run.
5
-
6
- ROADMAP_v2 §45 (SQ2). Pass 3a's `sources` list has always been URLs and two
7
- model-invented floats. The text those URLs served reached the model, was
8
- stringified into the pass thread, and was then unreachable: nothing in
9
- `03_grounding.json` could say what a page had said, so `similarity_score`
10
- could not be checked against anything and a quote could not be checked at
11
- all. `fetch_url.py`'s own comment already describes the artifact this
12
- builds ("output_writer builds each run's sources/ directory from it") --
13
- that directory has existed and been empty since §12.
14
-
15
- WHY A SECOND SINK RATHER THAN A CHANGE TO _translate. The orchestrator
16
- already sees every tool result and deliberately forwards only `ok` plus an
17
- error string to the UI, because a successful result body is not
18
- observability. That boundary is unchanged. This is a different consumer
19
- with a different need, attached beside it -- the corpus is read by the
20
- scoring stage and written to the run directory, and neither is a
21
- transcript.
22
-
23
- REDACTED ON ENTRY, THOUGH IT ARRIVES REDACTED. `registry.dispatch` runs
24
- every result through `check_output_policy` before the loop emits it, so
25
- what lands here has already been through `redact_output_text`. It is run
26
- again anyway, and for `_translate`'s stated reason: one rule owned at one
27
- boundary -- nothing is stored by this module unredacted -- beats depending
28
- on a guarantee made two layers away that a later refactor could quietly
29
- drop. The second pass is a no-op on already-clean text.
30
-
31
- THIS MODULE OWNS URL IDENTITY for the pipeline. Two spellings of one arXiv
32
- paper are one source, and a model that cites `/abs/2005.14165v3` after
33
- fetching `/abs/2005.14165` must not be told its own citation has no text.
34
- `scholar.py` resolves the same URLs and imports `arxiv_id_from_url` from
35
- here rather than re-deriving it -- a second copy of "what counts as the
36
- same paper" is how the corpus and the scholarly lookup come to disagree
37
- about one source in one artifact.
38
- """
39
-
40
- from __future__ import annotations
41
-
42
- import hashlib
43
- import logging
44
- import re
45
- from dataclasses import dataclass, field
46
- from datetime import datetime, timezone
47
- from urllib.parse import urlsplit, urlunsplit
48
-
49
- from safety.policy_enforcement import redact_output_text
50
-
51
- logger = logging.getLogger(__name__)
52
-
53
- # Bounds, not tuning. A research pass calls fetch_url a bounded number of
54
- # times (§31 budgets), so these are a backstop against a pathological run
55
- # rather than a knob anyone is expected to turn: the corpus is held in
56
- # memory for the length of a run and then written to disk.
57
- #
58
- # MAX_DOCUMENT_CHARS is deliberately well ABOVE fetch_url's own
59
- # MAX_CONTENT_CHARS (5000). It bounds this module against a future tool
60
- # that returns more; it is not a second truncation of the tools that exist,
61
- # because a cap that bites in normal operation would silently shorten the
62
- # text a similarity score is computed from.
63
- MAX_DOCUMENTS = 200
64
- MAX_DOCUMENT_CHARS = 20_000
65
-
66
- # Below this, a "document" is a title and a fragment -- a web_search
67
- # snippet for a result nobody fetched. Kept, because a 300-char snippet is
68
- # still the only text some cited URLs ever have, and scoring a claim
69
- # against it is better than reporting no evidence at all. The constant
70
- # exists to reject the empty and near-empty case, not the short one.
71
- MIN_DOCUMENT_CHARS = 16
72
-
73
- _ARXIV_HOSTS = {"arxiv.org", "www.arxiv.org", "export.arxiv.org"}
74
- # `2005.14165`, `2005.14165v3`, and the pre-2007 `math.GT/0309136` form --
75
- # whose archive prefix contains a SLASH, which is why the id group cannot
76
- # simply exclude one. A path with any further segment is not an arXiv id.
77
- _ARXIV_PATH_RE = re.compile(
78
- r"^/(?:abs|pdf)/"
79
- r"(?P<id>(?:[a-z-]+(?:\.[A-Za-z]{2})?/)?[^/?#]+?)"
80
- r"(?:v\d+)?(?:\.pdf)?$",
81
- re.IGNORECASE,
82
- )
83
-
84
-
85
- @dataclass
86
- class SourceDocument:
87
- """One URL's best available text, and where it came from.
88
-
89
- `tool` and `retrieved_at` are provenance rather than decoration: a
90
- similarity score computed from a 300-char search snippet and one
91
- computed from a fetched page are not the same measurement, and a
92
- reader of the artifact has to be able to tell which they are looking
93
- at.
94
- """
95
-
96
- url: str
97
- text: str
98
- title: str = ""
99
- tool: str = ""
100
- retrieved_at: str = ""
101
-
102
- @property
103
- def sha256(self) -> str:
104
- return hashlib.sha256(self.text.encode("utf-8")).hexdigest()
105
-
106
- def as_artifact(self) -> dict:
107
- return {
108
- "url": self.url,
109
- "title": self.title,
110
- "tool": self.tool,
111
- "retrieved_at": self.retrieved_at,
112
- "sha256": self.sha256,
113
- "chars": len(self.text),
114
- "text": self.text,
115
- }
116
-
117
-
118
- def _now() -> str:
119
- return datetime.now(timezone.utc).isoformat(timespec="seconds")
120
-
121
-
122
- def arxiv_id_from_url(url: str) -> str | None:
123
- """The bare arXiv id in `url`, version stripped, or None.
124
-
125
- Version stripped because a version is not a different paper for any
126
- purpose this pipeline has: the corpus keys on it, and OpenAlex's DOI
127
- for `2005.14165v3` is `10.48550/arXiv.2005.14165`.
128
- """
129
- try:
130
- parts = urlsplit(url)
131
- except ValueError:
132
- return None
133
- if parts.hostname is None or parts.hostname.lower() not in _ARXIV_HOSTS:
134
- return None
135
- match = _ARXIV_PATH_RE.match(parts.path)
136
- return match.group("id") if match else None
137
-
138
-
139
- def normalize_url(url: str) -> str:
140
- """The key two spellings of one source have to share.
141
-
142
- Case-folds the scheme and host, drops the fragment, drops a default
143
- port, drops a bare trailing slash, and collapses every arXiv `abs`/`pdf`
144
- spelling onto one canonical `abs` URL. The QUERY IS KEPT: `?id=7` is a
145
- different page on most sites, and dropping it would merge sources that
146
- say different things.
147
-
148
- An unparseable URL is returned stripped rather than raising. This runs
149
- on model-supplied strings; a malformed one should fail to match a
150
- document, not fail the run.
151
- """
152
- raw = (url or "").strip()
153
- if not raw:
154
- return ""
155
-
156
- arxiv_id = arxiv_id_from_url(raw)
157
- if arxiv_id:
158
- return f"https://arxiv.org/abs/{arxiv_id}"
159
-
160
- try:
161
- parts = urlsplit(raw)
162
- except ValueError:
163
- return raw
164
-
165
- host = (parts.hostname or "").lower()
166
- if not host:
167
- return raw
168
- if parts.port and not (
169
- (parts.scheme == "http" and parts.port == 80)
170
- or (parts.scheme == "https" and parts.port == 443)
171
- ):
172
- host = f"{host}:{parts.port}"
173
-
174
- path = parts.path
175
- if path == "/":
176
- path = ""
177
-
178
- return urlunsplit((parts.scheme.lower(), host, path, parts.query, ""))
179
-
180
-
181
- @dataclass
182
- class SourceCorpus:
183
- """Every source text one run retrieved, keyed by normalized URL.
184
-
185
- Run-scoped and passed by reference, the way `run.granted_calls` is
186
- (§25): one object means the corpus is already complete at the moment
187
- the scoring stage reads it, rather than being assembled at the end by a
188
- run that reached the end.
189
-
190
- THE LONGEST TEXT WINS. One URL can arrive three times -- as a 300-char
191
- web_search snippet, as a 600-char arXiv abstract, and as 5000 chars
192
- from fetch_url -- and the question every consumer asks is "what is the
193
- most this run knows about this page". Last-write-wins would make the
194
- answer depend on the order the model happened to call its tools in.
195
- """
196
-
197
- documents: dict = field(default_factory=dict)
198
- dropped: int = 0
199
-
200
- # -- ingest ----------------------------------------------------------
201
-
202
- def add(self, tool_name: str, result) -> int:
203
- """Absorb one tool result. Returns how many documents it added or
204
- improved.
205
-
206
- Anything unrecognised is ignored silently: this is attached to
207
- EVERY tool result a research pass produces, and a `todo_write`
208
- return value is not a defect worth a log line. A recognised tool
209
- whose result is the error shape is also ignored -- `arxiv.py` and
210
- `web_search.py` both return errors rather than raising, so an
211
- error dict is an ordinary occurrence here.
212
- """
213
- if not isinstance(result, dict) or "error" in result:
214
- return 0
215
- if tool_name == "fetch_url":
216
- return self._add_fetch_url(result)
217
- if tool_name == "arxiv_search":
218
- return self._add_arxiv(result)
219
- if tool_name == "web_search":
220
- return self._add_web_search(result)
221
- return 0
222
-
223
- def _add_fetch_url(self, result: dict) -> int:
224
- # `url` is the URL that ANSWERED, after redirects -- which is what
225
- # fetch_url.py:89-95 says the grounding pass attributes the text
226
- # to, and therefore what the model will cite.
227
- return self._store(
228
- url=result.get("url"),
229
- text=result.get("content"),
230
- title="",
231
- tool="fetch_url",
232
- )
233
-
234
- def _add_arxiv(self, result: dict) -> int:
235
- added = 0
236
- for entry in result.get("results") or ():
237
- if not isinstance(entry, dict):
238
- continue
239
- arxiv_id = (entry.get("arxiv_id") or "").strip()
240
- if not arxiv_id:
241
- continue
242
- title = (entry.get("title") or "").strip()
243
- summary = (entry.get("summary") or "").strip()
244
- # Title AND abstract, joined. The title carries the paper's
245
- # subject in the compressed form a claim is usually phrased in,
246
- # and a 600-char abstract that opens on background prose can
247
- # otherwise match a claim about the paper's own topic poorly.
248
- text = "\n\n".join(p for p in (title, summary) if p)
249
- added += self._store(
250
- url=f"https://arxiv.org/abs/{arxiv_id}",
251
- text=text,
252
- title=title,
253
- tool="arxiv_search",
254
- )
255
- return added
256
-
257
- def _add_web_search(self, result: dict) -> int:
258
- added = 0
259
- for entry in result.get("results") or ():
260
- if not isinstance(entry, dict):
261
- continue
262
- title = (entry.get("title") or "").strip()
263
- snippet = (entry.get("snippet") or "").strip()
264
- text = "\n\n".join(p for p in (title, snippet) if p)
265
- added += self._store(
266
- url=entry.get("url"),
267
- text=text,
268
- title=title,
269
- tool="web_search",
270
- )
271
- return added
272
-
273
- def _store(self, url, text, title: str, tool: str) -> int:
274
- if not isinstance(url, str) or not isinstance(text, str):
275
- return 0
276
- key = normalize_url(url)
277
- if not key:
278
- return 0
279
-
280
- cleaned = redact_output_text(text).strip()
281
- if len(cleaned) < MIN_DOCUMENT_CHARS:
282
- return 0
283
- cleaned = cleaned[:MAX_DOCUMENT_CHARS]
284
-
285
- existing = self.documents.get(key)
286
- if existing is not None and len(existing.text) >= len(cleaned):
287
- return 0
288
- if existing is None and len(self.documents) >= MAX_DOCUMENTS:
289
- # Counted rather than logged per drop: a run that hits this is
290
- # hitting it repeatedly, and the count is what the trace line
291
- # at the end of the stage reports.
292
- self.dropped += 1
293
- return 0
294
-
295
- self.documents[key] = SourceDocument(
296
- url=key, text=cleaned, title=title or (existing.title if existing else ""),
297
- tool=tool, retrieved_at=_now(),
298
- )
299
- return 1
300
-
301
- # -- read ------------------------------------------------------------
302
-
303
- def get(self, url: str):
304
- """The document held for `url`, or None. Normalizes first, so a
305
- caller passing the model's spelling gets the fetched page."""
306
- return self.documents.get(normalize_url(url))
307
-
308
- def text_for(self, url: str) -> str:
309
- document = self.get(url)
310
- return document.text if document is not None else ""
311
-
312
- def __len__(self) -> int:
313
- return len(self.documents)
314
-
315
- def __bool__(self) -> bool:
316
- # Explicit, because __len__ alone would make an empty corpus
317
- # falsy and every `if corpus:` guard in the orchestrator would
318
- # then mean "if anything was fetched" rather than "if a corpus is
319
- # being collected at all". Those are different questions and the
320
- # first one is never the one being asked.
321
- return True
322
-
323
- def artifact_entries(self) -> list[dict]:
324
- """Every document, newest spelling of each URL, sorted for a
325
- stable diff between runs."""
326
- return [self.documents[key].as_artifact()
327
- for key in sorted(self.documents)]
1
+ """
2
+ core/reasoning/source_corpus.py
3
+
4
+ What a claim was actually grounded IN, kept for the length of one run.
5
+
6
+ ROADMAP_v2 §45 (SQ2). Pass 3a's `sources` list has always been URLs and two
7
+ model-invented floats. The text those URLs served reached the model, was
8
+ stringified into the pass thread, and was then unreachable: nothing in
9
+ `03_grounding.json` could say what a page had said, so `similarity_score`
10
+ could not be checked against anything and a quote could not be checked at
11
+ all. `fetch_url.py`'s own comment already describes the artifact this
12
+ builds ("output_writer builds each run's sources/ directory from it") --
13
+ that directory has existed and been empty since §12.
14
+
15
+ WHY A SECOND SINK RATHER THAN A CHANGE TO _translate. The orchestrator
16
+ already sees every tool result and deliberately forwards only `ok` plus an
17
+ error string to the UI, because a successful result body is not
18
+ observability. That boundary is unchanged. This is a different consumer
19
+ with a different need, attached beside it -- the corpus is read by the
20
+ scoring stage and written to the run directory, and neither is a
21
+ transcript.
22
+
23
+ REDACTED ON ENTRY, THOUGH IT ARRIVES REDACTED. `registry.dispatch` runs
24
+ every result through `check_output_policy` before the loop emits it, so
25
+ what lands here has already been through `redact_output_text`. It is run
26
+ again anyway, and for `_translate`'s stated reason: one rule owned at one
27
+ boundary -- nothing is stored by this module unredacted -- beats depending
28
+ on a guarantee made two layers away that a later refactor could quietly
29
+ drop. The second pass is a no-op on already-clean text.
30
+
31
+ AND IT IS NOT WHERE §52's SPLIT-CREDENTIAL FIX WENT, deliberately (RB4).
32
+ §51 made this module the one place where two pages of a document become
33
+ adjacent, so a credential straddling a page boundary was rejoined HERE --
34
+ reproduced end to end in batch 106, into `artifact_entries()` and therefore
35
+ into `sources/<sha256>.txt` and the window sent to the embedding provider.
36
+ The register's proposed fix was to re-redact the whole merged text on every
37
+ append. It was declined: `fetch_url` redacts before it cuts now, so the
38
+ halves never exist, and re-scanning up to 20,000 characters per page would
39
+ buy nothing this module can see -- while leaving unfixed the model's own two
40
+ tool results, which the producer-side fix does close.
41
+
42
+ THIS MODULE OWNS URL IDENTITY for the pipeline. Two spellings of one arXiv
43
+ paper are one source, and a model that cites `/abs/2005.14165v3` after
44
+ fetching `/abs/2005.14165` must not be told its own citation has no text.
45
+ `scholar.py` resolves the same URLs and imports `arxiv_id_from_url` from
46
+ here rather than re-deriving it -- a second copy of "what counts as the
47
+ same paper" is how the corpus and the scholarly lookup come to disagree
48
+ about one source in one artifact.
49
+ """
50
+
51
+ from __future__ import annotations
52
+
53
+ import hashlib
54
+ import logging
55
+ import re
56
+ from dataclasses import dataclass, field
57
+ from datetime import UTC, datetime
58
+ from urllib.parse import urlsplit, urlunsplit
59
+
60
+ from safety.policy_enforcement import redact_output_text
61
+
62
+ logger = logging.getLogger(__name__)
63
+
64
+ # Bounds, not tuning. A research pass calls fetch_url a bounded number of
65
+ # times (§31 budgets), so these are a backstop against a pathological run
66
+ # rather than a knob anyone is expected to turn: the corpus is held in
67
+ # memory for the length of a run and then written to disk.
68
+ #
69
+ # MAX_DOCUMENT_CHARS is deliberately well ABOVE fetch_url's own
70
+ # MAX_CONTENT_CHARS (5000). It bounds this module against a future tool
71
+ # that returns more; it is not a second truncation of the tools that exist,
72
+ # because a cap that bites in normal operation would silently shorten the
73
+ # text a similarity score is computed from.
74
+ MAX_DOCUMENTS = 200
75
+ MAX_DOCUMENT_CHARS = 20_000
76
+
77
+ # Below this, a "document" is a title and a fragment -- a web_search
78
+ # snippet for a result nobody fetched. Kept, because a 300-char snippet is
79
+ # still the only text some cited URLs ever have, and scoring a claim
80
+ # against it is better than reporting no evidence at all. The constant
81
+ # exists to reject the empty and near-empty case, not the short one.
82
+ MIN_DOCUMENT_CHARS = 16
83
+
84
+ _ARXIV_HOSTS = {"arxiv.org", "www.arxiv.org", "export.arxiv.org"}
85
+ # `2005.14165`, `2005.14165v3`, and the pre-2007 `math.GT/0309136` form --
86
+ # whose archive prefix contains a SLASH, which is why the id group cannot
87
+ # simply exclude one. A path with any further segment is not an arXiv id.
88
+ _ARXIV_PATH_RE = re.compile(
89
+ r"^/(?:abs|pdf)/"
90
+ r"(?P<id>(?:[a-z-]+(?:\.[A-Za-z]{2})?/)?[^/?#]+?)"
91
+ r"(?:v\d+)?(?:\.pdf)?$",
92
+ re.IGNORECASE,
93
+ )
94
+
95
+
96
+ @dataclass
97
+ class SourceDocument:
98
+ """One URL's best available text, and where it came from.
99
+
100
+ `tool` and `retrieved_at` are provenance rather than decoration: a
101
+ similarity score computed from a 300-char search snippet and one
102
+ computed from a fetched page are not the same measurement, and a
103
+ reader of the artifact has to be able to tell which they are looking
104
+ at.
105
+ """
106
+
107
+ url: str
108
+ text: str
109
+ title: str = ""
110
+ tool: str = ""
111
+ retrieved_at: str = ""
112
+
113
+ #: ROADMAP_v3 §51 (PG7). How far into the SOURCE document the held text
114
+ #: reaches, in the source's own character positions -- which is what
115
+ #: `fetch_url` pages by, and not the same as `len(text)`. 0 for a
116
+ #: document no paging tool produced.
117
+ #:
118
+ #: §52 (RB1) removed ONE of the two reasons those numbers differ.
119
+ #: Redaction used to happen here, on each page, so a page's held length
120
+ #: could be shorter than the span it covered; `fetch_url` now redacts
121
+ #: before it cuts, so its pages arrive clean and that gap is gone. What
122
+ #: still separates the two is `.strip()` and `MAX_DOCUMENT_CHARS`, both
123
+ #: of which change how much is HELD and neither of which moves where a
124
+ #: page sat in the document it came from.
125
+ #:
126
+ #: It is what stops a page being absorbed twice. Without it, a model
127
+ #: that re-reads page 2 -- which it will, because re-reading is how a
128
+ #: model checks itself -- would append it again and the corpus would
129
+ #: hold the same paragraph twice, which a similarity score then counts
130
+ #: twice.
131
+ next_offset: int = 0
132
+
133
+ @property
134
+ def sha256(self) -> str:
135
+ return hashlib.sha256(self.text.encode("utf-8")).hexdigest()
136
+
137
+ def as_artifact(self) -> dict:
138
+ return {
139
+ "url": self.url,
140
+ "title": self.title,
141
+ "tool": self.tool,
142
+ "retrieved_at": self.retrieved_at,
143
+ "sha256": self.sha256,
144
+ "chars": len(self.text),
145
+ "text": self.text,
146
+ }
147
+
148
+
149
+ def _now() -> str:
150
+ return datetime.now(UTC).isoformat(timespec="seconds")
151
+
152
+
153
+ def arxiv_id_from_url(url: str) -> str | None:
154
+ """The bare arXiv id in `url`, version stripped, or None.
155
+
156
+ Version stripped because a version is not a different paper for any
157
+ purpose this pipeline has: the corpus keys on it, and OpenAlex's DOI
158
+ for `2005.14165v3` is `10.48550/arXiv.2005.14165`.
159
+ """
160
+ try:
161
+ parts = urlsplit(url)
162
+ except ValueError:
163
+ return None
164
+ if parts.hostname is None or parts.hostname.lower() not in _ARXIV_HOSTS:
165
+ return None
166
+ match = _ARXIV_PATH_RE.match(parts.path)
167
+ return match.group("id") if match else None
168
+
169
+
170
+ def normalize_url(url: str) -> str:
171
+ """The key two spellings of one source have to share.
172
+
173
+ Case-folds the scheme and host, drops the fragment, drops a default
174
+ port, drops a bare trailing slash, and collapses every arXiv `abs`/`pdf`
175
+ spelling onto one canonical `abs` URL. The QUERY IS KEPT: `?id=7` is a
176
+ different page on most sites, and dropping it would merge sources that
177
+ say different things.
178
+
179
+ An unparseable URL is returned stripped rather than raising. This runs
180
+ on model-supplied strings; a malformed one should fail to match a
181
+ document, not fail the run.
182
+ """
183
+ raw = (url or "").strip()
184
+ if not raw:
185
+ return ""
186
+
187
+ arxiv_id = arxiv_id_from_url(raw)
188
+ if arxiv_id:
189
+ return f"https://arxiv.org/abs/{arxiv_id}"
190
+
191
+ try:
192
+ parts = urlsplit(raw)
193
+ except ValueError:
194
+ return raw
195
+
196
+ host = (parts.hostname or "").lower()
197
+ if not host:
198
+ return raw
199
+ if parts.port and not (
200
+ (parts.scheme == "http" and parts.port == 80)
201
+ or (parts.scheme == "https" and parts.port == 443)
202
+ ):
203
+ host = f"{host}:{parts.port}"
204
+
205
+ path = parts.path
206
+ if path == "/":
207
+ path = ""
208
+
209
+ return urlunsplit((parts.scheme.lower(), host, path, parts.query, ""))
210
+
211
+
212
+ @dataclass
213
+ class SourceCorpus:
214
+ """Every source text one run retrieved, keyed by normalized URL.
215
+
216
+ Run-scoped and passed by reference, the way `run.granted_calls` is
217
+ (§25): one object means the corpus is already complete at the moment
218
+ the scoring stage reads it, rather than being assembled at the end by a
219
+ run that reached the end.
220
+
221
+ THE LONGEST TEXT WINS. One URL can arrive three times -- as a 300-char
222
+ web_search snippet, as a 600-char arXiv abstract, and as 5000 chars
223
+ from fetch_url -- and the question every consumer asks is "what is the
224
+ most this run knows about this page". Last-write-wins would make the
225
+ answer depend on the order the model happened to call its tools in.
226
+ """
227
+
228
+ documents: dict = field(default_factory=dict)
229
+ dropped: int = 0
230
+
231
+ # -- ingest ----------------------------------------------------------
232
+
233
+ def add(self, tool_name: str, result) -> int:
234
+ """Absorb one tool result. Returns how many documents it added or
235
+ improved.
236
+
237
+ Anything unrecognised is ignored silently: this is attached to
238
+ EVERY tool result a research pass produces, and a `todo_write`
239
+ return value is not a defect worth a log line. A recognised tool
240
+ whose result is the error shape is also ignored -- `arxiv.py` and
241
+ `web_search.py` both return errors rather than raising, so an
242
+ error dict is an ordinary occurrence here.
243
+ """
244
+ if not isinstance(result, dict) or "error" in result:
245
+ return 0
246
+ if tool_name == "fetch_url":
247
+ return self._add_fetch_url(result)
248
+ if tool_name == "arxiv_search":
249
+ return self._add_arxiv(result)
250
+ if tool_name == "web_search":
251
+ return self._add_web_search(result)
252
+ return 0
253
+
254
+ def _add_fetch_url(self, result: dict) -> int:
255
+ # `url` is the URL that ANSWERED, after redirects -- which is what
256
+ # fetch_url.py:89-95 says the grounding pass attributes the text
257
+ # to, and therefore what the model will cite.
258
+ #
259
+ # `offset` is ROADMAP_v3 §51 (PG7). A result without one is a
260
+ # pre-§51 shape or a hand-built fixture, and 0 is the right reading
261
+ # of both: a whole document starting at the beginning.
262
+ offset = result.get("offset")
263
+ if not isinstance(offset, int) or offset < 0:
264
+ offset = 0
265
+ return self._store(
266
+ url=result.get("url"),
267
+ text=result.get("content"),
268
+ title="",
269
+ tool="fetch_url",
270
+ offset=offset,
271
+ )
272
+
273
+ def _add_arxiv(self, result: dict) -> int:
274
+ added = 0
275
+ for entry in result.get("results") or ():
276
+ if not isinstance(entry, dict):
277
+ continue
278
+ arxiv_id = (entry.get("arxiv_id") or "").strip()
279
+ if not arxiv_id:
280
+ continue
281
+ title = (entry.get("title") or "").strip()
282
+ summary = (entry.get("summary") or "").strip()
283
+ # Title AND abstract, joined. The title carries the paper's
284
+ # subject in the compressed form a claim is usually phrased in,
285
+ # and a 600-char abstract that opens on background prose can
286
+ # otherwise match a claim about the paper's own topic poorly.
287
+ text = "\n\n".join(p for p in (title, summary) if p)
288
+ added += self._store(
289
+ url=f"https://arxiv.org/abs/{arxiv_id}",
290
+ text=text,
291
+ title=title,
292
+ tool="arxiv_search",
293
+ )
294
+ return added
295
+
296
+ def _add_web_search(self, result: dict) -> int:
297
+ added = 0
298
+ for entry in result.get("results") or ():
299
+ if not isinstance(entry, dict):
300
+ continue
301
+ title = (entry.get("title") or "").strip()
302
+ snippet = (entry.get("snippet") or "").strip()
303
+ text = "\n\n".join(p for p in (title, snippet) if p)
304
+ added += self._store(
305
+ url=entry.get("url"),
306
+ text=text,
307
+ title=title,
308
+ tool="web_search",
309
+ )
310
+ return added
311
+
312
+ def _store(self, url, text, title: str, tool: str, offset=None) -> int:
313
+ """Absorb one document, or one PAGE of one (ROADMAP_v3 §51, PG7).
314
+
315
+ `offset` is the source position this text starts at, or None from a
316
+ tool that does not page. None and 0 are NOT the same thing here:
317
+ None means "this is the whole of what that tool returns", 0 means
318
+ "this is the first page of something that may have more".
319
+
320
+ WHY THE PAGED CASE CANNOT USE LONGEST-WINS. Measured, batch 105,
321
+ before any of this was written: a corpus given page 1 and then page
322
+ 2 of one URL returned 1 then **0** -- `len(existing.text) >=
323
+ len(cleaned)` is 5000 >= 5000, so page 2 was DROPPED and the held
324
+ text stayed page 1. Paging would then have made the model better
325
+ informed and the grounding pass no better, and the divergence is
326
+ invisible from both sides: the model reads four pages and the score
327
+ is computed against the first. The near miss is worse than the
328
+ miss -- a page 2 one character longer REPLACED page 1 outright,
329
+ leaving the corpus holding the middle of a document and attributing
330
+ it to the whole URL.
331
+ """
332
+ if not isinstance(url, str) or not isinstance(text, str):
333
+ return 0
334
+ key = normalize_url(url)
335
+ if not key:
336
+ return 0
337
+
338
+ existing = self.documents.get(key)
339
+
340
+ # A continuation extends what is held; anything else competes on
341
+ # length exactly as it did before §51.
342
+ continuing = (
343
+ offset is not None
344
+ and offset > 0
345
+ and existing is not None
346
+ and existing.tool == tool
347
+ )
348
+
349
+ if continuing:
350
+ # How much of this page is already held. Covers all three
351
+ # shapes with one subtraction: an exact continuation overlaps
352
+ # by nothing, a re-read of the same page overlaps entirely and
353
+ # contributes the empty string, and a page that starts inside
354
+ # what is held contributes only its tail. A page starting
355
+ # BEYOND the held text leaves a gap -- appended anyway, because
356
+ # the text is real and the alternative is losing it, and an
357
+ # incomplete document is what this corpus already holds for
358
+ # every source the byte cap cut.
359
+ #
360
+ # There is deliberately NO `if overlap >= len(text): return 0`
361
+ # above this. One was written, and a mutation pass showed it
362
+ # could be deleted with no test failing anywhere -- because the
363
+ # slice already yields "" in exactly the cases it guarded, and
364
+ # the empty-addition check below catches them. Batch 103's
365
+ # lesson applies to a guard that cannot fire as much as to one
366
+ # that fires in the wrong place: it reads as the thing
367
+ # protecting you.
368
+ overlap = existing.next_offset - offset
369
+ addition = redact_output_text(text[max(0, overlap):])
370
+ if not addition.strip():
371
+ return 0
372
+ merged = (existing.text + addition)[:MAX_DOCUMENT_CHARS]
373
+ if merged == existing.text:
374
+ return 0
375
+ self.documents[key] = SourceDocument(
376
+ url=key, text=merged, title=existing.title,
377
+ tool=tool, retrieved_at=_now(),
378
+ next_offset=max(existing.next_offset, offset + len(text)),
379
+ )
380
+ return 1
381
+
382
+ cleaned = redact_output_text(text).strip()
383
+ if len(cleaned) < MIN_DOCUMENT_CHARS:
384
+ return 0
385
+ cleaned = cleaned[:MAX_DOCUMENT_CHARS]
386
+
387
+ if existing is not None and len(existing.text) >= len(cleaned):
388
+ return 0
389
+ if existing is None and len(self.documents) >= MAX_DOCUMENTS:
390
+ # Counted rather than logged per drop: a run that hits this is
391
+ # hitting it repeatedly, and the count is what the trace line
392
+ # at the end of the stage reports.
393
+ self.dropped += 1
394
+ return 0
395
+
396
+ self.documents[key] = SourceDocument(
397
+ url=key, text=cleaned, title=title or (existing.title if existing else ""),
398
+ tool=tool, retrieved_at=_now(),
399
+ # In the SOURCE's positions, so the next page's offset can be
400
+ # compared against it. `len(text)` and not `len(cleaned)`:
401
+ # redaction and stripping change how much is HELD, never where
402
+ # the page sat in the document it came from.
403
+ next_offset=(offset + len(text)) if offset is not None else 0,
404
+ )
405
+ return 1
406
+
407
+ # -- read ------------------------------------------------------------
408
+
409
+ def get(self, url: str):
410
+ """The document held for `url`, or None. Normalizes first, so a
411
+ caller passing the model's spelling gets the fetched page."""
412
+ return self.documents.get(normalize_url(url))
413
+
414
+ def text_for(self, url: str) -> str:
415
+ document = self.get(url)
416
+ return document.text if document is not None else ""
417
+
418
+ def __len__(self) -> int:
419
+ return len(self.documents)
420
+
421
+ def __bool__(self) -> bool:
422
+ # Explicit, because __len__ alone would make an empty corpus
423
+ # falsy and every `if corpus:` guard in the orchestrator would
424
+ # then mean "if anything was fetched" rather than "if a corpus is
425
+ # being collected at all". Those are different questions and the
426
+ # first one is never the one being asked.
427
+ return True
428
+
429
+ def artifact_entries(self) -> list[dict]:
430
+ """Every document, newest spelling of each URL, sorted for a
431
+ stable diff between runs."""
432
+ return [self.documents[key].as_artifact()
433
+ for key in sorted(self.documents)]