@yahyahammad/venastine 0.3.8 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/.env.example +19 -19
  2. package/CONFIG_ARCHITECTURE.md +2030 -2002
  3. package/LICENSE +201 -201
  4. package/README.md +1034 -1034
  5. package/agents/builtin/compactor.md +75 -75
  6. package/agents/builtin/explore.md +77 -77
  7. package/agents/builtin/pipeline-reviewer.md +100 -100
  8. package/agents/builtin/plan.md +106 -106
  9. package/agents/builtin/review.md +74 -74
  10. package/agents/manager.py +261 -261
  11. package/agents/subagent_tool.py +472 -472
  12. package/agents/tui_commands.py +238 -146
  13. package/bin/venastine.mjs +528 -528
  14. package/bin/venastine.py +71 -71
  15. package/cli_commands.py +838 -835
  16. package/config.yaml +13 -0
  17. package/config_edit.py +1301 -1301
  18. package/config_schema.py +971 -966
  19. package/config_update.py +749 -749
  20. package/core/agent_activity.py +256 -256
  21. package/core/ansi.py +452 -452
  22. package/core/approval.py +137 -137
  23. package/core/call_view.py +364 -364
  24. package/core/cancel.py +121 -121
  25. package/core/chatgpt_auth.py +747 -747
  26. package/core/claims_view.py +107 -107
  27. package/core/client.py +2715 -2715
  28. package/core/commands.py +460 -454
  29. package/core/compaction.py +1735 -1735
  30. package/core/config_loader.py +1982 -1982
  31. package/core/connect.py +686 -686
  32. package/core/events.py +123 -123
  33. package/core/file_mentions.py +280 -280
  34. package/core/interaction.py +411 -411
  35. package/core/loop.py +2566 -2566
  36. package/core/memory.py +482 -482
  37. package/core/pipeline_models.py +190 -190
  38. package/core/provider_errors.py +234 -234
  39. package/core/reasoning/authorization.py +130 -130
  40. package/core/reasoning/base.py +219 -219
  41. package/core/reasoning/confidence_scoring.py +484 -484
  42. package/core/reasoning/events.py +149 -149
  43. package/core/reasoning/orchestrator.py +1561 -1561
  44. package/core/reasoning/output_writer.py +345 -345
  45. package/core/reasoning/payload_validation.py +524 -524
  46. package/core/reasoning/pipeline_storage.py +427 -427
  47. package/core/reasoning/scholar.py +458 -458
  48. package/core/reasoning/source_corpus.py +433 -433
  49. package/core/reasoning/source_scoring.py +855 -855
  50. package/core/replay.py +315 -315
  51. package/core/shell_sessions.py +1553 -1553
  52. package/credentials.py +307 -307
  53. package/database.py +333 -333
  54. package/docs/PRIVACY.md +78 -78
  55. package/docs/SECURITY.md +108 -108
  56. package/docs/THIRD_PARTY_NOTICES.md +148 -148
  57. package/env_secrets.py +118 -118
  58. package/json_store.py +343 -343
  59. package/logging_setup.py +295 -295
  60. package/main.py +3433 -3433
  61. package/mcp_client/client.py +733 -733
  62. package/mcp_client/config.py +484 -484
  63. package/mcp_client/registration.py +343 -343
  64. package/memories/manager.py +147 -147
  65. package/memories/tui_commands.py +319 -239
  66. package/package.json +303 -303
  67. package/project_init/config_files.py +282 -282
  68. package/project_init/generator.py +728 -728
  69. package/project_init/manifest.py +418 -418
  70. package/project_init/tui_commands.py +147 -147
  71. package/prompts/critic_pass.md +24 -24
  72. package/prompts/revalidate.md +25 -25
  73. package/prompts/revise.md +17 -17
  74. package/prompts/source_grounding.md +66 -66
  75. package/prompts/system_prompts.py +316 -316
  76. package/providers.json.example +94 -94
  77. package/pyproject.toml +149 -149
  78. package/requirements.txt +295 -295
  79. package/safety/policy_enforcement.py +1082 -1082
  80. package/security/askpass.py +419 -419
  81. package/security/capability.py +207 -207
  82. package/security/confine.py +484 -484
  83. package/security/hosted.py +182 -182
  84. package/security/permissions.py +237 -237
  85. package/security/protected_paths.py +501 -501
  86. package/security/sandbox.py +4108 -4108
  87. package/skills/builtin/math/formal-specification.md +67 -67
  88. package/skills/builtin/math/numerical-methods.md +66 -66
  89. package/skills/builtin/research/experiment-design.md +62 -62
  90. package/skills/builtin/research/reproducibility.md +55 -55
  91. package/skills/builtin/research/source-evaluation.md +70 -70
  92. package/skills/builtin/research/statistical-inference.md +63 -63
  93. package/skills/builtin/research/technical-writing.md +59 -59
  94. package/skills/builtin/software/code-review.md +61 -61
  95. package/skills/builtin/software/debugging.md +61 -61
  96. package/skills/builtin/software/test-design.md +67 -67
  97. package/skills/tui_commands.py +110 -110
  98. package/storage.py +1338 -1338
  99. package/tools/base.py +465 -465
  100. package/tools/builtin/_math_common.py +667 -667
  101. package/tools/builtin/_net_common.py +267 -267
  102. package/tools/builtin/arxiv.py +681 -681
  103. package/tools/builtin/fetch_url.py +371 -371
  104. package/tools/builtin/file_ops.py +720 -720
  105. package/tools/builtin/geometry.py +184 -184
  106. package/tools/builtin/load_skill.py +76 -76
  107. package/tools/builtin/pin.py +227 -227
  108. package/tools/builtin/probability_stats.py +224 -224
  109. package/tools/builtin/project_docs.py +464 -464
  110. package/tools/builtin/remember.py +175 -175
  111. package/tools/builtin/shell.py +980 -980
  112. package/tools/builtin/todo.py +149 -149
  113. package/tools/builtin/web_search.py +328 -328
  114. package/tools/context.py +159 -159
  115. package/tools/isolation.py +387 -387
  116. package/tools/registry.py +1242 -1242
  117. package/tui/app.py +8318 -8307
  118. package/tui/app.tcss +747 -739
  119. package/tui/commands.py +26 -26
  120. package/tui/diffs.py +286 -286
  121. package/tui/history.py +128 -128
  122. package/tui/hosted_driver.py +102 -102
  123. package/tui/markdown.py +1329 -1329
  124. package/tui/meters.py +364 -364
  125. package/tui/ravens.py +146 -146
  126. package/tui/screens.py +1255 -1197
  127. package/tui/themes.py +599 -599
  128. package/tui/widgets.py +3813 -3796
@@ -1,433 +1,433 @@
1
- """
2
- core/reasoning/source_corpus.py
3
-
4
- What a claim was actually grounded IN, kept for the length of one run.
5
-
6
- ROADMAP_v2 §45 (SQ2). Pass 3a's `sources` list has always been URLs and two
7
- model-invented floats. The text those URLs served reached the model, was
8
- stringified into the pass thread, and was then unreachable: nothing in
9
- `03_grounding.json` could say what a page had said, so `similarity_score`
10
- could not be checked against anything and a quote could not be checked at
11
- all. `fetch_url.py`'s own comment already describes the artifact this
12
- builds ("output_writer builds each run's sources/ directory from it") --
13
- that directory has existed and been empty since §12.
14
-
15
- WHY A SECOND SINK RATHER THAN A CHANGE TO _translate. The orchestrator
16
- already sees every tool result and deliberately forwards only `ok` plus an
17
- error string to the UI, because a successful result body is not
18
- observability. That boundary is unchanged. This is a different consumer
19
- with a different need, attached beside it -- the corpus is read by the
20
- scoring stage and written to the run directory, and neither is a
21
- transcript.
22
-
23
- REDACTED ON ENTRY, THOUGH IT ARRIVES REDACTED. `registry.dispatch` runs
24
- every result through `check_output_policy` before the loop emits it, so
25
- what lands here has already been through `redact_output_text`. It is run
26
- again anyway, and for `_translate`'s stated reason: one rule owned at one
27
- boundary -- nothing is stored by this module unredacted -- beats depending
28
- on a guarantee made two layers away that a later refactor could quietly
29
- drop. The second pass is a no-op on already-clean text.
30
-
31
- AND IT IS NOT WHERE §52's SPLIT-CREDENTIAL FIX WENT, deliberately (RB4).
32
- §51 made this module the one place where two pages of a document become
33
- adjacent, so a credential straddling a page boundary was rejoined HERE --
34
- reproduced end to end in batch 106, into `artifact_entries()` and therefore
35
- into `sources/<sha256>.txt` and the window sent to the embedding provider.
36
- The register's proposed fix was to re-redact the whole merged text on every
37
- append. It was declined: `fetch_url` redacts before it cuts now, so the
38
- halves never exist, and re-scanning up to 20,000 characters per page would
39
- buy nothing this module can see -- while leaving unfixed the model's own two
40
- tool results, which the producer-side fix does close.
41
-
42
- THIS MODULE OWNS URL IDENTITY for the pipeline. Two spellings of one arXiv
43
- paper are one source, and a model that cites `/abs/2005.14165v3` after
44
- fetching `/abs/2005.14165` must not be told its own citation has no text.
45
- `scholar.py` resolves the same URLs and imports `arxiv_id_from_url` from
46
- here rather than re-deriving it -- a second copy of "what counts as the
47
- same paper" is how the corpus and the scholarly lookup come to disagree
48
- about one source in one artifact.
49
- """
50
-
51
- from __future__ import annotations
52
-
53
- import hashlib
54
- import logging
55
- import re
56
- from dataclasses import dataclass, field
57
- from datetime import UTC, datetime
58
- from urllib.parse import urlsplit, urlunsplit
59
-
60
- from safety.policy_enforcement import redact_output_text
61
-
62
- logger = logging.getLogger(__name__)
63
-
64
- # Bounds, not tuning. A research pass calls fetch_url a bounded number of
65
- # times (§31 budgets), so these are a backstop against a pathological run
66
- # rather than a knob anyone is expected to turn: the corpus is held in
67
- # memory for the length of a run and then written to disk.
68
- #
69
- # MAX_DOCUMENT_CHARS is deliberately well ABOVE fetch_url's own
70
- # MAX_CONTENT_CHARS (5000). It bounds this module against a future tool
71
- # that returns more; it is not a second truncation of the tools that exist,
72
- # because a cap that bites in normal operation would silently shorten the
73
- # text a similarity score is computed from.
74
- MAX_DOCUMENTS = 200
75
- MAX_DOCUMENT_CHARS = 20_000
76
-
77
- # Below this, a "document" is a title and a fragment -- a web_search
78
- # snippet for a result nobody fetched. Kept, because a 300-char snippet is
79
- # still the only text some cited URLs ever have, and scoring a claim
80
- # against it is better than reporting no evidence at all. The constant
81
- # exists to reject the empty and near-empty case, not the short one.
82
- MIN_DOCUMENT_CHARS = 16
83
-
84
- _ARXIV_HOSTS = {"arxiv.org", "www.arxiv.org", "export.arxiv.org"}
85
- # `2005.14165`, `2005.14165v3`, and the pre-2007 `math.GT/0309136` form --
86
- # whose archive prefix contains a SLASH, which is why the id group cannot
87
- # simply exclude one. A path with any further segment is not an arXiv id.
88
- _ARXIV_PATH_RE = re.compile(
89
- r"^/(?:abs|pdf)/"
90
- r"(?P<id>(?:[a-z-]+(?:\.[A-Za-z]{2})?/)?[^/?#]+?)"
91
- r"(?:v\d+)?(?:\.pdf)?$",
92
- re.IGNORECASE,
93
- )
94
-
95
-
96
- @dataclass
97
- class SourceDocument:
98
- """One URL's best available text, and where it came from.
99
-
100
- `tool` and `retrieved_at` are provenance rather than decoration: a
101
- similarity score computed from a 300-char search snippet and one
102
- computed from a fetched page are not the same measurement, and a
103
- reader of the artifact has to be able to tell which they are looking
104
- at.
105
- """
106
-
107
- url: str
108
- text: str
109
- title: str = ""
110
- tool: str = ""
111
- retrieved_at: str = ""
112
-
113
- #: ROADMAP_v3 §51 (PG7). How far into the SOURCE document the held text
114
- #: reaches, in the source's own character positions -- which is what
115
- #: `fetch_url` pages by, and not the same as `len(text)`. 0 for a
116
- #: document no paging tool produced.
117
- #:
118
- #: §52 (RB1) removed ONE of the two reasons those numbers differ.
119
- #: Redaction used to happen here, on each page, so a page's held length
120
- #: could be shorter than the span it covered; `fetch_url` now redacts
121
- #: before it cuts, so its pages arrive clean and that gap is gone. What
122
- #: still separates the two is `.strip()` and `MAX_DOCUMENT_CHARS`, both
123
- #: of which change how much is HELD and neither of which moves where a
124
- #: page sat in the document it came from.
125
- #:
126
- #: It is what stops a page being absorbed twice. Without it, a model
127
- #: that re-reads page 2 -- which it will, because re-reading is how a
128
- #: model checks itself -- would append it again and the corpus would
129
- #: hold the same paragraph twice, which a similarity score then counts
130
- #: twice.
131
- next_offset: int = 0
132
-
133
- @property
134
- def sha256(self) -> str:
135
- return hashlib.sha256(self.text.encode("utf-8")).hexdigest()
136
-
137
- def as_artifact(self) -> dict:
138
- return {
139
- "url": self.url,
140
- "title": self.title,
141
- "tool": self.tool,
142
- "retrieved_at": self.retrieved_at,
143
- "sha256": self.sha256,
144
- "chars": len(self.text),
145
- "text": self.text,
146
- }
147
-
148
-
149
- def _now() -> str:
150
- return datetime.now(UTC).isoformat(timespec="seconds")
151
-
152
-
153
- def arxiv_id_from_url(url: str) -> str | None:
154
- """The bare arXiv id in `url`, version stripped, or None.
155
-
156
- Version stripped because a version is not a different paper for any
157
- purpose this pipeline has: the corpus keys on it, and OpenAlex's DOI
158
- for `2005.14165v3` is `10.48550/arXiv.2005.14165`.
159
- """
160
- try:
161
- parts = urlsplit(url)
162
- except ValueError:
163
- return None
164
- if parts.hostname is None or parts.hostname.lower() not in _ARXIV_HOSTS:
165
- return None
166
- match = _ARXIV_PATH_RE.match(parts.path)
167
- return match.group("id") if match else None
168
-
169
-
170
- def normalize_url(url: str) -> str:
171
- """The key two spellings of one source have to share.
172
-
173
- Case-folds the scheme and host, drops the fragment, drops a default
174
- port, drops a bare trailing slash, and collapses every arXiv `abs`/`pdf`
175
- spelling onto one canonical `abs` URL. The QUERY IS KEPT: `?id=7` is a
176
- different page on most sites, and dropping it would merge sources that
177
- say different things.
178
-
179
- An unparseable URL is returned stripped rather than raising. This runs
180
- on model-supplied strings; a malformed one should fail to match a
181
- document, not fail the run.
182
- """
183
- raw = (url or "").strip()
184
- if not raw:
185
- return ""
186
-
187
- arxiv_id = arxiv_id_from_url(raw)
188
- if arxiv_id:
189
- return f"https://arxiv.org/abs/{arxiv_id}"
190
-
191
- try:
192
- parts = urlsplit(raw)
193
- except ValueError:
194
- return raw
195
-
196
- host = (parts.hostname or "").lower()
197
- if not host:
198
- return raw
199
- if parts.port and not (
200
- (parts.scheme == "http" and parts.port == 80)
201
- or (parts.scheme == "https" and parts.port == 443)
202
- ):
203
- host = f"{host}:{parts.port}"
204
-
205
- path = parts.path
206
- if path == "/":
207
- path = ""
208
-
209
- return urlunsplit((parts.scheme.lower(), host, path, parts.query, ""))
210
-
211
-
212
- @dataclass
213
- class SourceCorpus:
214
- """Every source text one run retrieved, keyed by normalized URL.
215
-
216
- Run-scoped and passed by reference, the way `run.granted_calls` is
217
- (§25): one object means the corpus is already complete at the moment
218
- the scoring stage reads it, rather than being assembled at the end by a
219
- run that reached the end.
220
-
221
- THE LONGEST TEXT WINS. One URL can arrive three times -- as a 300-char
222
- web_search snippet, as a 600-char arXiv abstract, and as 5000 chars
223
- from fetch_url -- and the question every consumer asks is "what is the
224
- most this run knows about this page". Last-write-wins would make the
225
- answer depend on the order the model happened to call its tools in.
226
- """
227
-
228
- documents: dict = field(default_factory=dict)
229
- dropped: int = 0
230
-
231
- # -- ingest ----------------------------------------------------------
232
-
233
- def add(self, tool_name: str, result) -> int:
234
- """Absorb one tool result. Returns how many documents it added or
235
- improved.
236
-
237
- Anything unrecognised is ignored silently: this is attached to
238
- EVERY tool result a research pass produces, and a `todo_write`
239
- return value is not a defect worth a log line. A recognised tool
240
- whose result is the error shape is also ignored -- `arxiv.py` and
241
- `web_search.py` both return errors rather than raising, so an
242
- error dict is an ordinary occurrence here.
243
- """
244
- if not isinstance(result, dict) or "error" in result:
245
- return 0
246
- if tool_name == "fetch_url":
247
- return self._add_fetch_url(result)
248
- if tool_name == "arxiv_search":
249
- return self._add_arxiv(result)
250
- if tool_name == "web_search":
251
- return self._add_web_search(result)
252
- return 0
253
-
254
- def _add_fetch_url(self, result: dict) -> int:
255
- # `url` is the URL that ANSWERED, after redirects -- which is what
256
- # fetch_url.py:89-95 says the grounding pass attributes the text
257
- # to, and therefore what the model will cite.
258
- #
259
- # `offset` is ROADMAP_v3 §51 (PG7). A result without one is a
260
- # pre-§51 shape or a hand-built fixture, and 0 is the right reading
261
- # of both: a whole document starting at the beginning.
262
- offset = result.get("offset")
263
- if not isinstance(offset, int) or offset < 0:
264
- offset = 0
265
- return self._store(
266
- url=result.get("url"),
267
- text=result.get("content"),
268
- title="",
269
- tool="fetch_url",
270
- offset=offset,
271
- )
272
-
273
- def _add_arxiv(self, result: dict) -> int:
274
- added = 0
275
- for entry in result.get("results") or ():
276
- if not isinstance(entry, dict):
277
- continue
278
- arxiv_id = (entry.get("arxiv_id") or "").strip()
279
- if not arxiv_id:
280
- continue
281
- title = (entry.get("title") or "").strip()
282
- summary = (entry.get("summary") or "").strip()
283
- # Title AND abstract, joined. The title carries the paper's
284
- # subject in the compressed form a claim is usually phrased in,
285
- # and a 600-char abstract that opens on background prose can
286
- # otherwise match a claim about the paper's own topic poorly.
287
- text = "\n\n".join(p for p in (title, summary) if p)
288
- added += self._store(
289
- url=f"https://arxiv.org/abs/{arxiv_id}",
290
- text=text,
291
- title=title,
292
- tool="arxiv_search",
293
- )
294
- return added
295
-
296
- def _add_web_search(self, result: dict) -> int:
297
- added = 0
298
- for entry in result.get("results") or ():
299
- if not isinstance(entry, dict):
300
- continue
301
- title = (entry.get("title") or "").strip()
302
- snippet = (entry.get("snippet") or "").strip()
303
- text = "\n\n".join(p for p in (title, snippet) if p)
304
- added += self._store(
305
- url=entry.get("url"),
306
- text=text,
307
- title=title,
308
- tool="web_search",
309
- )
310
- return added
311
-
312
- def _store(self, url, text, title: str, tool: str, offset=None) -> int:
313
- """Absorb one document, or one PAGE of one (ROADMAP_v3 §51, PG7).
314
-
315
- `offset` is the source position this text starts at, or None from a
316
- tool that does not page. None and 0 are NOT the same thing here:
317
- None means "this is the whole of what that tool returns", 0 means
318
- "this is the first page of something that may have more".
319
-
320
- WHY THE PAGED CASE CANNOT USE LONGEST-WINS. Measured, batch 105,
321
- before any of this was written: a corpus given page 1 and then page
322
- 2 of one URL returned 1 then **0** -- `len(existing.text) >=
323
- len(cleaned)` is 5000 >= 5000, so page 2 was DROPPED and the held
324
- text stayed page 1. Paging would then have made the model better
325
- informed and the grounding pass no better, and the divergence is
326
- invisible from both sides: the model reads four pages and the score
327
- is computed against the first. The near miss is worse than the
328
- miss -- a page 2 one character longer REPLACED page 1 outright,
329
- leaving the corpus holding the middle of a document and attributing
330
- it to the whole URL.
331
- """
332
- if not isinstance(url, str) or not isinstance(text, str):
333
- return 0
334
- key = normalize_url(url)
335
- if not key:
336
- return 0
337
-
338
- existing = self.documents.get(key)
339
-
340
- # A continuation extends what is held; anything else competes on
341
- # length exactly as it did before §51.
342
- continuing = (
343
- offset is not None
344
- and offset > 0
345
- and existing is not None
346
- and existing.tool == tool
347
- )
348
-
349
- if continuing:
350
- # How much of this page is already held. Covers all three
351
- # shapes with one subtraction: an exact continuation overlaps
352
- # by nothing, a re-read of the same page overlaps entirely and
353
- # contributes the empty string, and a page that starts inside
354
- # what is held contributes only its tail. A page starting
355
- # BEYOND the held text leaves a gap -- appended anyway, because
356
- # the text is real and the alternative is losing it, and an
357
- # incomplete document is what this corpus already holds for
358
- # every source the byte cap cut.
359
- #
360
- # There is deliberately NO `if overlap >= len(text): return 0`
361
- # above this. One was written, and a mutation pass showed it
362
- # could be deleted with no test failing anywhere -- because the
363
- # slice already yields "" in exactly the cases it guarded, and
364
- # the empty-addition check below catches them. Batch 103's
365
- # lesson applies to a guard that cannot fire as much as to one
366
- # that fires in the wrong place: it reads as the thing
367
- # protecting you.
368
- overlap = existing.next_offset - offset
369
- addition = redact_output_text(text[max(0, overlap):])
370
- if not addition.strip():
371
- return 0
372
- merged = (existing.text + addition)[:MAX_DOCUMENT_CHARS]
373
- if merged == existing.text:
374
- return 0
375
- self.documents[key] = SourceDocument(
376
- url=key, text=merged, title=existing.title,
377
- tool=tool, retrieved_at=_now(),
378
- next_offset=max(existing.next_offset, offset + len(text)),
379
- )
380
- return 1
381
-
382
- cleaned = redact_output_text(text).strip()
383
- if len(cleaned) < MIN_DOCUMENT_CHARS:
384
- return 0
385
- cleaned = cleaned[:MAX_DOCUMENT_CHARS]
386
-
387
- if existing is not None and len(existing.text) >= len(cleaned):
388
- return 0
389
- if existing is None and len(self.documents) >= MAX_DOCUMENTS:
390
- # Counted rather than logged per drop: a run that hits this is
391
- # hitting it repeatedly, and the count is what the trace line
392
- # at the end of the stage reports.
393
- self.dropped += 1
394
- return 0
395
-
396
- self.documents[key] = SourceDocument(
397
- url=key, text=cleaned, title=title or (existing.title if existing else ""),
398
- tool=tool, retrieved_at=_now(),
399
- # In the SOURCE's positions, so the next page's offset can be
400
- # compared against it. `len(text)` and not `len(cleaned)`:
401
- # redaction and stripping change how much is HELD, never where
402
- # the page sat in the document it came from.
403
- next_offset=(offset + len(text)) if offset is not None else 0,
404
- )
405
- return 1
406
-
407
- # -- read ------------------------------------------------------------
408
-
409
- def get(self, url: str):
410
- """The document held for `url`, or None. Normalizes first, so a
411
- caller passing the model's spelling gets the fetched page."""
412
- return self.documents.get(normalize_url(url))
413
-
414
- def text_for(self, url: str) -> str:
415
- document = self.get(url)
416
- return document.text if document is not None else ""
417
-
418
- def __len__(self) -> int:
419
- return len(self.documents)
420
-
421
- def __bool__(self) -> bool:
422
- # Explicit, because __len__ alone would make an empty corpus
423
- # falsy and every `if corpus:` guard in the orchestrator would
424
- # then mean "if anything was fetched" rather than "if a corpus is
425
- # being collected at all". Those are different questions and the
426
- # first one is never the one being asked.
427
- return True
428
-
429
- def artifact_entries(self) -> list[dict]:
430
- """Every document, newest spelling of each URL, sorted for a
431
- stable diff between runs."""
432
- return [self.documents[key].as_artifact()
433
- for key in sorted(self.documents)]
1
+ """
2
+ core/reasoning/source_corpus.py
3
+
4
+ What a claim was actually grounded IN, kept for the length of one run.
5
+
6
+ ROADMAP_v2 §45 (SQ2). Pass 3a's `sources` list has always been URLs and two
7
+ model-invented floats. The text those URLs served reached the model, was
8
+ stringified into the pass thread, and was then unreachable: nothing in
9
+ `03_grounding.json` could say what a page had said, so `similarity_score`
10
+ could not be checked against anything and a quote could not be checked at
11
+ all. `fetch_url.py`'s own comment already describes the artifact this
12
+ builds ("output_writer builds each run's sources/ directory from it") --
13
+ that directory has existed and been empty since §12.
14
+
15
+ WHY A SECOND SINK RATHER THAN A CHANGE TO _translate. The orchestrator
16
+ already sees every tool result and deliberately forwards only `ok` plus an
17
+ error string to the UI, because a successful result body is not
18
+ observability. That boundary is unchanged. This is a different consumer
19
+ with a different need, attached beside it -- the corpus is read by the
20
+ scoring stage and written to the run directory, and neither is a
21
+ transcript.
22
+
23
+ REDACTED ON ENTRY, THOUGH IT ARRIVES REDACTED. `registry.dispatch` runs
24
+ every result through `check_output_policy` before the loop emits it, so
25
+ what lands here has already been through `redact_output_text`. It is run
26
+ again anyway, and for `_translate`'s stated reason: one rule owned at one
27
+ boundary -- nothing is stored by this module unredacted -- beats depending
28
+ on a guarantee made two layers away that a later refactor could quietly
29
+ drop. The second pass is a no-op on already-clean text.
30
+
31
+ AND IT IS NOT WHERE §52's SPLIT-CREDENTIAL FIX WENT, deliberately (RB4).
32
+ §51 made this module the one place where two pages of a document become
33
+ adjacent, so a credential straddling a page boundary was rejoined HERE --
34
+ reproduced end to end in batch 106, into `artifact_entries()` and therefore
35
+ into `sources/<sha256>.txt` and the window sent to the embedding provider.
36
+ The register's proposed fix was to re-redact the whole merged text on every
37
+ append. It was declined: `fetch_url` redacts before it cuts now, so the
38
+ halves never exist, and re-scanning up to 20,000 characters per page would
39
+ buy nothing this module can see -- while leaving unfixed the model's own two
40
+ tool results, which the producer-side fix does close.
41
+
42
+ THIS MODULE OWNS URL IDENTITY for the pipeline. Two spellings of one arXiv
43
+ paper are one source, and a model that cites `/abs/2005.14165v3` after
44
+ fetching `/abs/2005.14165` must not be told its own citation has no text.
45
+ `scholar.py` resolves the same URLs and imports `arxiv_id_from_url` from
46
+ here rather than re-deriving it -- a second copy of "what counts as the
47
+ same paper" is how the corpus and the scholarly lookup come to disagree
48
+ about one source in one artifact.
49
+ """
50
+
51
+ from __future__ import annotations
52
+
53
+ import hashlib
54
+ import logging
55
+ import re
56
+ from dataclasses import dataclass, field
57
+ from datetime import UTC, datetime
58
+ from urllib.parse import urlsplit, urlunsplit
59
+
60
+ from safety.policy_enforcement import redact_output_text
61
+
62
+ logger = logging.getLogger(__name__)
63
+
64
+ # Bounds, not tuning. A research pass calls fetch_url a bounded number of
65
+ # times (§31 budgets), so these are a backstop against a pathological run
66
+ # rather than a knob anyone is expected to turn: the corpus is held in
67
+ # memory for the length of a run and then written to disk.
68
+ #
69
+ # MAX_DOCUMENT_CHARS is deliberately well ABOVE fetch_url's own
70
+ # MAX_CONTENT_CHARS (5000). It bounds this module against a future tool
71
+ # that returns more; it is not a second truncation of the tools that exist,
72
+ # because a cap that bites in normal operation would silently shorten the
73
+ # text a similarity score is computed from.
74
+ MAX_DOCUMENTS = 200
75
+ MAX_DOCUMENT_CHARS = 20_000
76
+
77
+ # Below this, a "document" is a title and a fragment -- a web_search
78
+ # snippet for a result nobody fetched. Kept, because a 300-char snippet is
79
+ # still the only text some cited URLs ever have, and scoring a claim
80
+ # against it is better than reporting no evidence at all. The constant
81
+ # exists to reject the empty and near-empty case, not the short one.
82
+ MIN_DOCUMENT_CHARS = 16
83
+
84
+ _ARXIV_HOSTS = {"arxiv.org", "www.arxiv.org", "export.arxiv.org"}
85
+ # `2005.14165`, `2005.14165v3`, and the pre-2007 `math.GT/0309136` form --
86
+ # whose archive prefix contains a SLASH, which is why the id group cannot
87
+ # simply exclude one. A path with any further segment is not an arXiv id.
88
+ _ARXIV_PATH_RE = re.compile(
89
+ r"^/(?:abs|pdf)/"
90
+ r"(?P<id>(?:[a-z-]+(?:\.[A-Za-z]{2})?/)?[^/?#]+?)"
91
+ r"(?:v\d+)?(?:\.pdf)?$",
92
+ re.IGNORECASE,
93
+ )
94
+
95
+
96
+ @dataclass
97
+ class SourceDocument:
98
+ """One URL's best available text, and where it came from.
99
+
100
+ `tool` and `retrieved_at` are provenance rather than decoration: a
101
+ similarity score computed from a 300-char search snippet and one
102
+ computed from a fetched page are not the same measurement, and a
103
+ reader of the artifact has to be able to tell which they are looking
104
+ at.
105
+ """
106
+
107
+ url: str
108
+ text: str
109
+ title: str = ""
110
+ tool: str = ""
111
+ retrieved_at: str = ""
112
+
113
+ #: ROADMAP_v3 §51 (PG7). How far into the SOURCE document the held text
114
+ #: reaches, in the source's own character positions -- which is what
115
+ #: `fetch_url` pages by, and not the same as `len(text)`. 0 for a
116
+ #: document no paging tool produced.
117
+ #:
118
+ #: §52 (RB1) removed ONE of the two reasons those numbers differ.
119
+ #: Redaction used to happen here, on each page, so a page's held length
120
+ #: could be shorter than the span it covered; `fetch_url` now redacts
121
+ #: before it cuts, so its pages arrive clean and that gap is gone. What
122
+ #: still separates the two is `.strip()` and `MAX_DOCUMENT_CHARS`, both
123
+ #: of which change how much is HELD and neither of which moves where a
124
+ #: page sat in the document it came from.
125
+ #:
126
+ #: It is what stops a page being absorbed twice. Without it, a model
127
+ #: that re-reads page 2 -- which it will, because re-reading is how a
128
+ #: model checks itself -- would append it again and the corpus would
129
+ #: hold the same paragraph twice, which a similarity score then counts
130
+ #: twice.
131
+ next_offset: int = 0
132
+
133
+ @property
134
+ def sha256(self) -> str:
135
+ return hashlib.sha256(self.text.encode("utf-8")).hexdigest()
136
+
137
+ def as_artifact(self) -> dict:
138
+ return {
139
+ "url": self.url,
140
+ "title": self.title,
141
+ "tool": self.tool,
142
+ "retrieved_at": self.retrieved_at,
143
+ "sha256": self.sha256,
144
+ "chars": len(self.text),
145
+ "text": self.text,
146
+ }
147
+
148
+
149
+ def _now() -> str:
150
+ return datetime.now(UTC).isoformat(timespec="seconds")
151
+
152
+
153
+ def arxiv_id_from_url(url: str) -> str | None:
154
+ """The bare arXiv id in `url`, version stripped, or None.
155
+
156
+ Version stripped because a version is not a different paper for any
157
+ purpose this pipeline has: the corpus keys on it, and OpenAlex's DOI
158
+ for `2005.14165v3` is `10.48550/arXiv.2005.14165`.
159
+ """
160
+ try:
161
+ parts = urlsplit(url)
162
+ except ValueError:
163
+ return None
164
+ if parts.hostname is None or parts.hostname.lower() not in _ARXIV_HOSTS:
165
+ return None
166
+ match = _ARXIV_PATH_RE.match(parts.path)
167
+ return match.group("id") if match else None
168
+
169
+
170
+ def normalize_url(url: str) -> str:
171
+ """The key two spellings of one source have to share.
172
+
173
+ Case-folds the scheme and host, drops the fragment, drops a default
174
+ port, drops a bare trailing slash, and collapses every arXiv `abs`/`pdf`
175
+ spelling onto one canonical `abs` URL. The QUERY IS KEPT: `?id=7` is a
176
+ different page on most sites, and dropping it would merge sources that
177
+ say different things.
178
+
179
+ An unparseable URL is returned stripped rather than raising. This runs
180
+ on model-supplied strings; a malformed one should fail to match a
181
+ document, not fail the run.
182
+ """
183
+ raw = (url or "").strip()
184
+ if not raw:
185
+ return ""
186
+
187
+ arxiv_id = arxiv_id_from_url(raw)
188
+ if arxiv_id:
189
+ return f"https://arxiv.org/abs/{arxiv_id}"
190
+
191
+ try:
192
+ parts = urlsplit(raw)
193
+ except ValueError:
194
+ return raw
195
+
196
+ host = (parts.hostname or "").lower()
197
+ if not host:
198
+ return raw
199
+ if parts.port and not (
200
+ (parts.scheme == "http" and parts.port == 80)
201
+ or (parts.scheme == "https" and parts.port == 443)
202
+ ):
203
+ host = f"{host}:{parts.port}"
204
+
205
+ path = parts.path
206
+ if path == "/":
207
+ path = ""
208
+
209
+ return urlunsplit((parts.scheme.lower(), host, path, parts.query, ""))
210
+
211
+
212
+ @dataclass
213
+ class SourceCorpus:
214
+ """Every source text one run retrieved, keyed by normalized URL.
215
+
216
+ Run-scoped and passed by reference, the way `run.granted_calls` is
217
+ (§25): one object means the corpus is already complete at the moment
218
+ the scoring stage reads it, rather than being assembled at the end by a
219
+ run that reached the end.
220
+
221
+ THE LONGEST TEXT WINS. One URL can arrive three times -- as a 300-char
222
+ web_search snippet, as a 600-char arXiv abstract, and as 5000 chars
223
+ from fetch_url -- and the question every consumer asks is "what is the
224
+ most this run knows about this page". Last-write-wins would make the
225
+ answer depend on the order the model happened to call its tools in.
226
+ """
227
+
228
+ documents: dict = field(default_factory=dict)
229
+ dropped: int = 0
230
+
231
+ # -- ingest ----------------------------------------------------------
232
+
233
+ def add(self, tool_name: str, result) -> int:
234
+ """Absorb one tool result. Returns how many documents it added or
235
+ improved.
236
+
237
+ Anything unrecognised is ignored silently: this is attached to
238
+ EVERY tool result a research pass produces, and a `todo_write`
239
+ return value is not a defect worth a log line. A recognised tool
240
+ whose result is the error shape is also ignored -- `arxiv.py` and
241
+ `web_search.py` both return errors rather than raising, so an
242
+ error dict is an ordinary occurrence here.
243
+ """
244
+ if not isinstance(result, dict) or "error" in result:
245
+ return 0
246
+ if tool_name == "fetch_url":
247
+ return self._add_fetch_url(result)
248
+ if tool_name == "arxiv_search":
249
+ return self._add_arxiv(result)
250
+ if tool_name == "web_search":
251
+ return self._add_web_search(result)
252
+ return 0
253
+
254
+ def _add_fetch_url(self, result: dict) -> int:
255
+ # `url` is the URL that ANSWERED, after redirects -- which is what
256
+ # fetch_url.py:89-95 says the grounding pass attributes the text
257
+ # to, and therefore what the model will cite.
258
+ #
259
+ # `offset` is ROADMAP_v3 §51 (PG7). A result without one is a
260
+ # pre-§51 shape or a hand-built fixture, and 0 is the right reading
261
+ # of both: a whole document starting at the beginning.
262
+ offset = result.get("offset")
263
+ if not isinstance(offset, int) or offset < 0:
264
+ offset = 0
265
+ return self._store(
266
+ url=result.get("url"),
267
+ text=result.get("content"),
268
+ title="",
269
+ tool="fetch_url",
270
+ offset=offset,
271
+ )
272
+
273
+ def _add_arxiv(self, result: dict) -> int:
274
+ added = 0
275
+ for entry in result.get("results") or ():
276
+ if not isinstance(entry, dict):
277
+ continue
278
+ arxiv_id = (entry.get("arxiv_id") or "").strip()
279
+ if not arxiv_id:
280
+ continue
281
+ title = (entry.get("title") or "").strip()
282
+ summary = (entry.get("summary") or "").strip()
283
+ # Title AND abstract, joined. The title carries the paper's
284
+ # subject in the compressed form a claim is usually phrased in,
285
+ # and a 600-char abstract that opens on background prose can
286
+ # otherwise match a claim about the paper's own topic poorly.
287
+ text = "\n\n".join(p for p in (title, summary) if p)
288
+ added += self._store(
289
+ url=f"https://arxiv.org/abs/{arxiv_id}",
290
+ text=text,
291
+ title=title,
292
+ tool="arxiv_search",
293
+ )
294
+ return added
295
+
296
+ def _add_web_search(self, result: dict) -> int:
297
+ added = 0
298
+ for entry in result.get("results") or ():
299
+ if not isinstance(entry, dict):
300
+ continue
301
+ title = (entry.get("title") or "").strip()
302
+ snippet = (entry.get("snippet") or "").strip()
303
+ text = "\n\n".join(p for p in (title, snippet) if p)
304
+ added += self._store(
305
+ url=entry.get("url"),
306
+ text=text,
307
+ title=title,
308
+ tool="web_search",
309
+ )
310
+ return added
311
+
312
+ def _store(self, url, text, title: str, tool: str, offset=None) -> int:
313
+ """Absorb one document, or one PAGE of one (ROADMAP_v3 §51, PG7).
314
+
315
+ `offset` is the source position this text starts at, or None from a
316
+ tool that does not page. None and 0 are NOT the same thing here:
317
+ None means "this is the whole of what that tool returns", 0 means
318
+ "this is the first page of something that may have more".
319
+
320
+ WHY THE PAGED CASE CANNOT USE LONGEST-WINS. Measured, batch 105,
321
+ before any of this was written: a corpus given page 1 and then page
322
+ 2 of one URL returned 1 then **0** -- `len(existing.text) >=
323
+ len(cleaned)` is 5000 >= 5000, so page 2 was DROPPED and the held
324
+ text stayed page 1. Paging would then have made the model better
325
+ informed and the grounding pass no better, and the divergence is
326
+ invisible from both sides: the model reads four pages and the score
327
+ is computed against the first. The near miss is worse than the
328
+ miss -- a page 2 one character longer REPLACED page 1 outright,
329
+ leaving the corpus holding the middle of a document and attributing
330
+ it to the whole URL.
331
+ """
332
+ if not isinstance(url, str) or not isinstance(text, str):
333
+ return 0
334
+ key = normalize_url(url)
335
+ if not key:
336
+ return 0
337
+
338
+ existing = self.documents.get(key)
339
+
340
+ # A continuation extends what is held; anything else competes on
341
+ # length exactly as it did before §51.
342
+ continuing = (
343
+ offset is not None
344
+ and offset > 0
345
+ and existing is not None
346
+ and existing.tool == tool
347
+ )
348
+
349
+ if continuing:
350
+ # How much of this page is already held. Covers all three
351
+ # shapes with one subtraction: an exact continuation overlaps
352
+ # by nothing, a re-read of the same page overlaps entirely and
353
+ # contributes the empty string, and a page that starts inside
354
+ # what is held contributes only its tail. A page starting
355
+ # BEYOND the held text leaves a gap -- appended anyway, because
356
+ # the text is real and the alternative is losing it, and an
357
+ # incomplete document is what this corpus already holds for
358
+ # every source the byte cap cut.
359
+ #
360
+ # There is deliberately NO `if overlap >= len(text): return 0`
361
+ # above this. One was written, and a mutation pass showed it
362
+ # could be deleted with no test failing anywhere -- because the
363
+ # slice already yields "" in exactly the cases it guarded, and
364
+ # the empty-addition check below catches them. Batch 103's
365
+ # lesson applies to a guard that cannot fire as much as to one
366
+ # that fires in the wrong place: it reads as the thing
367
+ # protecting you.
368
+ overlap = existing.next_offset - offset
369
+ addition = redact_output_text(text[max(0, overlap):])
370
+ if not addition.strip():
371
+ return 0
372
+ merged = (existing.text + addition)[:MAX_DOCUMENT_CHARS]
373
+ if merged == existing.text:
374
+ return 0
375
+ self.documents[key] = SourceDocument(
376
+ url=key, text=merged, title=existing.title,
377
+ tool=tool, retrieved_at=_now(),
378
+ next_offset=max(existing.next_offset, offset + len(text)),
379
+ )
380
+ return 1
381
+
382
+ cleaned = redact_output_text(text).strip()
383
+ if len(cleaned) < MIN_DOCUMENT_CHARS:
384
+ return 0
385
+ cleaned = cleaned[:MAX_DOCUMENT_CHARS]
386
+
387
+ if existing is not None and len(existing.text) >= len(cleaned):
388
+ return 0
389
+ if existing is None and len(self.documents) >= MAX_DOCUMENTS:
390
+ # Counted rather than logged per drop: a run that hits this is
391
+ # hitting it repeatedly, and the count is what the trace line
392
+ # at the end of the stage reports.
393
+ self.dropped += 1
394
+ return 0
395
+
396
+ self.documents[key] = SourceDocument(
397
+ url=key, text=cleaned, title=title or (existing.title if existing else ""),
398
+ tool=tool, retrieved_at=_now(),
399
+ # In the SOURCE's positions, so the next page's offset can be
400
+ # compared against it. `len(text)` and not `len(cleaned)`:
401
+ # redaction and stripping change how much is HELD, never where
402
+ # the page sat in the document it came from.
403
+ next_offset=(offset + len(text)) if offset is not None else 0,
404
+ )
405
+ return 1
406
+
407
+ # -- read ------------------------------------------------------------
408
+
409
+ def get(self, url: str):
410
+ """The document held for `url`, or None. Normalizes first, so a
411
+ caller passing the model's spelling gets the fetched page."""
412
+ return self.documents.get(normalize_url(url))
413
+
414
+ def text_for(self, url: str) -> str:
415
+ document = self.get(url)
416
+ return document.text if document is not None else ""
417
+
418
+ def __len__(self) -> int:
419
+ return len(self.documents)
420
+
421
+ def __bool__(self) -> bool:
422
+ # Explicit, because __len__ alone would make an empty corpus
423
+ # falsy and every `if corpus:` guard in the orchestrator would
424
+ # then mean "if anything was fetched" rather than "if a corpus is
425
+ # being collected at all". Those are different questions and the
426
+ # first one is never the one being asked.
427
+ return True
428
+
429
+ def artifact_entries(self) -> list[dict]:
430
+ """Every document, newest spelling of each URL, sorted for a
431
+ stable diff between runs."""
432
+ return [self.documents[key].as_artifact()
433
+ for key in sorted(self.documents)]