@yahyahammad/venastine 0.3.8 → 0.3.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/.env.example +19 -19
  2. package/CONFIG_ARCHITECTURE.md +2139 -2002
  3. package/LICENSE +201 -201
  4. package/README.md +1043 -1034
  5. package/agents/builtin/compactor.md +75 -75
  6. package/agents/builtin/explore.md +77 -77
  7. package/agents/builtin/pipeline-reviewer.md +100 -100
  8. package/agents/builtin/plan.md +106 -106
  9. package/agents/builtin/review.md +74 -74
  10. package/agents/manager.py +261 -261
  11. package/agents/subagent_tool.py +472 -472
  12. package/agents/tui_commands.py +238 -146
  13. package/bin/venastine.mjs +584 -528
  14. package/bin/venastine.py +73 -71
  15. package/cli_commands.py +848 -835
  16. package/config.yaml +64 -0
  17. package/config_edit.py +1301 -1301
  18. package/config_schema.py +987 -966
  19. package/config_update.py +749 -749
  20. package/core/agent_activity.py +256 -256
  21. package/core/ansi.py +452 -452
  22. package/core/approval.py +137 -137
  23. package/core/call_view.py +364 -364
  24. package/core/cancel.py +121 -121
  25. package/core/chatgpt_auth.py +747 -747
  26. package/core/claims_view.py +107 -107
  27. package/core/client.py +2756 -2715
  28. package/core/commands.py +464 -454
  29. package/core/compaction.py +1735 -1735
  30. package/core/config_loader.py +1982 -1982
  31. package/core/connect.py +686 -686
  32. package/core/events.py +123 -123
  33. package/core/file_mentions.py +280 -280
  34. package/core/interaction.py +411 -411
  35. package/core/loop.py +2566 -2566
  36. package/core/memory.py +482 -482
  37. package/core/pipeline_models.py +190 -190
  38. package/core/provider_errors.py +274 -234
  39. package/core/reasoning/authorization.py +130 -130
  40. package/core/reasoning/base.py +219 -219
  41. package/core/reasoning/confidence_scoring.py +484 -484
  42. package/core/reasoning/events.py +149 -149
  43. package/core/reasoning/orchestrator.py +1561 -1561
  44. package/core/reasoning/output_writer.py +345 -345
  45. package/core/reasoning/payload_validation.py +524 -524
  46. package/core/reasoning/pipeline_storage.py +427 -427
  47. package/core/reasoning/scholar.py +467 -458
  48. package/core/reasoning/source_corpus.py +433 -433
  49. package/core/reasoning/source_scoring.py +855 -855
  50. package/core/replay.py +315 -315
  51. package/core/shell_sessions.py +1553 -1553
  52. package/credentials.py +307 -307
  53. package/database.py +333 -333
  54. package/docs/PRIVACY.md +78 -78
  55. package/docs/SECURITY.md +114 -108
  56. package/docs/THIRD_PARTY_NOTICES.md +148 -148
  57. package/env_secrets.py +118 -118
  58. package/json_store.py +343 -343
  59. package/logging_setup.py +295 -295
  60. package/main.py +3433 -3433
  61. package/mcp_client/client.py +780 -733
  62. package/mcp_client/config.py +484 -484
  63. package/mcp_client/registration.py +343 -343
  64. package/memories/manager.py +147 -147
  65. package/memories/tui_commands.py +319 -239
  66. package/package.json +303 -303
  67. package/project_init/config_files.py +282 -282
  68. package/project_init/generator.py +728 -728
  69. package/project_init/manifest.py +418 -418
  70. package/project_init/tui_commands.py +147 -147
  71. package/prompts/critic_pass.md +24 -24
  72. package/prompts/revalidate.md +25 -25
  73. package/prompts/revise.md +17 -17
  74. package/prompts/source_grounding.md +66 -66
  75. package/prompts/system_prompts.py +316 -316
  76. package/providers.json.example +94 -94
  77. package/pyproject.toml +149 -149
  78. package/requirements.txt +314 -295
  79. package/safety/policy_enforcement.py +1082 -1082
  80. package/security/askpass.py +419 -419
  81. package/security/capability.py +207 -207
  82. package/security/confine.py +484 -484
  83. package/security/hosted.py +182 -182
  84. package/security/permissions.py +237 -237
  85. package/security/protected_paths.py +501 -501
  86. package/security/sandbox.py +4108 -4108
  87. package/skills/builtin/math/formal-specification.md +67 -67
  88. package/skills/builtin/math/numerical-methods.md +66 -66
  89. package/skills/builtin/research/experiment-design.md +62 -62
  90. package/skills/builtin/research/reproducibility.md +55 -55
  91. package/skills/builtin/research/source-evaluation.md +70 -70
  92. package/skills/builtin/research/statistical-inference.md +63 -63
  93. package/skills/builtin/research/technical-writing.md +59 -59
  94. package/skills/builtin/software/code-review.md +61 -61
  95. package/skills/builtin/software/debugging.md +61 -61
  96. package/skills/builtin/software/test-design.md +67 -67
  97. package/skills/tui_commands.py +110 -110
  98. package/storage.py +1338 -1338
  99. package/tools/base.py +465 -465
  100. package/tools/builtin/_math_common.py +667 -667
  101. package/tools/builtin/_net_common.py +1145 -267
  102. package/tools/builtin/_page_text.py +490 -0
  103. package/tools/builtin/_pdf_text.py +113 -0
  104. package/tools/builtin/arxiv.py +685 -681
  105. package/tools/builtin/fetch_url.py +639 -371
  106. package/tools/builtin/file_ops.py +720 -720
  107. package/tools/builtin/geometry.py +184 -184
  108. package/tools/builtin/load_skill.py +76 -76
  109. package/tools/builtin/pin.py +227 -227
  110. package/tools/builtin/probability_stats.py +224 -224
  111. package/tools/builtin/project_docs.py +464 -464
  112. package/tools/builtin/remember.py +175 -175
  113. package/tools/builtin/shell.py +980 -980
  114. package/tools/builtin/todo.py +149 -149
  115. package/tools/builtin/web_search.py +402 -328
  116. package/tools/context.py +159 -159
  117. package/tools/isolation.py +387 -387
  118. package/tools/registry.py +1252 -1242
  119. package/tui/app.py +8372 -8307
  120. package/tui/app.tcss +754 -739
  121. package/tui/commands.py +26 -26
  122. package/tui/diffs.py +286 -286
  123. package/tui/history.py +128 -128
  124. package/tui/hosted_driver.py +102 -102
  125. package/tui/markdown.py +1329 -1329
  126. package/tui/meters.py +364 -364
  127. package/tui/ravens.py +146 -146
  128. package/tui/screens.py +1255 -1197
  129. package/tui/themes.py +599 -599
  130. package/tui/widgets.py +3861 -3796
@@ -0,0 +1,490 @@
1
+ """
2
+ tools/builtin/_page_text.py
3
+
4
+ What `fetch_url` makes of a web page before the model reads it (batch 240).
5
+
6
+ WHY THIS EXISTS. `fetch_url` used to hand the model the decoded body as it
7
+ arrived. Measured on seven live pages, the first 5,000-character page was
8
+ 80-96% markup -- `<head>`, scripts, styles, navigation -- and the text a
9
+ person reads was 1-25% of the HTML. On Wikipedia the article's first
10
+ sentence sat at HTML character 73,396, past the 64 KB read cap, so no part
11
+ of any Wikipedia article was ever retrievable; GitHub's README began on the
12
+ fifth page. Every one of those characters was context the model paid for
13
+ and could not use.
14
+
15
+ MAIN CONTENT, STDLIB, NO NEW DEPENDENCY (the owner's choice). Python's
16
+ `html.parser`, and rules this project can read and test:
17
+
18
+ * THE REGION. The page's `<main>` or `role="main"` element, else its
19
+ first `<article>`, else `<body>`, else everything. That one choice is
20
+ most of the win: it is where Wikipedia's article, MDN's reference text,
21
+ Python's documentation and GitHub's README live, and it leaves the
22
+ site's banner, menus and footer outside.
23
+ * WHAT IS DROPPED INSIDE IT. Elements that carry no prose (`script`,
24
+ `style`, `svg`, forms, embedded frames), page furniture by tag (`nav`,
25
+ `aside`, `footer`, and `header` when there is no main region to hold an
26
+ article's own), by ARIA role (navigation, banner, complementary,
27
+ contentinfo, search), by being hidden (`hidden`, `aria-hidden`,
28
+ `display:none`), and by a short list of class and id names that mean
29
+ furniture on the web as it is (`navbox`, `breadcrumb`, `sr-only`, ...).
30
+ * WHAT IS KEPT AS STRUCTURE. Headings as `#` lines, list items as `-` or
31
+ `1.`, table rows as cells joined by ` | `, preformatted text verbatim,
32
+ paragraphs separated by a blank line.
33
+
34
+ LINKS: TEXT INLINE, URLS LISTED APART (the owner's choice). A link keeps its
35
+ text in the prose and gains a short marker, `[L<n>]`; `fetch_url` lists the
36
+ URL for each marker with the page of text it appears on, so a citation can
37
+ be followed without every sentence carrying a URL. Distinct URLs are
38
+ numbered once per document, up to MAX_LINKS.
39
+
40
+ TOTAL OVER MALFORMED HTML. Real pages leave elements unclosed, so an end
41
+ tag closes whatever it finds open above it, and an element that never
42
+ closes is closed by its parent's end. A dropped element that was never
43
+ closed therefore cannot swallow the rest of the document.
44
+
45
+ PURE. No network, no config, no logging: `fetch_url` decides what to fetch
46
+ and what a link may be; this module only turns one string into another.
47
+ """
48
+
49
+ from __future__ import annotations
50
+
51
+ import hashlib
52
+ import re
53
+ from collections.abc import Callable
54
+ from dataclasses import dataclass, field
55
+ from html.parser import HTMLParser
56
+ from typing import Optional
57
+ from urllib.parse import urldefrag, urljoin
58
+
59
+ #: How many distinct links one document may number. Past it a link keeps
60
+ #: its text and gains no marker: the listing is a route to a citation, not
61
+ #: a sitemap, and a Wikipedia article carries hundreds.
62
+ MAX_LINKS = 200
63
+
64
+ #: A title past this is cut; the HTML spec puts no bound on `<title>`.
65
+ MAX_TITLE_CHARS = 300
66
+
67
+ #: How deep the open-element stack may grow. A page of unclosed `<p>`s
68
+ #: closed only by `</body>` would otherwise grow it without bound.
69
+ _MAX_DEPTH = 4000
70
+
71
+ #: Elements with no closing tag. Never pushed, so they never wait to close.
72
+ _VOID = frozenset({
73
+ "area", "base", "br", "col", "embed", "hr", "img", "input", "link",
74
+ "meta", "param", "source", "track", "wbr",
75
+ })
76
+
77
+ #: Elements whose whole subtree carries no prose a reader wants.
78
+ _DROP_TAGS = frozenset({
79
+ "script", "style", "noscript", "template", "svg", "math", "canvas",
80
+ "iframe", "object", "video", "audio", "picture", "map", "form",
81
+ "button", "select", "textarea", "label", "nav", "aside", "footer",
82
+ "dialog", "head", "title",
83
+ })
84
+
85
+ #: ARIA roles that are page furniture whatever element carries them.
86
+ _DROP_ROLES = frozenset({
87
+ "navigation", "banner", "complementary", "contentinfo", "search",
88
+ "dialog", "alertdialog", "menu", "menubar", "toolbar", "tablist",
89
+ })
90
+
91
+ #: Whole class or id names that mean furniture -- matched as written,
92
+ #: because the hyphenated ones only mean it as a whole (`sr-only`).
93
+ _DROP_NAMES = frozenset({
94
+ "sr-only", "visually-hidden", "visuallyhidden", "screen-reader-text",
95
+ "skip-link", "skiplink", "skip-to-content", "mw-editsection",
96
+ "mw-jump-link", "noprint", "navbox", "navbar", "breadcrumb",
97
+ "breadcrumbs", "sidebar", "toc", "cookie-banner", "cookie-notice",
98
+ # Sphinx's pilcrow beside every heading: a link to the heading itself.
99
+ "headerlink",
100
+ })
101
+
102
+ #: Words that mean furniture wherever they sit in a class or id: the value
103
+ #: is split on anything that is not a letter or digit, so `site-navbox`
104
+ #: yields `navbox` while `menuetto` yields nothing. Kept short on purpose --
105
+ #: each entry is prose this could drop if it were wrong.
106
+ _DROP_WORDS = frozenset({
107
+ "navbox", "breadcrumb", "breadcrumbs", "noprint", "editsection",
108
+ "cookie", "cookies", "consent", "advert", "advertisement", "newsletter",
109
+ # Measured on Wikipedia (batch 240): its article tabs, page tools and
110
+ # 37-language list sit INSIDE <main>, in `vector-menu-content`,
111
+ # `vector-dropdown` and `interlanguage-link` containers, and the
112
+ # language list alone used every one of the MAX_LINKS numbers.
113
+ "menu", "dropdown", "interlanguage",
114
+ })
115
+
116
+ #: Elements that end a paragraph, and those that end a line.
117
+ _PARAGRAPH = frozenset({"p", "blockquote", "table", "figure", "section",
118
+ "article", "details"})
119
+ _LINE = frozenset({
120
+ "address", "dd", "div", "dl", "dt", "fieldset", "figcaption", "header",
121
+ "hgroup", "li", "main", "summary", "tbody", "thead", "tfoot", "tr",
122
+ "caption", "ul", "ol",
123
+ })
124
+
125
+ _HEADING = {f"h{n}": n for n in range(1, 7)}
126
+ _WORD_SPLIT = re.compile(r"[^a-z0-9]+")
127
+ _SPACES = re.compile(r"[ \t\f\v\r\n]+")
128
+ _MARKER = re.compile(r"\[L(\d+)\]")
129
+ _LIST_ITEM = re.compile(r"^( *)(- |\d+\. )")
130
+
131
+ # A `<pre>` block's text travels between these two so that tidying the
132
+ # prose around it cannot reflow it. Neither can occur in parsed HTML text:
133
+ # the parser hands control characters through, but these two are removed
134
+ # from every data chunk before they could be confused with the sentinels.
135
+ _PRE_OPEN, _PRE_CLOSE = "\x02", "\x03"
136
+
137
+
138
+ @dataclass
139
+ class Document:
140
+ """One fetched document as the model will read it."""
141
+
142
+ text: str
143
+ kind: str # "html", "pdf" or "text"
144
+ title: Optional[str] = None
145
+ links: dict = field(default_factory=dict) # marker number -> url
146
+ pages: Optional[int] = None # a PDF's page count
147
+ digest: str = "" # of the whole text, to tell versions apart
148
+
149
+ def __post_init__(self) -> None:
150
+ if not self.digest:
151
+ self.digest = text_digest(self.text)
152
+
153
+
154
+ def text_digest(text: str) -> str:
155
+ """Whether two fetches of one URL produced the same document."""
156
+ return hashlib.sha256(text.encode("utf-8", "surrogatepass")).hexdigest()
157
+
158
+
159
+ def markers_in(text: str) -> list:
160
+ """The link numbers a slice of text refers to, first appearance first."""
161
+ seen: list = []
162
+ for match in _MARKER.finditer(text):
163
+ number = int(match.group(1))
164
+ if number not in seen:
165
+ seen.append(number)
166
+ return seen
167
+
168
+
169
+ def split_marker_end(text: str, end: int) -> int:
170
+ """`end`, moved past an `[L<n>]` marker it would otherwise cut in two.
171
+
172
+ A page boundary that fell inside a two-digit marker would leave its
173
+ first digit on one page and its second on the next, and neither half
174
+ would name a link.
175
+ """
176
+ start = text.rfind("[L", max(0, end - 8), end)
177
+ if start == -1:
178
+ return end
179
+ close = text.find("]", start, start + 10)
180
+ if close != -1 and close >= end and _MARKER.fullmatch(text[start:close + 1]):
181
+ return close + 1
182
+ return end
183
+
184
+
185
+ def text_document(text: str) -> Document:
186
+ """A body that is already text: JSON, plain text, XML, source code."""
187
+ return Document(text=text, kind="text")
188
+
189
+
190
+ # ---- HTML -----------------------------------------------------------------
191
+
192
+ def _attrs(raw) -> dict:
193
+ return {name.lower(): (value or "") for name, value in raw}
194
+
195
+
196
+ def _is_furniture(tag: str, attrs: dict, drop_header: bool) -> bool:
197
+ if tag in _DROP_TAGS or (drop_header and tag == "header"):
198
+ return True
199
+ if "hidden" in attrs or attrs.get("aria-hidden", "").lower() == "true":
200
+ return True
201
+ if attrs.get("role", "").lower() in _DROP_ROLES:
202
+ return True
203
+ style = attrs.get("style", "").lower().replace(" ", "")
204
+ if "display:none" in style or "visibility:hidden" in style:
205
+ return True
206
+ for name in ("class", "id"):
207
+ value = attrs.get(name, "").lower()
208
+ if not value:
209
+ continue
210
+ if _DROP_NAMES.intersection(value.split()):
211
+ return True
212
+ if _DROP_WORDS.intersection(_WORD_SPLIT.split(value)):
213
+ return True
214
+ return False
215
+
216
+
217
+ class _Scan(HTMLParser):
218
+ """Pass one: where the main region starts, and the page's title."""
219
+
220
+ def __init__(self) -> None:
221
+ super().__init__(convert_charrefs=True)
222
+ self.index = -1
223
+ self.main: Optional[int] = None
224
+ self.article: Optional[int] = None
225
+ self.body: Optional[int] = None
226
+ self._in_title = False
227
+ self.title_parts: list = []
228
+
229
+ def handle_starttag(self, tag, attrs):
230
+ self.index += 1
231
+ tag = tag.lower()
232
+ if tag == "title" and not self.title_parts:
233
+ self._in_title = True
234
+ if self.main is None and (
235
+ tag == "main" or _attrs(attrs).get("role", "").lower() == "main"):
236
+ self.main = self.index
237
+ elif tag == "article" and self.article is None:
238
+ self.article = self.index
239
+ elif tag == "body" and self.body is None:
240
+ self.body = self.index
241
+
242
+ def handle_startendtag(self, tag, attrs):
243
+ self.index += 1
244
+
245
+ def handle_endtag(self, tag):
246
+ if tag.lower() == "title":
247
+ self._in_title = False
248
+
249
+ def handle_data(self, data):
250
+ if self._in_title:
251
+ self.title_parts.append(data)
252
+
253
+
254
+ class _Render(HTMLParser):
255
+ """Pass two: the region's prose, with structure and link markers."""
256
+
257
+ def __init__(self, region: Optional[int], drop_header: bool,
258
+ base_url: str,
259
+ keep_link: Callable[[str], Optional[str]]) -> None:
260
+ super().__init__(convert_charrefs=True)
261
+ self.region = region # start-tag index of the region, or None
262
+ self.drop_header = drop_header
263
+ self.base_url = base_url
264
+ self.page_url = urldefrag(base_url)[0]
265
+ self.keep_link = keep_link
266
+ self.index = -1
267
+ self.stack: list = [] # (tag, is furniture)
268
+ self.inside = region is None # the whole document when no region
269
+ self.region_depth: Optional[int] = None
270
+ self.dropped = 0 # how many furniture elements are open
271
+ self.pre = 0
272
+ self.out: list = []
273
+ self.lists: list = [] # per open list: None for ul, a count for ol
274
+ self.row_cells = 0
275
+ self.link: Optional[str] = None
276
+ self.link_text = False
277
+ self.links: dict = {} # url -> number
278
+ self.done = False
279
+
280
+ def _visible(self) -> bool:
281
+ return self.inside and not self.dropped and not self.done
282
+
283
+ def handle_starttag(self, tag, attrs):
284
+ self.index += 1
285
+ tag = tag.lower()
286
+ attrs = _attrs(attrs)
287
+ if tag in _VOID:
288
+ self._void(tag)
289
+ return
290
+ is_region = self.region is not None and self.index == self.region
291
+ if is_region:
292
+ self.inside = True
293
+ self.region_depth = len(self.stack)
294
+ furniture = (self.inside and not is_region
295
+ and _is_furniture(tag, attrs, self.drop_header))
296
+ if len(self.stack) >= _MAX_DEPTH:
297
+ # Pathological nesting: stop opening, keep reading the text.
298
+ return
299
+ self.stack.append((tag, furniture))
300
+ if furniture:
301
+ self.dropped += 1
302
+ elif self._visible():
303
+ self._open(tag, attrs)
304
+
305
+ def handle_startendtag(self, tag, attrs):
306
+ # `<br/>` and friends: a start with no end to wait for.
307
+ self.index += 1
308
+ self._void(tag.lower())
309
+
310
+ def _void(self, tag):
311
+ if not self._visible():
312
+ return
313
+ if tag == "br":
314
+ self.out.append("\n")
315
+ elif tag == "hr":
316
+ self.out.append("\n\n")
317
+
318
+ def handle_endtag(self, tag):
319
+ tag = tag.lower()
320
+ if tag in _VOID:
321
+ return
322
+ # An end tag closes the nearest open element of its name and every
323
+ # element opened above it; one with no open match is ignored.
324
+ for depth in range(len(self.stack) - 1, -1, -1):
325
+ if self.stack[depth][0] == tag:
326
+ break
327
+ else:
328
+ return
329
+ while len(self.stack) > depth:
330
+ name, furniture = self.stack.pop()
331
+ if furniture:
332
+ self.dropped -= 1
333
+ elif self._visible():
334
+ self._close(name)
335
+ if (self.region_depth is not None
336
+ and len(self.stack) == self.region_depth):
337
+ # The region itself just closed: nothing after it is read.
338
+ self.done = True
339
+
340
+ def handle_data(self, data):
341
+ if not self._visible():
342
+ return
343
+ data = data.replace(_PRE_OPEN, "").replace(_PRE_CLOSE, "")
344
+ self.out.append(data if self.pre else _SPACES.sub(" ", data))
345
+ if self.link is not None and data.strip():
346
+ self.link_text = True
347
+
348
+ def _open(self, tag, attrs):
349
+ if tag in _HEADING:
350
+ self.out.append("\n\n" + "#" * _HEADING[tag] + " ")
351
+ elif tag == "li":
352
+ if self.lists and self.lists[-1] is not None:
353
+ self.lists[-1] += 1
354
+ bullet = f"{self.lists[-1]}. "
355
+ else:
356
+ bullet = "- "
357
+ self.out.append("\n" + " " * max(len(self.lists) - 1, 0) + bullet)
358
+ elif tag in ("ul", "ol"):
359
+ # A list nested in an item continues that item's lines.
360
+ if not self.lists:
361
+ self.out.append("\n")
362
+ self.lists.append(0 if tag == "ol" else None)
363
+ elif tag == "tr":
364
+ self.row_cells = 0
365
+ self.out.append("\n")
366
+ elif tag in ("td", "th"):
367
+ if self.row_cells:
368
+ self.out.append(" | ")
369
+ self.row_cells += 1
370
+ elif tag == "pre":
371
+ self.pre += 1
372
+ if self.pre == 1:
373
+ self.out.append("\n\n" + _PRE_OPEN)
374
+ elif tag == "a":
375
+ href = attrs.get("href", "")
376
+ self.link = self._link(href) if href else None
377
+ self.link_text = False
378
+ elif tag in _PARAGRAPH:
379
+ self.out.append("\n\n")
380
+ elif tag in _LINE:
381
+ self.out.append("\n")
382
+
383
+ def _close(self, tag):
384
+ if tag in _HEADING:
385
+ self.out.append("\n\n")
386
+ elif tag in ("ul", "ol"):
387
+ if self.lists:
388
+ self.lists.pop()
389
+ if not self.lists:
390
+ self.out.append("\n")
391
+ elif tag == "pre":
392
+ self.pre = max(self.pre - 1, 0)
393
+ if self.pre == 0:
394
+ self.out.append(_PRE_CLOSE + "\n\n")
395
+ elif tag == "a":
396
+ if self.link is not None and self.link_text:
397
+ self.out.append(f"[L{self.links[self.link]}]")
398
+ self.link = None
399
+ elif tag in _PARAGRAPH:
400
+ self.out.append("\n\n")
401
+ elif tag in _LINE and tag not in ("li", "tr"):
402
+ # A list item and a table row START their line; ending it too
403
+ # would leave a blank line between every two of them.
404
+ self.out.append("\n")
405
+
406
+ def _link(self, href: str) -> Optional[str]:
407
+ """The link's key in `self.links` if it is numbered, else None."""
408
+ href = href.strip()
409
+ if href.startswith(("#", "javascript:", "mailto:", "tel:", "data:")):
410
+ return None
411
+ absolute = urldefrag(urljoin(self.base_url, href))[0]
412
+ if not absolute.startswith(("http://", "https://")):
413
+ return None
414
+ if absolute == self.page_url:
415
+ return None # a link to this same page
416
+ kept = self.keep_link(absolute)
417
+ if kept is None:
418
+ return None
419
+ if kept not in self.links:
420
+ if len(self.links) >= MAX_LINKS:
421
+ return None
422
+ self.links[kept] = len(self.links) + 1
423
+ return kept
424
+
425
+
426
+ def _prose(piece: str) -> str:
427
+ """One stretch of text between `<pre>` blocks, tidied."""
428
+ lines = []
429
+ for line in piece.split("\n"):
430
+ item = _LIST_ITEM.match(line)
431
+ lines.append(line.rstrip() if item else line.strip())
432
+ text = "\n".join(lines)
433
+ # A heading whose text was all furniture leaves its `#` behind, and one
434
+ # whose text began inside an inline element leaves extra spaces.
435
+ text = re.sub(r"(?m)^#+ *$", "", text)
436
+ text = re.sub(r"(?m)^(#+) {2,}", r"\1 ", text)
437
+ # A table row's empty leading cells (a logo, a spacer) leave their
438
+ # separators behind; so do runs of empty cells between real ones.
439
+ text = re.sub(r"(?m)^(?:\| *)+", "", text)
440
+ text = re.sub(r"(?: *\|){2,} *", " | ", text)
441
+ text = re.sub(r"\n[ \t]*\n(?:[ \t]*\n)+", "\n\n", text)
442
+ return text.strip()
443
+
444
+
445
+ def _tidy(raw: str) -> str:
446
+ """Prose tidied stretch by stretch; a `<pre>` block kept exactly, its
447
+ own blank lines included -- code's layout is its meaning."""
448
+ pieces = re.split(f"{_PRE_OPEN}(.*?){_PRE_CLOSE}", raw, flags=re.S)
449
+ parts: list = []
450
+ for number, piece in enumerate(pieces):
451
+ if number % 2:
452
+ block = "\n".join(line.rstrip() for line in piece.split("\n"))
453
+ block = block.strip("\n")
454
+ if block.strip():
455
+ parts.append(block)
456
+ else:
457
+ prose = _prose(piece)
458
+ if prose:
459
+ parts.append(prose)
460
+ return "\n\n".join(parts)
461
+
462
+
463
+ def html_document(html: str, base_url: str,
464
+ keep_link: Callable[[str], Optional[str]] = lambda url: url
465
+ ) -> Document:
466
+ """The readable text of an HTML page, its title and its numbered links.
467
+
468
+ `keep_link(url)` decides whether a link may be offered, and may return
469
+ a rewritten URL (a redacted one, say); None leaves the link as plain
470
+ text. `fetch_url` passes its URL policy and its length bound.
471
+ """
472
+ scan = _Scan()
473
+ scan.feed(html)
474
+ scan.close()
475
+ if scan.main is not None:
476
+ region, drop_header = scan.main, False
477
+ elif scan.article is not None:
478
+ region, drop_header = scan.article, False
479
+ else:
480
+ # No region of its own: a top-level <header> is the site's banner.
481
+ region, drop_header = scan.body, True
482
+ render = _Render(region, drop_header, base_url, keep_link)
483
+ render.feed(html)
484
+ render.close()
485
+ title = _SPACES.sub(" ", "".join(scan.title_parts)).strip() or None
486
+ if title is not None:
487
+ title = title[:MAX_TITLE_CHARS]
488
+ links = {number: url for url, number in render.links.items()}
489
+ return Document(text=_tidy("".join(render.out)), kind="html",
490
+ title=title, links=links)
@@ -0,0 +1,113 @@
1
+ """
2
+ tools/builtin/_pdf_text.py
3
+
4
+ A PDF's text, read in the killable child `tools/isolation.py` runs (batch
5
+ 240, the owner's choice of both the library and the process).
6
+
7
+ pypdf, NOT markitdown[pdf], AND THE CHOICE WAS MEASURED. Three arXiv papers
8
+ -- Attention (one column), BERT (two columns, ACL layout) and the 77-page
9
+ Llama 2 report -- each checked for whole running sentences:
10
+
11
+ * pypdf 6.21.4: every probe sentence intact on Attention and BERT; 0.8%
12
+ of words run together on Llama 2; 1.2 s, 0.5 s and 4.2 s. 4.1 MB, no
13
+ dependencies.
14
+ * markitdown 0.1.9 [pdf]: words run together on every paper (22% on
15
+ Llama 2), and BERT's two columns came out as a Markdown table of word
16
+ fragments. 188 MB of dependencies, onnxruntime and numpy among them.
17
+
18
+ IN A CHILD PROCESS, because PDF parsers have a long history of hostile files
19
+ that make them spin or exhaust memory, and a thread cannot be stopped (the
20
+ argument `tools/isolation.py` makes at length). The child is under
21
+ `tool_compute_timeout_s` and, on POSIX, the CPU and address-space rlimits.
22
+
23
+ pypdf IS IMPORTED AT MODULE TOP, on purpose. The child imports this module
24
+ BEFORE a shared hosted deployment confines it, and the import is what needs
25
+ the filesystem; importing inside the handler would run after the walls go
26
+ up. The parent imports this module only when a PDF arrives.
27
+
28
+ REDACTED HERE, BEFORE THE WINDOW IS CUT (§52's RB1). The child sees the
29
+ whole text exactly once, and a credential straddling the edge of the window
30
+ it returns would be invisible to a redactor run on the window afterwards.
31
+ Positions are therefore positions in redacted text, the one space
32
+ `fetch_url` speaks (RB3).
33
+
34
+ WHAT CROSSES BACK. Only the window asked for, never the whole text:
35
+ `tools/isolation` refuses a result over 4 MB, and a long report's text is
36
+ larger than the window `fetch_url` holds anyway.
37
+ """
38
+
39
+ from __future__ import annotations
40
+
41
+ import base64
42
+ import io
43
+ import re
44
+
45
+ from pypdf import PdfReader
46
+
47
+ from safety.policy_enforcement import redact_output_text
48
+ from tools.builtin._page_text import text_digest
49
+
50
+ #: Ligatures a PDF's fonts encode as one glyph. Expanded, so "finetune"
51
+ #: is "finetune" to anything that searches the text.
52
+ _LIGATURES = {"ff": "ff", "fi": "fi", "fl": "fl", "ffi": "ffi",
53
+ "ffl": "ffl", "ſt": "st", "st": "st"}
54
+
55
+ # A word broken across lines -- "representa-" / "tion" -- joined back up.
56
+ # Only before a lower-case letter: "Rad-" / "ford" is a name split by the
57
+ # column, while "BERT-" / "Base" is a real hyphen.
58
+ _LINE_HYPHEN = re.compile(r"(\w)-\n(?=[a-z])")
59
+ _SPACES = re.compile(r"[ \t]+")
60
+
61
+
62
+ def _clean(text: str) -> str:
63
+ for glyph, letters in _LIGATURES.items():
64
+ text = text.replace(glyph, letters)
65
+ text = _LINE_HYPHEN.sub(r"\1", text)
66
+ text = "\n".join(_SPACES.sub(" ", line).strip() for line in text.split("\n"))
67
+ return re.sub(r"\n{3,}", "\n\n", text).strip()
68
+
69
+
70
+ def pdf_window(params: dict) -> dict:
71
+ """The child's handler: `{"data": base64, "start": int, "limit": int}`.
72
+
73
+ Returns `{"text", "total", "digest", "pages", "title", "redacted"}` --
74
+ the window `[start, start + limit)` of the whole document's text, with
75
+ `[page N]` before each page so a claim can cite one -- or `{"error"}`.
76
+ """
77
+ try:
78
+ reader = PdfReader(io.BytesIO(base64.b64decode(params["data"])))
79
+ if reader.is_encrypted and not reader.decrypt(""):
80
+ return {"error": "this PDF is encrypted with a password."}
81
+ parts = []
82
+ for number, page in enumerate(reader.pages, start=1):
83
+ parts.append(f"[page {number}]\n{_clean(page.extract_text() or '')}")
84
+ except Exception as exc: # noqa: BLE001 -- a hostile file raises anything
85
+ return {"error": f"this PDF could not be read ({type(exc).__name__}: "
86
+ f"{str(exc)[:200]})."}
87
+ whole = "\n\n".join(parts)
88
+ clean = redact_output_text(whole)
89
+ start = max(int(params.get("start", 0)), 0)
90
+ limit = max(int(params.get("limit", 0)), 0)
91
+ title = None
92
+ try:
93
+ meta = reader.metadata
94
+ if meta is not None and isinstance(meta.title, str) and meta.title.strip():
95
+ title = meta.title.strip()[:300]
96
+ except Exception: # noqa: BLE001 -- metadata is optional and often broken
97
+ title = None
98
+ return {
99
+ "text": clean[start:start + limit],
100
+ "total": len(clean),
101
+ "digest": text_digest(clean),
102
+ "pages": len(parts),
103
+ "title": title,
104
+ "redacted": clean != whole,
105
+ # A scanned PDF has pages and no text layer: say so rather than
106
+ # return a document of page markers.
107
+ "no_text": not any(part.split("\n", 1)[-1].strip() for part in parts),
108
+ }
109
+
110
+
111
+ def encode(data: bytes) -> str:
112
+ """The PDF's bytes as the child's JSON request carries them."""
113
+ return base64.b64encode(data).decode("ascii")