@yahyahammad/venastine 0.3.9 → 0.3.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONFIG_ARCHITECTURE.md +109 -0
- package/README.md +13 -4
- package/bin/venastine.mjs +62 -6
- package/bin/venastine.py +3 -1
- package/cli_commands.py +10 -0
- package/config.yaml +51 -0
- package/config_schema.py +16 -0
- package/core/client.py +45 -4
- package/core/commands.py +4 -0
- package/core/provider_errors.py +40 -0
- package/core/reasoning/scholar.py +14 -5
- package/docs/SECURITY.md +6 -0
- package/mcp_client/client.py +49 -2
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/requirements.txt +19 -0
- package/tools/builtin/_net_common.py +878 -0
- package/tools/builtin/_page_text.py +490 -0
- package/tools/builtin/_pdf_text.py +113 -0
- package/tools/builtin/arxiv.py +5 -1
- package/tools/builtin/fetch_url.py +388 -120
- package/tools/builtin/web_search.py +81 -7
- package/tools/registry.py +10 -0
- package/tui/app.py +54 -0
- package/tui/app.tcss +7 -0
- package/tui/widgets.py +48 -0
|
@@ -0,0 +1,490 @@
|
|
|
1
|
+
"""
|
|
2
|
+
tools/builtin/_page_text.py
|
|
3
|
+
|
|
4
|
+
What `fetch_url` makes of a web page before the model reads it (batch 240).
|
|
5
|
+
|
|
6
|
+
WHY THIS EXISTS. `fetch_url` used to hand the model the decoded body as it
|
|
7
|
+
arrived. Measured on seven live pages, the first 5,000-character page was
|
|
8
|
+
80-96% markup -- `<head>`, scripts, styles, navigation -- and the text a
|
|
9
|
+
person reads was 1-25% of the HTML. On Wikipedia the article's first
|
|
10
|
+
sentence sat at HTML character 73,396, past the 64 KB read cap, so no part
|
|
11
|
+
of any Wikipedia article was ever retrievable; GitHub's README began on the
|
|
12
|
+
fifth page. Every one of those characters was context the model paid for
|
|
13
|
+
and could not use.
|
|
14
|
+
|
|
15
|
+
MAIN CONTENT, STDLIB, NO NEW DEPENDENCY (the owner's choice). Python's
|
|
16
|
+
`html.parser`, and rules this project can read and test:
|
|
17
|
+
|
|
18
|
+
* THE REGION. The page's `<main>` or `role="main"` element, else its
|
|
19
|
+
first `<article>`, else `<body>`, else everything. That one choice is
|
|
20
|
+
most of the win: it is where Wikipedia's article, MDN's reference text,
|
|
21
|
+
Python's documentation and GitHub's README live, and it leaves the
|
|
22
|
+
site's banner, menus and footer outside.
|
|
23
|
+
* WHAT IS DROPPED INSIDE IT. Elements that carry no prose (`script`,
|
|
24
|
+
`style`, `svg`, forms, embedded frames), page furniture by tag (`nav`,
|
|
25
|
+
`aside`, `footer`, and `header` when there is no main region to hold an
|
|
26
|
+
article's own), by ARIA role (navigation, banner, complementary,
|
|
27
|
+
contentinfo, search), by being hidden (`hidden`, `aria-hidden`,
|
|
28
|
+
`display:none`), and by a short list of class and id names that mean
|
|
29
|
+
furniture on the web as it is (`navbox`, `breadcrumb`, `sr-only`, ...).
|
|
30
|
+
* WHAT IS KEPT AS STRUCTURE. Headings as `#` lines, list items as `-` or
|
|
31
|
+
`1.`, table rows as cells joined by ` | `, preformatted text verbatim,
|
|
32
|
+
paragraphs separated by a blank line.
|
|
33
|
+
|
|
34
|
+
LINKS: TEXT INLINE, URLS LISTED APART (the owner's choice). A link keeps its
|
|
35
|
+
text in the prose and gains a short marker, `[L<n>]`; `fetch_url` lists the
|
|
36
|
+
URL for each marker with the page of text it appears on, so a citation can
|
|
37
|
+
be followed without every sentence carrying a URL. Distinct URLs are
|
|
38
|
+
numbered once per document, up to MAX_LINKS.
|
|
39
|
+
|
|
40
|
+
TOTAL OVER MALFORMED HTML. Real pages leave elements unclosed, so an end
|
|
41
|
+
tag closes whatever it finds open above it, and an element that never
|
|
42
|
+
closes is closed by its parent's end. A dropped element that was never
|
|
43
|
+
closed therefore cannot swallow the rest of the document.
|
|
44
|
+
|
|
45
|
+
PURE. No network, no config, no logging: `fetch_url` decides what to fetch
|
|
46
|
+
and what a link may be; this module only turns one string into another.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
from __future__ import annotations
|
|
50
|
+
|
|
51
|
+
import hashlib
|
|
52
|
+
import re
|
|
53
|
+
from collections.abc import Callable
|
|
54
|
+
from dataclasses import dataclass, field
|
|
55
|
+
from html.parser import HTMLParser
|
|
56
|
+
from typing import Optional
|
|
57
|
+
from urllib.parse import urldefrag, urljoin
|
|
58
|
+
|
|
59
|
+
#: How many distinct links one document may number. Past it a link keeps
|
|
60
|
+
#: its text and gains no marker: the listing is a route to a citation, not
|
|
61
|
+
#: a sitemap, and a Wikipedia article carries hundreds.
|
|
62
|
+
MAX_LINKS = 200
|
|
63
|
+
|
|
64
|
+
#: A title past this is cut; the HTML spec puts no bound on `<title>`.
|
|
65
|
+
MAX_TITLE_CHARS = 300
|
|
66
|
+
|
|
67
|
+
#: How deep the open-element stack may grow. A page of unclosed `<p>`s
|
|
68
|
+
#: closed only by `</body>` would otherwise grow it without bound.
|
|
69
|
+
_MAX_DEPTH = 4000
|
|
70
|
+
|
|
71
|
+
#: Elements with no closing tag. Never pushed, so they never wait to close.
|
|
72
|
+
_VOID = frozenset({
|
|
73
|
+
"area", "base", "br", "col", "embed", "hr", "img", "input", "link",
|
|
74
|
+
"meta", "param", "source", "track", "wbr",
|
|
75
|
+
})
|
|
76
|
+
|
|
77
|
+
#: Elements whose whole subtree carries no prose a reader wants.
|
|
78
|
+
_DROP_TAGS = frozenset({
|
|
79
|
+
"script", "style", "noscript", "template", "svg", "math", "canvas",
|
|
80
|
+
"iframe", "object", "video", "audio", "picture", "map", "form",
|
|
81
|
+
"button", "select", "textarea", "label", "nav", "aside", "footer",
|
|
82
|
+
"dialog", "head", "title",
|
|
83
|
+
})
|
|
84
|
+
|
|
85
|
+
#: ARIA roles that are page furniture whatever element carries them.
|
|
86
|
+
_DROP_ROLES = frozenset({
|
|
87
|
+
"navigation", "banner", "complementary", "contentinfo", "search",
|
|
88
|
+
"dialog", "alertdialog", "menu", "menubar", "toolbar", "tablist",
|
|
89
|
+
})
|
|
90
|
+
|
|
91
|
+
#: Whole class or id names that mean furniture -- matched as written,
|
|
92
|
+
#: because the hyphenated ones only mean it as a whole (`sr-only`).
|
|
93
|
+
_DROP_NAMES = frozenset({
|
|
94
|
+
"sr-only", "visually-hidden", "visuallyhidden", "screen-reader-text",
|
|
95
|
+
"skip-link", "skiplink", "skip-to-content", "mw-editsection",
|
|
96
|
+
"mw-jump-link", "noprint", "navbox", "navbar", "breadcrumb",
|
|
97
|
+
"breadcrumbs", "sidebar", "toc", "cookie-banner", "cookie-notice",
|
|
98
|
+
# Sphinx's pilcrow beside every heading: a link to the heading itself.
|
|
99
|
+
"headerlink",
|
|
100
|
+
})
|
|
101
|
+
|
|
102
|
+
#: Words that mean furniture wherever they sit in a class or id: the value
|
|
103
|
+
#: is split on anything that is not a letter or digit, so `site-navbox`
|
|
104
|
+
#: yields `navbox` while `menuetto` yields nothing. Kept short on purpose --
|
|
105
|
+
#: each entry is prose this could drop if it were wrong.
|
|
106
|
+
_DROP_WORDS = frozenset({
|
|
107
|
+
"navbox", "breadcrumb", "breadcrumbs", "noprint", "editsection",
|
|
108
|
+
"cookie", "cookies", "consent", "advert", "advertisement", "newsletter",
|
|
109
|
+
# Measured on Wikipedia (batch 240): its article tabs, page tools and
|
|
110
|
+
# 37-language list sit INSIDE <main>, in `vector-menu-content`,
|
|
111
|
+
# `vector-dropdown` and `interlanguage-link` containers, and the
|
|
112
|
+
# language list alone used every one of the MAX_LINKS numbers.
|
|
113
|
+
"menu", "dropdown", "interlanguage",
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
#: Elements that end a paragraph, and those that end a line.
|
|
117
|
+
_PARAGRAPH = frozenset({"p", "blockquote", "table", "figure", "section",
|
|
118
|
+
"article", "details"})
|
|
119
|
+
_LINE = frozenset({
|
|
120
|
+
"address", "dd", "div", "dl", "dt", "fieldset", "figcaption", "header",
|
|
121
|
+
"hgroup", "li", "main", "summary", "tbody", "thead", "tfoot", "tr",
|
|
122
|
+
"caption", "ul", "ol",
|
|
123
|
+
})
|
|
124
|
+
|
|
125
|
+
_HEADING = {f"h{n}": n for n in range(1, 7)}
|
|
126
|
+
_WORD_SPLIT = re.compile(r"[^a-z0-9]+")
|
|
127
|
+
_SPACES = re.compile(r"[ \t\f\v\r\n]+")
|
|
128
|
+
_MARKER = re.compile(r"\[L(\d+)\]")
|
|
129
|
+
_LIST_ITEM = re.compile(r"^( *)(- |\d+\. )")
|
|
130
|
+
|
|
131
|
+
# A `<pre>` block's text travels between these two so that tidying the
|
|
132
|
+
# prose around it cannot reflow it. Neither can occur in parsed HTML text:
|
|
133
|
+
# the parser hands control characters through, but these two are removed
|
|
134
|
+
# from every data chunk before they could be confused with the sentinels.
|
|
135
|
+
_PRE_OPEN, _PRE_CLOSE = "\x02", "\x03"
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
@dataclass
|
|
139
|
+
class Document:
|
|
140
|
+
"""One fetched document as the model will read it."""
|
|
141
|
+
|
|
142
|
+
text: str
|
|
143
|
+
kind: str # "html", "pdf" or "text"
|
|
144
|
+
title: Optional[str] = None
|
|
145
|
+
links: dict = field(default_factory=dict) # marker number -> url
|
|
146
|
+
pages: Optional[int] = None # a PDF's page count
|
|
147
|
+
digest: str = "" # of the whole text, to tell versions apart
|
|
148
|
+
|
|
149
|
+
def __post_init__(self) -> None:
|
|
150
|
+
if not self.digest:
|
|
151
|
+
self.digest = text_digest(self.text)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def text_digest(text: str) -> str:
|
|
155
|
+
"""Whether two fetches of one URL produced the same document."""
|
|
156
|
+
return hashlib.sha256(text.encode("utf-8", "surrogatepass")).hexdigest()
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def markers_in(text: str) -> list:
|
|
160
|
+
"""The link numbers a slice of text refers to, first appearance first."""
|
|
161
|
+
seen: list = []
|
|
162
|
+
for match in _MARKER.finditer(text):
|
|
163
|
+
number = int(match.group(1))
|
|
164
|
+
if number not in seen:
|
|
165
|
+
seen.append(number)
|
|
166
|
+
return seen
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def split_marker_end(text: str, end: int) -> int:
|
|
170
|
+
"""`end`, moved past an `[L<n>]` marker it would otherwise cut in two.
|
|
171
|
+
|
|
172
|
+
A page boundary that fell inside a two-digit marker would leave its
|
|
173
|
+
first digit on one page and its second on the next, and neither half
|
|
174
|
+
would name a link.
|
|
175
|
+
"""
|
|
176
|
+
start = text.rfind("[L", max(0, end - 8), end)
|
|
177
|
+
if start == -1:
|
|
178
|
+
return end
|
|
179
|
+
close = text.find("]", start, start + 10)
|
|
180
|
+
if close != -1 and close >= end and _MARKER.fullmatch(text[start:close + 1]):
|
|
181
|
+
return close + 1
|
|
182
|
+
return end
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def text_document(text: str) -> Document:
|
|
186
|
+
"""A body that is already text: JSON, plain text, XML, source code."""
|
|
187
|
+
return Document(text=text, kind="text")
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
# ---- HTML -----------------------------------------------------------------
|
|
191
|
+
|
|
192
|
+
def _attrs(raw) -> dict:
|
|
193
|
+
return {name.lower(): (value or "") for name, value in raw}
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _is_furniture(tag: str, attrs: dict, drop_header: bool) -> bool:
|
|
197
|
+
if tag in _DROP_TAGS or (drop_header and tag == "header"):
|
|
198
|
+
return True
|
|
199
|
+
if "hidden" in attrs or attrs.get("aria-hidden", "").lower() == "true":
|
|
200
|
+
return True
|
|
201
|
+
if attrs.get("role", "").lower() in _DROP_ROLES:
|
|
202
|
+
return True
|
|
203
|
+
style = attrs.get("style", "").lower().replace(" ", "")
|
|
204
|
+
if "display:none" in style or "visibility:hidden" in style:
|
|
205
|
+
return True
|
|
206
|
+
for name in ("class", "id"):
|
|
207
|
+
value = attrs.get(name, "").lower()
|
|
208
|
+
if not value:
|
|
209
|
+
continue
|
|
210
|
+
if _DROP_NAMES.intersection(value.split()):
|
|
211
|
+
return True
|
|
212
|
+
if _DROP_WORDS.intersection(_WORD_SPLIT.split(value)):
|
|
213
|
+
return True
|
|
214
|
+
return False
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
class _Scan(HTMLParser):
|
|
218
|
+
"""Pass one: where the main region starts, and the page's title."""
|
|
219
|
+
|
|
220
|
+
def __init__(self) -> None:
|
|
221
|
+
super().__init__(convert_charrefs=True)
|
|
222
|
+
self.index = -1
|
|
223
|
+
self.main: Optional[int] = None
|
|
224
|
+
self.article: Optional[int] = None
|
|
225
|
+
self.body: Optional[int] = None
|
|
226
|
+
self._in_title = False
|
|
227
|
+
self.title_parts: list = []
|
|
228
|
+
|
|
229
|
+
def handle_starttag(self, tag, attrs):
|
|
230
|
+
self.index += 1
|
|
231
|
+
tag = tag.lower()
|
|
232
|
+
if tag == "title" and not self.title_parts:
|
|
233
|
+
self._in_title = True
|
|
234
|
+
if self.main is None and (
|
|
235
|
+
tag == "main" or _attrs(attrs).get("role", "").lower() == "main"):
|
|
236
|
+
self.main = self.index
|
|
237
|
+
elif tag == "article" and self.article is None:
|
|
238
|
+
self.article = self.index
|
|
239
|
+
elif tag == "body" and self.body is None:
|
|
240
|
+
self.body = self.index
|
|
241
|
+
|
|
242
|
+
def handle_startendtag(self, tag, attrs):
|
|
243
|
+
self.index += 1
|
|
244
|
+
|
|
245
|
+
def handle_endtag(self, tag):
|
|
246
|
+
if tag.lower() == "title":
|
|
247
|
+
self._in_title = False
|
|
248
|
+
|
|
249
|
+
def handle_data(self, data):
|
|
250
|
+
if self._in_title:
|
|
251
|
+
self.title_parts.append(data)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
class _Render(HTMLParser):
|
|
255
|
+
"""Pass two: the region's prose, with structure and link markers."""
|
|
256
|
+
|
|
257
|
+
def __init__(self, region: Optional[int], drop_header: bool,
|
|
258
|
+
base_url: str,
|
|
259
|
+
keep_link: Callable[[str], Optional[str]]) -> None:
|
|
260
|
+
super().__init__(convert_charrefs=True)
|
|
261
|
+
self.region = region # start-tag index of the region, or None
|
|
262
|
+
self.drop_header = drop_header
|
|
263
|
+
self.base_url = base_url
|
|
264
|
+
self.page_url = urldefrag(base_url)[0]
|
|
265
|
+
self.keep_link = keep_link
|
|
266
|
+
self.index = -1
|
|
267
|
+
self.stack: list = [] # (tag, is furniture)
|
|
268
|
+
self.inside = region is None # the whole document when no region
|
|
269
|
+
self.region_depth: Optional[int] = None
|
|
270
|
+
self.dropped = 0 # how many furniture elements are open
|
|
271
|
+
self.pre = 0
|
|
272
|
+
self.out: list = []
|
|
273
|
+
self.lists: list = [] # per open list: None for ul, a count for ol
|
|
274
|
+
self.row_cells = 0
|
|
275
|
+
self.link: Optional[str] = None
|
|
276
|
+
self.link_text = False
|
|
277
|
+
self.links: dict = {} # url -> number
|
|
278
|
+
self.done = False
|
|
279
|
+
|
|
280
|
+
def _visible(self) -> bool:
|
|
281
|
+
return self.inside and not self.dropped and not self.done
|
|
282
|
+
|
|
283
|
+
def handle_starttag(self, tag, attrs):
|
|
284
|
+
self.index += 1
|
|
285
|
+
tag = tag.lower()
|
|
286
|
+
attrs = _attrs(attrs)
|
|
287
|
+
if tag in _VOID:
|
|
288
|
+
self._void(tag)
|
|
289
|
+
return
|
|
290
|
+
is_region = self.region is not None and self.index == self.region
|
|
291
|
+
if is_region:
|
|
292
|
+
self.inside = True
|
|
293
|
+
self.region_depth = len(self.stack)
|
|
294
|
+
furniture = (self.inside and not is_region
|
|
295
|
+
and _is_furniture(tag, attrs, self.drop_header))
|
|
296
|
+
if len(self.stack) >= _MAX_DEPTH:
|
|
297
|
+
# Pathological nesting: stop opening, keep reading the text.
|
|
298
|
+
return
|
|
299
|
+
self.stack.append((tag, furniture))
|
|
300
|
+
if furniture:
|
|
301
|
+
self.dropped += 1
|
|
302
|
+
elif self._visible():
|
|
303
|
+
self._open(tag, attrs)
|
|
304
|
+
|
|
305
|
+
def handle_startendtag(self, tag, attrs):
|
|
306
|
+
# `<br/>` and friends: a start with no end to wait for.
|
|
307
|
+
self.index += 1
|
|
308
|
+
self._void(tag.lower())
|
|
309
|
+
|
|
310
|
+
def _void(self, tag):
|
|
311
|
+
if not self._visible():
|
|
312
|
+
return
|
|
313
|
+
if tag == "br":
|
|
314
|
+
self.out.append("\n")
|
|
315
|
+
elif tag == "hr":
|
|
316
|
+
self.out.append("\n\n")
|
|
317
|
+
|
|
318
|
+
def handle_endtag(self, tag):
|
|
319
|
+
tag = tag.lower()
|
|
320
|
+
if tag in _VOID:
|
|
321
|
+
return
|
|
322
|
+
# An end tag closes the nearest open element of its name and every
|
|
323
|
+
# element opened above it; one with no open match is ignored.
|
|
324
|
+
for depth in range(len(self.stack) - 1, -1, -1):
|
|
325
|
+
if self.stack[depth][0] == tag:
|
|
326
|
+
break
|
|
327
|
+
else:
|
|
328
|
+
return
|
|
329
|
+
while len(self.stack) > depth:
|
|
330
|
+
name, furniture = self.stack.pop()
|
|
331
|
+
if furniture:
|
|
332
|
+
self.dropped -= 1
|
|
333
|
+
elif self._visible():
|
|
334
|
+
self._close(name)
|
|
335
|
+
if (self.region_depth is not None
|
|
336
|
+
and len(self.stack) == self.region_depth):
|
|
337
|
+
# The region itself just closed: nothing after it is read.
|
|
338
|
+
self.done = True
|
|
339
|
+
|
|
340
|
+
def handle_data(self, data):
|
|
341
|
+
if not self._visible():
|
|
342
|
+
return
|
|
343
|
+
data = data.replace(_PRE_OPEN, "").replace(_PRE_CLOSE, "")
|
|
344
|
+
self.out.append(data if self.pre else _SPACES.sub(" ", data))
|
|
345
|
+
if self.link is not None and data.strip():
|
|
346
|
+
self.link_text = True
|
|
347
|
+
|
|
348
|
+
def _open(self, tag, attrs):
|
|
349
|
+
if tag in _HEADING:
|
|
350
|
+
self.out.append("\n\n" + "#" * _HEADING[tag] + " ")
|
|
351
|
+
elif tag == "li":
|
|
352
|
+
if self.lists and self.lists[-1] is not None:
|
|
353
|
+
self.lists[-1] += 1
|
|
354
|
+
bullet = f"{self.lists[-1]}. "
|
|
355
|
+
else:
|
|
356
|
+
bullet = "- "
|
|
357
|
+
self.out.append("\n" + " " * max(len(self.lists) - 1, 0) + bullet)
|
|
358
|
+
elif tag in ("ul", "ol"):
|
|
359
|
+
# A list nested in an item continues that item's lines.
|
|
360
|
+
if not self.lists:
|
|
361
|
+
self.out.append("\n")
|
|
362
|
+
self.lists.append(0 if tag == "ol" else None)
|
|
363
|
+
elif tag == "tr":
|
|
364
|
+
self.row_cells = 0
|
|
365
|
+
self.out.append("\n")
|
|
366
|
+
elif tag in ("td", "th"):
|
|
367
|
+
if self.row_cells:
|
|
368
|
+
self.out.append(" | ")
|
|
369
|
+
self.row_cells += 1
|
|
370
|
+
elif tag == "pre":
|
|
371
|
+
self.pre += 1
|
|
372
|
+
if self.pre == 1:
|
|
373
|
+
self.out.append("\n\n" + _PRE_OPEN)
|
|
374
|
+
elif tag == "a":
|
|
375
|
+
href = attrs.get("href", "")
|
|
376
|
+
self.link = self._link(href) if href else None
|
|
377
|
+
self.link_text = False
|
|
378
|
+
elif tag in _PARAGRAPH:
|
|
379
|
+
self.out.append("\n\n")
|
|
380
|
+
elif tag in _LINE:
|
|
381
|
+
self.out.append("\n")
|
|
382
|
+
|
|
383
|
+
def _close(self, tag):
|
|
384
|
+
if tag in _HEADING:
|
|
385
|
+
self.out.append("\n\n")
|
|
386
|
+
elif tag in ("ul", "ol"):
|
|
387
|
+
if self.lists:
|
|
388
|
+
self.lists.pop()
|
|
389
|
+
if not self.lists:
|
|
390
|
+
self.out.append("\n")
|
|
391
|
+
elif tag == "pre":
|
|
392
|
+
self.pre = max(self.pre - 1, 0)
|
|
393
|
+
if self.pre == 0:
|
|
394
|
+
self.out.append(_PRE_CLOSE + "\n\n")
|
|
395
|
+
elif tag == "a":
|
|
396
|
+
if self.link is not None and self.link_text:
|
|
397
|
+
self.out.append(f"[L{self.links[self.link]}]")
|
|
398
|
+
self.link = None
|
|
399
|
+
elif tag in _PARAGRAPH:
|
|
400
|
+
self.out.append("\n\n")
|
|
401
|
+
elif tag in _LINE and tag not in ("li", "tr"):
|
|
402
|
+
# A list item and a table row START their line; ending it too
|
|
403
|
+
# would leave a blank line between every two of them.
|
|
404
|
+
self.out.append("\n")
|
|
405
|
+
|
|
406
|
+
def _link(self, href: str) -> Optional[str]:
|
|
407
|
+
"""The link's key in `self.links` if it is numbered, else None."""
|
|
408
|
+
href = href.strip()
|
|
409
|
+
if href.startswith(("#", "javascript:", "mailto:", "tel:", "data:")):
|
|
410
|
+
return None
|
|
411
|
+
absolute = urldefrag(urljoin(self.base_url, href))[0]
|
|
412
|
+
if not absolute.startswith(("http://", "https://")):
|
|
413
|
+
return None
|
|
414
|
+
if absolute == self.page_url:
|
|
415
|
+
return None # a link to this same page
|
|
416
|
+
kept = self.keep_link(absolute)
|
|
417
|
+
if kept is None:
|
|
418
|
+
return None
|
|
419
|
+
if kept not in self.links:
|
|
420
|
+
if len(self.links) >= MAX_LINKS:
|
|
421
|
+
return None
|
|
422
|
+
self.links[kept] = len(self.links) + 1
|
|
423
|
+
return kept
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _prose(piece: str) -> str:
|
|
427
|
+
"""One stretch of text between `<pre>` blocks, tidied."""
|
|
428
|
+
lines = []
|
|
429
|
+
for line in piece.split("\n"):
|
|
430
|
+
item = _LIST_ITEM.match(line)
|
|
431
|
+
lines.append(line.rstrip() if item else line.strip())
|
|
432
|
+
text = "\n".join(lines)
|
|
433
|
+
# A heading whose text was all furniture leaves its `#` behind, and one
|
|
434
|
+
# whose text began inside an inline element leaves extra spaces.
|
|
435
|
+
text = re.sub(r"(?m)^#+ *$", "", text)
|
|
436
|
+
text = re.sub(r"(?m)^(#+) {2,}", r"\1 ", text)
|
|
437
|
+
# A table row's empty leading cells (a logo, a spacer) leave their
|
|
438
|
+
# separators behind; so do runs of empty cells between real ones.
|
|
439
|
+
text = re.sub(r"(?m)^(?:\| *)+", "", text)
|
|
440
|
+
text = re.sub(r"(?: *\|){2,} *", " | ", text)
|
|
441
|
+
text = re.sub(r"\n[ \t]*\n(?:[ \t]*\n)+", "\n\n", text)
|
|
442
|
+
return text.strip()
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def _tidy(raw: str) -> str:
|
|
446
|
+
"""Prose tidied stretch by stretch; a `<pre>` block kept exactly, its
|
|
447
|
+
own blank lines included -- code's layout is its meaning."""
|
|
448
|
+
pieces = re.split(f"{_PRE_OPEN}(.*?){_PRE_CLOSE}", raw, flags=re.S)
|
|
449
|
+
parts: list = []
|
|
450
|
+
for number, piece in enumerate(pieces):
|
|
451
|
+
if number % 2:
|
|
452
|
+
block = "\n".join(line.rstrip() for line in piece.split("\n"))
|
|
453
|
+
block = block.strip("\n")
|
|
454
|
+
if block.strip():
|
|
455
|
+
parts.append(block)
|
|
456
|
+
else:
|
|
457
|
+
prose = _prose(piece)
|
|
458
|
+
if prose:
|
|
459
|
+
parts.append(prose)
|
|
460
|
+
return "\n\n".join(parts)
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def html_document(html: str, base_url: str,
|
|
464
|
+
keep_link: Callable[[str], Optional[str]] = lambda url: url
|
|
465
|
+
) -> Document:
|
|
466
|
+
"""The readable text of an HTML page, its title and its numbered links.
|
|
467
|
+
|
|
468
|
+
`keep_link(url)` decides whether a link may be offered, and may return
|
|
469
|
+
a rewritten URL (a redacted one, say); None leaves the link as plain
|
|
470
|
+
text. `fetch_url` passes its URL policy and its length bound.
|
|
471
|
+
"""
|
|
472
|
+
scan = _Scan()
|
|
473
|
+
scan.feed(html)
|
|
474
|
+
scan.close()
|
|
475
|
+
if scan.main is not None:
|
|
476
|
+
region, drop_header = scan.main, False
|
|
477
|
+
elif scan.article is not None:
|
|
478
|
+
region, drop_header = scan.article, False
|
|
479
|
+
else:
|
|
480
|
+
# No region of its own: a top-level <header> is the site's banner.
|
|
481
|
+
region, drop_header = scan.body, True
|
|
482
|
+
render = _Render(region, drop_header, base_url, keep_link)
|
|
483
|
+
render.feed(html)
|
|
484
|
+
render.close()
|
|
485
|
+
title = _SPACES.sub(" ", "".join(scan.title_parts)).strip() or None
|
|
486
|
+
if title is not None:
|
|
487
|
+
title = title[:MAX_TITLE_CHARS]
|
|
488
|
+
links = {number: url for url, number in render.links.items()}
|
|
489
|
+
return Document(text=_tidy("".join(render.out)), kind="html",
|
|
490
|
+
title=title, links=links)
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""
|
|
2
|
+
tools/builtin/_pdf_text.py
|
|
3
|
+
|
|
4
|
+
A PDF's text, read in the killable child `tools/isolation.py` runs (batch
|
|
5
|
+
240, the owner's choice of both the library and the process).
|
|
6
|
+
|
|
7
|
+
pypdf, NOT markitdown[pdf], AND THE CHOICE WAS MEASURED. Three arXiv papers
|
|
8
|
+
-- Attention (one column), BERT (two columns, ACL layout) and the 77-page
|
|
9
|
+
Llama 2 report -- each checked for whole running sentences:
|
|
10
|
+
|
|
11
|
+
* pypdf 6.21.4: every probe sentence intact on Attention and BERT; 0.8%
|
|
12
|
+
of words run together on Llama 2; 1.2 s, 0.5 s and 4.2 s. 4.1 MB, no
|
|
13
|
+
dependencies.
|
|
14
|
+
* markitdown 0.1.9 [pdf]: words run together on every paper (22% on
|
|
15
|
+
Llama 2), and BERT's two columns came out as a Markdown table of word
|
|
16
|
+
fragments. 188 MB of dependencies, onnxruntime and numpy among them.
|
|
17
|
+
|
|
18
|
+
IN A CHILD PROCESS, because PDF parsers have a long history of hostile files
|
|
19
|
+
that make them spin or exhaust memory, and a thread cannot be stopped (the
|
|
20
|
+
argument `tools/isolation.py` makes at length). The child is under
|
|
21
|
+
`tool_compute_timeout_s` and, on POSIX, the CPU and address-space rlimits.
|
|
22
|
+
|
|
23
|
+
pypdf IS IMPORTED AT MODULE TOP, on purpose. The child imports this module
|
|
24
|
+
BEFORE a shared hosted deployment confines it, and the import is what needs
|
|
25
|
+
the filesystem; importing inside the handler would run after the walls go
|
|
26
|
+
up. The parent imports this module only when a PDF arrives.
|
|
27
|
+
|
|
28
|
+
REDACTED HERE, BEFORE THE WINDOW IS CUT (§52's RB1). The child sees the
|
|
29
|
+
whole text exactly once, and a credential straddling the edge of the window
|
|
30
|
+
it returns would be invisible to a redactor run on the window afterwards.
|
|
31
|
+
Positions are therefore positions in redacted text, the one space
|
|
32
|
+
`fetch_url` speaks (RB3).
|
|
33
|
+
|
|
34
|
+
WHAT CROSSES BACK. Only the window asked for, never the whole text:
|
|
35
|
+
`tools/isolation` refuses a result over 4 MB, and a long report's text is
|
|
36
|
+
larger than the window `fetch_url` holds anyway.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
import base64
|
|
42
|
+
import io
|
|
43
|
+
import re
|
|
44
|
+
|
|
45
|
+
from pypdf import PdfReader
|
|
46
|
+
|
|
47
|
+
from safety.policy_enforcement import redact_output_text
|
|
48
|
+
from tools.builtin._page_text import text_digest
|
|
49
|
+
|
|
50
|
+
#: Ligatures a PDF's fonts encode as one glyph. Expanded, so "finetune"
|
|
51
|
+
#: is "finetune" to anything that searches the text.
|
|
52
|
+
_LIGATURES = {"ff": "ff", "fi": "fi", "fl": "fl", "ffi": "ffi",
|
|
53
|
+
"ffl": "ffl", "ſt": "st", "st": "st"}
|
|
54
|
+
|
|
55
|
+
# A word broken across lines -- "representa-" / "tion" -- joined back up.
|
|
56
|
+
# Only before a lower-case letter: "Rad-" / "ford" is a name split by the
|
|
57
|
+
# column, while "BERT-" / "Base" is a real hyphen.
|
|
58
|
+
_LINE_HYPHEN = re.compile(r"(\w)-\n(?=[a-z])")
|
|
59
|
+
_SPACES = re.compile(r"[ \t]+")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _clean(text: str) -> str:
|
|
63
|
+
for glyph, letters in _LIGATURES.items():
|
|
64
|
+
text = text.replace(glyph, letters)
|
|
65
|
+
text = _LINE_HYPHEN.sub(r"\1", text)
|
|
66
|
+
text = "\n".join(_SPACES.sub(" ", line).strip() for line in text.split("\n"))
|
|
67
|
+
return re.sub(r"\n{3,}", "\n\n", text).strip()
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def pdf_window(params: dict) -> dict:
|
|
71
|
+
"""The child's handler: `{"data": base64, "start": int, "limit": int}`.
|
|
72
|
+
|
|
73
|
+
Returns `{"text", "total", "digest", "pages", "title", "redacted"}` --
|
|
74
|
+
the window `[start, start + limit)` of the whole document's text, with
|
|
75
|
+
`[page N]` before each page so a claim can cite one -- or `{"error"}`.
|
|
76
|
+
"""
|
|
77
|
+
try:
|
|
78
|
+
reader = PdfReader(io.BytesIO(base64.b64decode(params["data"])))
|
|
79
|
+
if reader.is_encrypted and not reader.decrypt(""):
|
|
80
|
+
return {"error": "this PDF is encrypted with a password."}
|
|
81
|
+
parts = []
|
|
82
|
+
for number, page in enumerate(reader.pages, start=1):
|
|
83
|
+
parts.append(f"[page {number}]\n{_clean(page.extract_text() or '')}")
|
|
84
|
+
except Exception as exc: # noqa: BLE001 -- a hostile file raises anything
|
|
85
|
+
return {"error": f"this PDF could not be read ({type(exc).__name__}: "
|
|
86
|
+
f"{str(exc)[:200]})."}
|
|
87
|
+
whole = "\n\n".join(parts)
|
|
88
|
+
clean = redact_output_text(whole)
|
|
89
|
+
start = max(int(params.get("start", 0)), 0)
|
|
90
|
+
limit = max(int(params.get("limit", 0)), 0)
|
|
91
|
+
title = None
|
|
92
|
+
try:
|
|
93
|
+
meta = reader.metadata
|
|
94
|
+
if meta is not None and isinstance(meta.title, str) and meta.title.strip():
|
|
95
|
+
title = meta.title.strip()[:300]
|
|
96
|
+
except Exception: # noqa: BLE001 -- metadata is optional and often broken
|
|
97
|
+
title = None
|
|
98
|
+
return {
|
|
99
|
+
"text": clean[start:start + limit],
|
|
100
|
+
"total": len(clean),
|
|
101
|
+
"digest": text_digest(clean),
|
|
102
|
+
"pages": len(parts),
|
|
103
|
+
"title": title,
|
|
104
|
+
"redacted": clean != whole,
|
|
105
|
+
# A scanned PDF has pages and no text layer: say so rather than
|
|
106
|
+
# return a document of page markers.
|
|
107
|
+
"no_text": not any(part.split("\n", 1)[-1].strip() for part in parts),
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def encode(data: bytes) -> str:
|
|
112
|
+
"""The PDF's bytes as the child's JSON request carries them."""
|
|
113
|
+
return base64.b64encode(data).decode("ascii")
|
package/tools/builtin/arxiv.py
CHANGED
|
@@ -31,6 +31,7 @@ import httpx
|
|
|
31
31
|
from defusedxml import DefusedXmlException
|
|
32
32
|
from pydantic import BaseModel, Field, field_validator
|
|
33
33
|
|
|
34
|
+
from tools.builtin import _net_common
|
|
34
35
|
from tools.builtin._net_common import MAX_URL_CHARS, ThirdPartyText, TTLCache
|
|
35
36
|
|
|
36
37
|
logger = logging.getLogger(__name__)
|
|
@@ -241,7 +242,10 @@ def _call_arxiv_api(search_query: str, max_results: int, sort_by: str,
|
|
|
241
242
|
is already in memory. Returns `str` as it always did, so every test that
|
|
242
243
|
patches this function is unaffected by the change underneath it.
|
|
243
244
|
"""
|
|
244
|
-
|
|
245
|
+
# ROADMAP_v3 §53: `httpx.stream`, admitted and counted against the tool
|
|
246
|
+
# meter. A refusal raises through `run()`'s except clauses, which name
|
|
247
|
+
# httpx's and the parser's errors only, to dispatch.
|
|
248
|
+
with _net_common.stream(
|
|
245
249
|
"GET",
|
|
246
250
|
ARXIV_API_URL,
|
|
247
251
|
params={
|