@yahyahammad/venastine 0.3.9 → 0.3.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONFIG_ARCHITECTURE.md +109 -0
- package/NOTICE +5 -3
- package/README.md +13 -4
- package/bin/venastine.mjs +62 -6
- package/bin/venastine.py +3 -1
- package/cli_commands.py +10 -0
- package/config.yaml +51 -0
- package/config_schema.py +16 -0
- package/core/client.py +45 -4
- package/core/commands.py +4 -0
- package/core/provider_errors.py +40 -0
- package/core/reasoning/scholar.py +14 -5
- package/docs/SECURITY.md +6 -0
- package/docs/THIRD_PARTY_NOTICES.md +133 -60
- package/mcp_client/client.py +49 -2
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/requirements.txt +19 -0
- package/tools/builtin/_net_common.py +878 -0
- package/tools/builtin/_page_text.py +490 -0
- package/tools/builtin/_pdf_text.py +113 -0
- package/tools/builtin/arxiv.py +5 -1
- package/tools/builtin/fetch_url.py +388 -120
- package/tools/builtin/web_search.py +81 -7
- package/tools/registry.py +10 -0
- package/tui/app.py +54 -0
- package/tui/app.tcss +7 -0
- package/tui/widgets.py +48 -0
|
@@ -12,7 +12,13 @@ from ddgs import DDGS
|
|
|
12
12
|
from pydantic import BaseModel, Field, field_validator
|
|
13
13
|
|
|
14
14
|
from safety.policy_enforcement import is_url_permitted
|
|
15
|
-
from tools.builtin
|
|
15
|
+
from tools.builtin import _net_common
|
|
16
|
+
from tools.builtin._net_common import (
|
|
17
|
+
MAX_URL_CHARS,
|
|
18
|
+
NetworkLimitReached,
|
|
19
|
+
ThirdPartyText,
|
|
20
|
+
TTLCache,
|
|
21
|
+
)
|
|
16
22
|
|
|
17
23
|
logger = logging.getLogger(__name__)
|
|
18
24
|
|
|
@@ -55,11 +61,19 @@ TOOL_SCHEMA = {
|
|
|
55
61
|
# ---- Normalized result shape ---------------------------------------------
|
|
56
62
|
|
|
57
63
|
class SearchResult(BaseModel):
|
|
64
|
+
"""One result as the model reads it.
|
|
65
|
+
|
|
66
|
+
BATCH 240 TOOK OUT WHAT CARRIED NOTHING. `published` was always None --
|
|
67
|
+
ddgs's text search returns no date, as its own comment here said -- and
|
|
68
|
+
both truncation flags were False on almost every row. Measured on a live
|
|
69
|
+
search, those three keys were 361 of the result's 2,651 characters. The
|
|
70
|
+
flags are now present only when TRUE (`model_dump(exclude_defaults=
|
|
71
|
+
True)`), the shape `arxiv_search` already uses for its own.
|
|
72
|
+
"""
|
|
73
|
+
|
|
58
74
|
title: str
|
|
59
75
|
url: str
|
|
60
76
|
snippet: str
|
|
61
|
-
published: Optional[str] = None # ddgs's text search doesn't return a
|
|
62
|
-
# publish date, so this stays None here
|
|
63
77
|
# ROADMAP_v3 §51 (PG8). Whether MAX_SNIPPET_CHARS cut this one, and
|
|
64
78
|
# where the rest is. Before §51 a cut snippet and a short one were
|
|
65
79
|
# indistinguishable, so "the source does not mention X" and "the first
|
|
@@ -85,6 +99,12 @@ class WebSearchError(Exception):
|
|
|
85
99
|
|
|
86
100
|
MAX_SNIPPET_CHARS = 300
|
|
87
101
|
|
|
102
|
+
# A cut snippet ends at a word, with this after it, rather than in the
|
|
103
|
+
# middle of one ("...into a vector th"): the flag says it was cut, and the
|
|
104
|
+
# mark says where. Not before 60% of the cap, so a snippet with one long
|
|
105
|
+
# word is still cut near the cap rather than to almost nothing.
|
|
106
|
+
_ELLIPSIS = "\u2026"
|
|
107
|
+
|
|
88
108
|
# Batch 156. 300 is the number `arxiv.py` settled for the same kind of
|
|
89
109
|
# value, measured there over sixty real entries whose longest title was
|
|
90
110
|
# 188 characters. NOT imported from it: two constants with one reason each
|
|
@@ -127,8 +147,24 @@ def _call_ddgs(query: str, num_results: int, page: int = 1) -> list[dict]:
|
|
|
127
147
|
same day: zero overlap), so a page here is "more results", not "the next
|
|
128
148
|
five distinct results", and nothing downstream may assume otherwise.
|
|
129
149
|
"""
|
|
130
|
-
|
|
131
|
-
|
|
150
|
+
# ROADMAP_v3 §53. Admitted against the tool meter before the search,
|
|
151
|
+
# and charged IO15's measured estimate after it, success or not: a
|
|
152
|
+
# failed attempt still spent most of what a good one does. `ddgs` talks
|
|
153
|
+
# through primp, which has no hook, so this is the one tool whose figure
|
|
154
|
+
# is not counted at the wire.
|
|
155
|
+
_net_common.TOOLS.admit()
|
|
156
|
+
try:
|
|
157
|
+
with DDGS(timeout=REQUEST_TIMEOUT_S) as ddgs_client:
|
|
158
|
+
return ddgs_client.text(query, max_results=num_results, page=page)
|
|
159
|
+
except Exception as exc:
|
|
160
|
+
# An empty search is an ANSWER. Retried as a failure, it cost two
|
|
161
|
+
# more full searches and then told the model "search failed after
|
|
162
|
+
# 3 attempts: No results found." -- measured, the same day.
|
|
163
|
+
if _empty(exc):
|
|
164
|
+
return []
|
|
165
|
+
raise
|
|
166
|
+
finally:
|
|
167
|
+
_net_common.record_web_search(num_results)
|
|
132
168
|
|
|
133
169
|
|
|
134
170
|
def _normalize(raw: list[dict]) -> list[SearchResult]:
|
|
@@ -220,13 +256,14 @@ def _normalize(raw: list[dict]) -> list[SearchResult]:
|
|
|
220
256
|
r.get("body"), MAX_SNIPPET_CHARS)
|
|
221
257
|
if snippet is None:
|
|
222
258
|
snippet = ""
|
|
259
|
+
elif snippet_truncated:
|
|
260
|
+
snippet = _at_a_word(snippet, MAX_SNIPPET_CHARS)
|
|
223
261
|
|
|
224
262
|
out.append(
|
|
225
263
|
SearchResult(
|
|
226
264
|
title=title,
|
|
227
265
|
url=url,
|
|
228
266
|
snippet=snippet,
|
|
229
|
-
published=None,
|
|
230
267
|
snippet_truncated=snippet_truncated,
|
|
231
268
|
title_truncated=title_truncated,
|
|
232
269
|
)
|
|
@@ -235,6 +272,32 @@ def _normalize(raw: list[dict]) -> list[SearchResult]:
|
|
|
235
272
|
return out
|
|
236
273
|
|
|
237
274
|
|
|
275
|
+
def _at_a_word(text: str, cap: int) -> str:
|
|
276
|
+
"""`text`, already cut to `cap`, cut back to a word and marked."""
|
|
277
|
+
room = text[:cap - len(_ELLIPSIS)]
|
|
278
|
+
space = room.rfind(" ")
|
|
279
|
+
if space >= cap * 0.6:
|
|
280
|
+
room = room[:space]
|
|
281
|
+
return room.rstrip(" ,;:-") + _ELLIPSIS
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _empty(exc: BaseException) -> bool:
|
|
285
|
+
"""ddgs's way of saying a search found nothing: it RAISES.
|
|
286
|
+
|
|
287
|
+
Measured (batch 240): `DDGS.text` raises `DDGSException("No results
|
|
288
|
+
found.")` when every backend answered and none had a row -- and only
|
|
289
|
+
then; a backend's own failure is raised under its own message. Read by
|
|
290
|
+
class NAME, as `core/provider_errors` reads an SDK's, because the suite
|
|
291
|
+
installs a fake `ddgs`.
|
|
292
|
+
"""
|
|
293
|
+
return (type(exc).__name__ == "DDGSException"
|
|
294
|
+
and str(exc).strip().rstrip(".").lower() == "no results found")
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _rate_limited(exc: BaseException) -> bool:
|
|
298
|
+
return type(exc).__name__ == "RatelimitException"
|
|
299
|
+
|
|
300
|
+
|
|
238
301
|
def _kept(results: list, asked: int) -> list:
|
|
239
302
|
"""`results`, cut to the number the caller asked for (A62).
|
|
240
303
|
|
|
@@ -284,7 +347,18 @@ def run(params: dict) -> dict:
|
|
|
284
347
|
results = _kept(_normalize(raw), parsed.num_results)
|
|
285
348
|
_cache.set(cache_key, results)
|
|
286
349
|
break
|
|
350
|
+
except NetworkLimitReached:
|
|
351
|
+
# §53 (IO6). A full meter is not weather: another attempt
|
|
352
|
+
# is refused the same way. Raised on, for dispatch to turn
|
|
353
|
+
# into the result the model reads.
|
|
354
|
+
raise
|
|
287
355
|
except Exception as e: # Simpler to try again then elaborate on every error type
|
|
356
|
+
if _rate_limited(e):
|
|
357
|
+
# Retrying a rate limit at once is asking again into
|
|
358
|
+
# the limit. Said once, plainly, so the model waits.
|
|
359
|
+
return {"error": (
|
|
360
|
+
"The search provider is rate-limiting this harness. "
|
|
361
|
+
"Wait a minute before searching again.")}
|
|
288
362
|
last_exc = e
|
|
289
363
|
logger.warning(
|
|
290
364
|
"web_search attempt %s/%s failed for %r: %s",
|
|
@@ -313,7 +387,7 @@ def run(params: dict) -> dict:
|
|
|
313
387
|
return {"results": [], "result_count": 0, "message": "No results found."}
|
|
314
388
|
|
|
315
389
|
out = {
|
|
316
|
-
"results": [r.model_dump() for r in results],
|
|
390
|
+
"results": [r.model_dump(exclude_defaults=True) for r in results],
|
|
317
391
|
"result_count": len(results),
|
|
318
392
|
"page": parsed.page,
|
|
319
393
|
}
|
package/tools/registry.py
CHANGED
|
@@ -73,6 +73,7 @@ from tools.builtin import (
|
|
|
73
73
|
todo,
|
|
74
74
|
web_search,
|
|
75
75
|
)
|
|
76
|
+
from tools.builtin._net_common import NetworkLimitReached
|
|
76
77
|
|
|
77
78
|
logger = logging.getLogger(__name__)
|
|
78
79
|
|
|
@@ -891,6 +892,15 @@ class ToolRegistry:
|
|
|
891
892
|
tool_name=tool_name)
|
|
892
893
|
else:
|
|
893
894
|
result = spec.handler(params, **injected)
|
|
895
|
+
except NetworkLimitReached as e:
|
|
896
|
+
# ROADMAP_v3 §53 (IO6). A refusal, not a failure: the model
|
|
897
|
+
# reads the meter's own sentence, which says what is full and
|
|
898
|
+
# when room returns, rather than "<tool> failed: ...". No
|
|
899
|
+
# traceback -- nothing is broken -- and no warning here, because
|
|
900
|
+
# the meter has already said it once for the whole run of
|
|
901
|
+
# refusals. ABOVE the containment below, which would otherwise
|
|
902
|
+
# log it as a crash.
|
|
903
|
+
result = {"error": e.reason}
|
|
894
904
|
except Exception as e: # noqa: BLE001 -- contained on purpose
|
|
895
905
|
# A raising tool used to take the whole run down with it. In
|
|
896
906
|
# chat that is a lost turn; in the research pipeline it is a
|
package/tui/app.py
CHANGED
|
@@ -126,6 +126,7 @@ from tui.widgets import (
|
|
|
126
126
|
CallSelected,
|
|
127
127
|
EffortRaven,
|
|
128
128
|
GoalBanner,
|
|
129
|
+
NetworkLine,
|
|
129
130
|
PostureBadge,
|
|
130
131
|
PromptInput,
|
|
131
132
|
RavenPanel,
|
|
@@ -990,6 +991,10 @@ class VenastineApp(App):
|
|
|
990
991
|
# gated call of a session died in refresh_usage_line with
|
|
991
992
|
# NoMatches. None until mounted, like the four above.
|
|
992
993
|
self._usage_widget: UsageLine | None = None
|
|
994
|
+
# ROADMAP_v3 §53, held for the same reason, and refreshed from the
|
|
995
|
+
# same per-event path at most once a second (`_network_drawn_at`).
|
|
996
|
+
self._network_widget: NetworkLine | None = None
|
|
997
|
+
self._network_drawn_at = 0.0
|
|
993
998
|
self._raven_widget: RavenPanel | None = None
|
|
994
999
|
# Goal banner, checklist and research progress, held for the same
|
|
995
1000
|
# reason: refresh_goal_banner/refresh_todo_panel run off notice
|
|
@@ -1237,6 +1242,9 @@ class VenastineApp(App):
|
|
|
1237
1242
|
# #4: billed-since-resume and current context size, one
|
|
1238
1243
|
# line. Hidden until a turn produces figures.
|
|
1239
1244
|
yield UsageLine(id="usage-line")
|
|
1245
|
+
# ROADMAP_v3 §53: the network meters, under the figures
|
|
1246
|
+
# they read with. Hidden while every meter is off.
|
|
1247
|
+
yield NetworkLine(id="network-line")
|
|
1240
1248
|
if self._todo_position not in ("top", "bottom"):
|
|
1241
1249
|
yield TodoPanel(id="todo-panel")
|
|
1242
1250
|
yield ResearchProgress(id="research-progress")
|
|
@@ -1250,6 +1258,7 @@ class VenastineApp(App):
|
|
|
1250
1258
|
# what makes that survivable.
|
|
1251
1259
|
self._transcript_widget = self.query_one("#transcript", Transcript)
|
|
1252
1260
|
self._usage_widget = self.query_one("#usage-line", UsageLine)
|
|
1261
|
+
self._network_widget = self.query_one("#network-line", NetworkLine)
|
|
1253
1262
|
self._raven_widget = self.query_one("#raven", RavenPanel)
|
|
1254
1263
|
self._goal_widget = self.query_one("#goal-banner", GoalBanner)
|
|
1255
1264
|
self._todo_widget = self.query_one("#todo-panel", TodoPanel)
|
|
@@ -1293,6 +1302,8 @@ class VenastineApp(App):
|
|
|
1293
1302
|
self.refresh_todo_panel()
|
|
1294
1303
|
self.refresh_agent_panel()
|
|
1295
1304
|
self.refresh_status()
|
|
1305
|
+
# A meter already full from an earlier session shows at once.
|
|
1306
|
+
self.refresh_network_line(force=True)
|
|
1296
1307
|
# §79. A hosted session's screen goes to a browser over the
|
|
1297
1308
|
# network, and a blinking cursor is two redraws a second to a tab
|
|
1298
1309
|
# nobody is typing in (RavenPanel's `still_when_idle` says what that
|
|
@@ -3583,6 +3594,33 @@ class VenastineApp(App):
|
|
|
3583
3594
|
self._memory.billed_tokens, self._memory.last_input_tokens,
|
|
3584
3595
|
ceiling, overridden, window)
|
|
3585
3596
|
|
|
3597
|
+
def refresh_network_line(self, force: bool = False) -> None:
|
|
3598
|
+
"""ROADMAP_v3 §53 (IO13): repaint the network meters' line.
|
|
3599
|
+
|
|
3600
|
+
AT MOST ONCE A SECOND unless forced. The per-event path calls this
|
|
3601
|
+
on every token delta, and a meter's state is a walk over its
|
|
3602
|
+
buckets -- cheap once, not cheap a hundred times a second for a
|
|
3603
|
+
figure that only moves when a request starts or a chunk lands.
|
|
3604
|
+
Forced on mount and on a final response, so a turn always ends
|
|
3605
|
+
with the line current.
|
|
3606
|
+
|
|
3607
|
+
Contained, like the usage line: a sidebar figure must never be the
|
|
3608
|
+
reason a turn dies.
|
|
3609
|
+
"""
|
|
3610
|
+
widget = self._network_widget
|
|
3611
|
+
if widget is None:
|
|
3612
|
+
return
|
|
3613
|
+
now = monotonic()
|
|
3614
|
+
if not force and now - self._network_drawn_at < 1.0:
|
|
3615
|
+
return
|
|
3616
|
+
self._network_drawn_at = now
|
|
3617
|
+
try:
|
|
3618
|
+
from tools.builtin import _net_common
|
|
3619
|
+
|
|
3620
|
+
widget.update_meters(_net_common.states())
|
|
3621
|
+
except Exception: # noqa: BLE001 -- see above
|
|
3622
|
+
logger.debug("network line refresh failed", exc_info=True)
|
|
3623
|
+
|
|
3586
3624
|
def on_agent_stack_changed(self, message: AgentStackChanged) -> None:
|
|
3587
3625
|
"""The sink spoke. Runs on the UI thread."""
|
|
3588
3626
|
self._agent_stack = message.stack
|
|
@@ -3739,6 +3777,7 @@ class VenastineApp(App):
|
|
|
3739
3777
|
logger.debug("loop event [%s] with %d screen(s) on the stack",
|
|
3740
3778
|
kinds, depth)
|
|
3741
3779
|
self.refresh_usage_line()
|
|
3780
|
+
self.refresh_network_line(force=event.final_response is not None)
|
|
3742
3781
|
# Batch 61. Counted whether or not the renderer draws it: the
|
|
3743
3782
|
# commit cap withholds a table from its header row with no
|
|
3744
3783
|
# HOLD_LIMIT to bound it, and a figure that stopped there
|
|
@@ -6100,6 +6139,20 @@ def _cmd_window(app: VenastineApp, args: str) -> None:
|
|
|
6100
6139
|
"value.")
|
|
6101
6140
|
|
|
6102
6141
|
|
|
6142
|
+
def _cmd_network(app: VenastineApp, args: str) -> None:
|
|
6143
|
+
"""ROADMAP_v3 §53 (IO13): every network meter, its use and its room.
|
|
6144
|
+
|
|
6145
|
+
The lines are `_net_common.network_report`'s, which the CLI prints
|
|
6146
|
+
too. Read-only: a ceiling is set in config.yaml, like every other
|
|
6147
|
+
limit, so there is no argument to take.
|
|
6148
|
+
"""
|
|
6149
|
+
from tools.builtin import _net_common
|
|
6150
|
+
|
|
6151
|
+
for line in _net_common.network_report():
|
|
6152
|
+
app._transcript.write_system(line)
|
|
6153
|
+
app.refresh_network_line(force=True)
|
|
6154
|
+
|
|
6155
|
+
|
|
6103
6156
|
def _cmd_trigger(app: VenastineApp, args: str) -> None:
|
|
6104
6157
|
"""Override the working-set compaction ceiling for this session.
|
|
6105
6158
|
|
|
@@ -8273,6 +8326,7 @@ def register_builtin_commands() -> None:
|
|
|
8273
8326
|
bind("compact", _cmd_compact),
|
|
8274
8327
|
bind("window", _cmd_window),
|
|
8275
8328
|
bind("trigger", _cmd_trigger),
|
|
8329
|
+
bind("network", _cmd_network),
|
|
8276
8330
|
bind("config", _cmd_config, complete=_config_rows),
|
|
8277
8331
|
bind("secrets", _cmd_secrets),
|
|
8278
8332
|
bind("claims", _cmd_claims),
|
package/tui/app.tcss
CHANGED
|
@@ -222,6 +222,13 @@
|
|
|
222
222
|
color: $text-muted;
|
|
223
223
|
}
|
|
224
224
|
|
|
225
|
+
/* ROADMAP_v3 §53 network meters, directly under the usage line it reads
|
|
226
|
+
with. Hidden while every meter is off. */
|
|
227
|
+
#network-line {
|
|
228
|
+
height: auto;
|
|
229
|
+
color: $text-muted;
|
|
230
|
+
}
|
|
231
|
+
|
|
225
232
|
/* §22 live research view. Auto height so it takes no sidebar space until
|
|
226
233
|
a run starts.
|
|
227
234
|
|
package/tui/widgets.py
CHANGED
|
@@ -563,6 +563,54 @@ class UsageLine(Static):
|
|
|
563
563
|
style=self._styles().get("system", ""))))
|
|
564
564
|
|
|
565
565
|
|
|
566
|
+
class NetworkLine(Static):
|
|
567
|
+
"""ROADMAP_v3 §53 (IO13): each network meter that is on, one line
|
|
568
|
+
under the usage line -- `net · tools 1.2M/500M · per 24 h`.
|
|
569
|
+
|
|
570
|
+
HIDDEN WHILE EVERY METER IS OFF, which is the shipped state, so a
|
|
571
|
+
harness with no limit set shows nothing for a feature it is not using.
|
|
572
|
+
|
|
573
|
+
MACHINE-WIDE, NOT PER THREAD, and so never reset on a thread switch the
|
|
574
|
+
way UsageLine is: the ledger counts every harness process, and a new
|
|
575
|
+
thread has not spent less of the allowance.
|
|
576
|
+
|
|
577
|
+
The text is `_net_common.sidebar_text`'s, so this and `/network` cannot
|
|
578
|
+
disagree about what a figure means; this widget decides only colour.
|
|
579
|
+
"""
|
|
580
|
+
|
|
581
|
+
def __init__(self, **kwargs):
|
|
582
|
+
super().__init__(**kwargs)
|
|
583
|
+
self.display = False
|
|
584
|
+
self._last = None
|
|
585
|
+
|
|
586
|
+
def _styles(self) -> dict:
|
|
587
|
+
try:
|
|
588
|
+
app = self.app
|
|
589
|
+
except Exception: # noqa: BLE001 -- no running app; render unstyled
|
|
590
|
+
return {}
|
|
591
|
+
return themes.styles_for(app)
|
|
592
|
+
|
|
593
|
+
def update_meters(self, states: list) -> None:
|
|
594
|
+
from tools.builtin import _net_common
|
|
595
|
+
|
|
596
|
+
text = _net_common.sidebar_text(states)
|
|
597
|
+
full = any(state.full for state in states)
|
|
598
|
+
key = (text, full)
|
|
599
|
+
if key == self._last:
|
|
600
|
+
return
|
|
601
|
+
self._last = key
|
|
602
|
+
if text is None:
|
|
603
|
+
self.display = False
|
|
604
|
+
self.update("")
|
|
605
|
+
return
|
|
606
|
+
self.display = True
|
|
607
|
+
# `warning` when a meter is full: the one state here that changes
|
|
608
|
+
# what the harness will do next.
|
|
609
|
+
role = "warning" if full else "system"
|
|
610
|
+
self.update(as_content(
|
|
611
|
+
Text(text, style=self._styles().get(role, ""))))
|
|
612
|
+
|
|
613
|
+
|
|
566
614
|
class GoalBanner(Static):
|
|
567
615
|
"""Persistent-objective banner (§18 goal mode). Hidden when the
|
|
568
616
|
thread has no goal; a one-line reminder of what the session is
|