@yahyahammad/venastine 0.2.0 → 0.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/.env.example +19 -14
  2. package/CONFIG_ARCHITECTURE.md +2002 -0
  3. package/NOTICE +1 -1
  4. package/README.md +233 -94
  5. package/agents/builtin/build.md +73 -0
  6. package/agents/builtin/general.md +80 -0
  7. package/agents/builtin/plan.md +23 -5
  8. package/agents/builtin/test.md +65 -0
  9. package/agents/builtin/writer.md +67 -0
  10. package/agents/manager.py +261 -195
  11. package/agents/subagent_tool.py +277 -43
  12. package/agents/tui_commands.py +7 -12
  13. package/bin/venastine.mjs +528 -382
  14. package/bin/venastine.py +71 -0
  15. package/cli_commands.py +835 -0
  16. package/config.py +95 -1198
  17. package/config.yaml +1090 -0
  18. package/config_edit.py +1301 -0
  19. package/config_schema.py +966 -0
  20. package/config_update.py +749 -0
  21. package/core/agent_activity.py +158 -20
  22. package/core/ansi.py +452 -0
  23. package/core/approval.py +27 -5
  24. package/core/call_view.py +364 -0
  25. package/core/cancel.py +121 -0
  26. package/core/chatgpt_auth.py +747 -0
  27. package/core/claims_view.py +107 -0
  28. package/core/client.py +1238 -82
  29. package/core/commands.py +454 -0
  30. package/core/compaction.py +830 -66
  31. package/core/config_loader.py +516 -97
  32. package/core/connect.py +686 -0
  33. package/core/events.py +68 -6
  34. package/core/file_mentions.py +280 -0
  35. package/core/interaction.py +87 -4
  36. package/core/line_pattern.py +128 -0
  37. package/core/loop.py +1516 -440
  38. package/core/memory.py +120 -8
  39. package/core/pipeline_models.py +23 -9
  40. package/core/provider_errors.py +234 -0
  41. package/core/reasoning/confidence_scoring.py +70 -54
  42. package/core/reasoning/json_retry.py +19 -1
  43. package/core/reasoning/orchestrator.py +232 -53
  44. package/core/reasoning/output_writer.py +90 -31
  45. package/core/reasoning/payload_validation.py +133 -7
  46. package/core/reasoning/pipeline_storage.py +427 -361
  47. package/core/reasoning/review.py +659 -615
  48. package/core/reasoning/scholar.py +38 -14
  49. package/core/reasoning/source_corpus.py +110 -4
  50. package/core/replay.py +158 -26
  51. package/core/session.py +36 -12
  52. package/core/session_view.py +259 -0
  53. package/core/session_wake.py +259 -0
  54. package/core/shell_sessions.py +1553 -0
  55. package/core/workspace_trust.py +1 -1
  56. package/credentials.py +308 -180
  57. package/database.py +82 -1
  58. package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
  59. package/docs/SECURITY.md +108 -0
  60. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  61. package/env_secrets.py +118 -77
  62. package/json_store.py +236 -16
  63. package/logging_setup.py +121 -11
  64. package/main.py +1671 -199
  65. package/mcp_client/client.py +257 -23
  66. package/mcp_client/config.py +484 -379
  67. package/mcp_client/registration.py +163 -2
  68. package/memories/manager.py +15 -0
  69. package/memories/tui_commands.py +7 -12
  70. package/package.json +234 -4
  71. package/project_init/config_files.py +4 -0
  72. package/project_init/generator.py +201 -37
  73. package/project_init/manifest.py +418 -406
  74. package/project_init/tui_commands.py +7 -4
  75. package/prompts/revalidate.md +1 -1
  76. package/prompts/revise.md +17 -15
  77. package/prompts/system_prompts.py +64 -10
  78. package/providers.json.example +94 -92
  79. package/pyproject.toml +51 -2
  80. package/requirements.txt +252 -29
  81. package/safety/policy_enforcement.py +353 -17
  82. package/security/askpass.py +419 -0
  83. package/security/capability.py +37 -7
  84. package/security/confine.py +484 -0
  85. package/security/hosted.py +182 -0
  86. package/security/permissions.py +54 -7
  87. package/security/posture.py +124 -12
  88. package/security/protected_paths.py +198 -6
  89. package/security/sandbox.py +3262 -139
  90. package/security/secrets.py +855 -0
  91. package/skills/tui_commands.py +3 -4
  92. package/storage.py +368 -35
  93. package/tools/base.py +465 -305
  94. package/tools/builtin/_math_common.py +306 -16
  95. package/tools/builtin/_net_common.py +267 -61
  96. package/tools/builtin/arxiv.py +681 -223
  97. package/tools/builtin/ask_user.py +231 -213
  98. package/tools/builtin/discrete_math.py +35 -4
  99. package/tools/builtin/fetch_url.py +239 -21
  100. package/tools/builtin/file_ops.py +341 -33
  101. package/tools/builtin/geometry.py +184 -148
  102. package/tools/builtin/get_time.py +2 -2
  103. package/tools/builtin/linear_algebra.py +66 -23
  104. package/tools/builtin/load_skill.py +10 -0
  105. package/tools/builtin/logic.py +171 -12
  106. package/tools/builtin/pin.py +46 -12
  107. package/tools/builtin/probability_stats.py +224 -228
  108. package/tools/builtin/project_docs.py +80 -21
  109. package/tools/builtin/remember.py +21 -4
  110. package/tools/builtin/shell.py +586 -67
  111. package/tools/builtin/shell_sessions.py +613 -0
  112. package/tools/builtin/symbolic_math.py +80 -10
  113. package/tools/builtin/todo.py +5 -3
  114. package/tools/builtin/web_search.py +329 -151
  115. package/tools/context.py +23 -0
  116. package/tools/isolation.py +387 -314
  117. package/tools/registry.py +347 -30
  118. package/tui/app.py +4645 -438
  119. package/tui/app.tcss +185 -2
  120. package/tui/commands.py +10 -154
  121. package/tui/diffs.py +286 -265
  122. package/tui/hosted_driver.py +102 -0
  123. package/tui/markdown.py +60 -10
  124. package/tui/meters.py +103 -7
  125. package/tui/screens.py +346 -92
  126. package/tui/themes.py +59 -19
  127. package/tui/widgets.py +1575 -111
  128. package/SECURITY.md +0 -88
  129. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -0,0 +1,102 @@
1
+ """
2
+ tui/hosted_driver.py
3
+
4
+ ROADMAP_v3 §79 (HD13). Textual's web driver, plus the three packets a hosted
5
+ bridge sends back about a ModelScope sign-in -- on either site -- it is
6
+ completing for this session.
7
+
8
+ The bridge -- `deploy/hosted/supervisor.py` -- runs every hosted session under
9
+ this driver. It is the only writer of the session's stdin, and everything a
10
+ browser sends reaches it there as screen input (a `D` packet), never as a
11
+ meta packet: so a `venastine_*` packet arriving here was written by the
12
+ bridge, and a page cannot forge one. The app still accepts a credential only
13
+ for the sign-in it is waiting for (tui/app.py), because "only the bridge can
14
+ send this" is a property of the deployment, and the app should not depend on
15
+ it alone.
16
+
17
+ Outside a hosted session nothing selects this driver, and nothing sends it
18
+ these packets.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ from textual.drivers.web_driver import WebDriver
24
+ from textual.message import Message
25
+
26
+
27
+ class BridgeSignInUrl(Message):
28
+ """The bridge started a sign-in: the consent page to open."""
29
+
30
+ def __init__(self, request: str, url: str) -> None:
31
+ super().__init__()
32
+ self.request = request
33
+ self.url = url
34
+
35
+
36
+ class BridgeCredential(Message):
37
+ """The bridge traded the pasted code for a token. Never logged: the
38
+ debug breadcrumbs name message types, not their fields (batch 78)."""
39
+
40
+ def __init__(self, request: str, token: str) -> None:
41
+ super().__init__()
42
+ self.request = request
43
+ self.token = token
44
+
45
+
46
+ class BridgeSignInFailed(Message):
47
+ """The sign-in did not finish; `retry` when the paste may be tried
48
+ again in the same flow."""
49
+
50
+ def __init__(self, request: str, reason: str, retry: bool) -> None:
51
+ super().__init__()
52
+ self.request = request
53
+ self.reason = reason
54
+ self.retry = retry
55
+
56
+
57
+ def bridge_message(packet_type: str, payload: dict) -> Message | None:
58
+ """The app message for one of the bridge's sign-in packets, or None for
59
+ any other packet -- including one of these with a field missing."""
60
+ request = payload.get("id")
61
+ if not isinstance(request, str):
62
+ return None
63
+ if packet_type == "venastine_sign_in_url":
64
+ url = payload.get("url")
65
+ if isinstance(url, str) and url.startswith("https://"):
66
+ return BridgeSignInUrl(request, url)
67
+ elif packet_type == "venastine_credential":
68
+ token = payload.get("token")
69
+ if isinstance(token, str) and token:
70
+ return BridgeCredential(request, token)
71
+ elif packet_type == "venastine_sign_in_failed":
72
+ return BridgeSignInFailed(request, str(payload.get("reason") or
73
+ "The sign-in failed."),
74
+ payload.get("retry") is True)
75
+ return None
76
+
77
+
78
+ class HostedWebDriver(WebDriver):
79
+ """The web driver a hosted bridge starts its sessions with."""
80
+
81
+ def on_meta(self, packet_type: str, payload: dict[str, object]) -> None:
82
+ message = bridge_message(packet_type, payload)
83
+ if message is not None:
84
+ self._app.post_message(message)
85
+ return
86
+ super().on_meta(packet_type, payload)
87
+
88
+ def send_to_bridge(self, meta: dict) -> None:
89
+ """One meta packet to the bridge. Call it on the UI thread, which
90
+ is the thread that writes the screen: two threads writing one pipe
91
+ could interleave a packet."""
92
+ self.write_meta(meta)
93
+
94
+
95
+ def send_to_bridge(app, meta: dict) -> bool:
96
+ """Send `meta` through the app's driver when it is this one; False when
97
+ the session is not running under a bridge that can hear it."""
98
+ driver = getattr(app, "_driver", None)
99
+ if not isinstance(driver, HostedWebDriver):
100
+ return False
101
+ driver.send_to_bridge(meta)
102
+ return True
package/tui/markdown.py CHANGED
@@ -1027,26 +1027,76 @@ def plain_split(line: str, width: int):
1027
1027
  hard cut at the width Rich would use), over plain characters rather
1028
1028
  than mark spans. A character is atomic: the cut never lands inside
1029
1029
  one, so a glyph wider than the row survives whole for Rich to wrap.
1030
+
1031
+ The rule itself is `_plain_cut`, shared with `plain_wrap`, so the
1032
+ streamed commit and the replayed wrap cannot disagree about a row.
1030
1033
  """
1031
- if sum(cell_len(char) for char in line) <= width:
1034
+ cut = _plain_cut(line, 0, width)
1035
+ if cut is None:
1032
1036
  return "", line
1037
+ return line[:cut], line[cut:]
1038
+
1039
+
1040
+ def _plain_cut(line: str, start: int, width: int):
1041
+ """Where `plain_split`'s rule ends the row that begins at `start`, or
1042
+ None when the rest of the line fits in one row or cannot be cut.
1043
+
1044
+ Last space that still fits, else a hard cut at the width -- and never
1045
+ at `start` itself, which is how a glyph wider than the row survives
1046
+ whole. It scans about one row past `start`, never the whole
1047
+ remainder, which is what lets `plain_wrap` walk a long line once.
1048
+ """
1033
1049
  cells, space_at = 0, -1
1034
- for index, char in enumerate(line):
1050
+ for index in range(start, len(line)):
1051
+ char = line[index]
1035
1052
  if char == " " and cells <= width:
1036
1053
  space_at = index
1037
1054
  cells += cell_len(char)
1038
1055
  if cells > width:
1039
1056
  break
1040
1057
  else:
1041
- return "", line
1042
- if space_at > 0:
1043
- return line[:space_at + 1], line[space_at + 1:]
1058
+ return None
1059
+ if space_at > start:
1060
+ return space_at + 1
1044
1061
  cells = 0
1045
- for index, char in enumerate(line):
1046
- if cells + cell_len(char) > width:
1047
- return (line[:index], line[index:]) if index > 0 else ("", line)
1048
- cells += cell_len(char)
1049
- return "", line
1062
+ for index in range(start, len(line)):
1063
+ char_cells = cell_len(line[index])
1064
+ if cells + char_cells > width:
1065
+ return index if index > start else None
1066
+ cells += char_cells
1067
+ return None
1068
+
1069
+
1070
+ def plain_wrap(line: str, width: int) -> list[str]:
1071
+ """The rows `line` occupies when drawn at `width` cells, by
1072
+ `plain_split`'s rule -- the thinking span's wrap.
1073
+
1074
+ `plain_split` applied until it commits nothing, but in one walk over
1075
+ the line: calling it in a loop re-measures the whole remainder for
1076
+ every row, which is quadratic on a long paragraph and paid again on
1077
+ every replay (`wrap_display`'s reason, one grammar over). The last row
1078
+ is what that rule would have kept buffered, so a line that fits comes
1079
+ back as itself and an empty line as one empty row.
1080
+
1081
+ A cut at the very END of the line -- a trailing space that overflows
1082
+ the row by its own cell -- closes the last row rather than opening an
1083
+ empty one. `plain_split` hands that empty remainder back, and Rich
1084
+ folds the space into the row it ends, so drawing the remainder put a
1085
+ bar with nothing beside it under every chunk `thinking_delta` commits
1086
+ that way (measured: three extra rows in a streamed run of wide glyphs).
1087
+ """
1088
+ if width <= 0:
1089
+ return [line]
1090
+ rows, start = [], 0
1091
+ while True:
1092
+ cut = _plain_cut(line, start, width)
1093
+ if cut is None:
1094
+ break
1095
+ rows.append(line[start:cut])
1096
+ start = cut
1097
+ if start < len(line) or not rows:
1098
+ rows.append(line[start:])
1099
+ return rows
1050
1100
 
1051
1101
 
1052
1102
  def wrap_display(line: str, width: int, *, block: bool = False):
package/tui/meters.py CHANGED
@@ -29,17 +29,32 @@ WHAT THIS OWNS
29
29
 
30
30
  - `start` / `stop` -- turn boundaries, and the elapsed time between.
31
31
  - `set_blocked` -- the pause while a modal is waiting on a human.
32
+ - `set_tool_running` -- the pause while a TOOL is running (rate only).
32
33
  - `add_output_chars` -- the live estimate's numerator.
33
34
  - `subtitle` -- the three live figures as one string, or None.
34
35
  - `completion` -- the line written under the finished answer.
35
36
 
37
+ TWO CLOCKS, AND THEY ANSWER DIFFERENT QUESTIONS (batch 118). `elapsed` is
38
+ wall clock minus the human wait, because what it answers is "how long did I
39
+ wait for this turn", and a tool taking two minutes is part of that. The RATE
40
+ divides by elapsed minus TOOL time, because what it answers is "how fast is
41
+ the model writing", and a tool running is not the model writing.
42
+
43
+ Until batch 118 there was one clock and the rate used it, so the live figure
44
+ read `~0 tok/s` on exactly the turns it exists for: it needs only fewer than
45
+ two characters per active second to round to zero, and a turn that thinks for
46
+ a minute and then runs a two-minute tool produces a handful of characters
47
+ across three minutes. The numerator was wrong in the same direction --
48
+ thinking and tool-call arguments are generated output and neither was
49
+ counted -- so both sides were fixed together or neither was.
50
+
36
51
  TWO INSTRUMENTS, AND THEY ARE NOT THE SAME ONE. The live rate is an
37
52
  ESTIMATE from characters, because `StreamToken` carries no incremental
38
- usage and only three of the nineteen configured providers report usage on
53
+ usage and only three of the fifteen configured providers report usage on
39
54
  a streaming call at all (D21: OpenAI-compatible streaming returns none
40
55
  unless `stream_options` is sent, and Mistral rejects that parameter
41
56
  outright). An exact live rate would therefore read `0 tok/s` forever on
42
- sixteen providers, which is D21's own failure mode -- correct-looking
57
+ twelve providers, which is D21's own failure mode -- correct-looking
43
58
  output -- one layer up in the UI. The estimate also keeps MOVING during a
44
59
  table hold, because the deltas are still arriving; it is the renderer that
45
60
  is withholding, not the stream. That is the whole point of the feature, so
@@ -121,6 +136,13 @@ class TurnMeter:
121
136
  # how much blocked time this turn has already accumulated.
122
137
  self._blocked_at = None
123
138
  self._blocked_total = 0.0
139
+ # The tool pause, same shape as the modal one, plus a DEPTH --
140
+ # `_dispatch_parallel` can have several tools outstanding at once,
141
+ # so a boolean would un-pause on the first result while two were
142
+ # still running.
143
+ self._tool_at = None
144
+ self._tool_total = 0.0
145
+ self._tool_depth = 0
124
146
  self._chars = 0
125
147
 
126
148
  # -- session ------------------------------------------------------------
@@ -151,6 +173,9 @@ class TurnMeter:
151
173
  self._turn_at = now
152
174
  self._blocked_at = None
153
175
  self._blocked_total = 0.0
176
+ self._tool_at = None
177
+ self._tool_total = 0.0
178
+ self._tool_depth = 0
154
179
  self._chars = 0
155
180
 
156
181
  def stop(self, now: float):
@@ -168,6 +193,8 @@ class TurnMeter:
168
193
  elapsed = self.elapsed(now)
169
194
  self._turn_at = None
170
195
  self._blocked_at = None
196
+ self._tool_at = None
197
+ self._tool_depth = 0
171
198
  return elapsed
172
199
 
173
200
  def set_blocked(self, blocked: bool, now: float) -> None:
@@ -185,18 +212,68 @@ class TurnMeter:
185
212
  if blocked:
186
213
  if self._blocked_at is None:
187
214
  self._blocked_at = now
215
+ # The two pauses MUST NOT OVERLAP, and this is the half
216
+ # that keeps them apart. Approval is asked for from inside
217
+ # a tool call, so a human wait sits inside a tool window;
218
+ # `elapsed` already removes the human wait, so letting the
219
+ # tool clock keep running across it would subtract the same
220
+ # seconds twice and could drive the rate's denominator
221
+ # negative. The tool clock is closed here and reopened when
222
+ # the modal goes away, which is `set_blocked(False)` below.
223
+ self._close_tool_window(now)
188
224
  return
189
225
  if self._blocked_at is not None:
190
226
  self._blocked_total += max(0.0, now - self._blocked_at)
191
227
  self._blocked_at = None
228
+ if self._tool_depth > 0:
229
+ self._tool_at = now
230
+
231
+ def set_tool_running(self, delta: int, now: float) -> None:
232
+ """A tool started (+1) or finished (-1).
233
+
234
+ DEPTH, NOT A BOOLEAN: `core/loop.py`'s `_dispatch_parallel` runs a
235
+ response's tool calls together, so several `tool_call_start` events
236
+ can be outstanding before the first `tool_result` arrives. The clock
237
+ is paused while the depth is above zero.
238
+
239
+ This leaves `elapsed` alone, deliberately. A tool taking two minutes
240
+ IS part of how long the turn took, and the transcript line says
241
+ `took`. It is not part of how fast the model writes, which is what
242
+ the rate claims to be -- so only the rate subtracts it.
243
+ """
244
+ if self._turn_at is None:
245
+ return
246
+ previous = self._tool_depth
247
+ self._tool_depth = max(0, previous + int(delta))
248
+ if self._tool_depth and previous == 0:
249
+ # Not while a modal is up: see set_blocked.
250
+ if self._blocked_at is None:
251
+ self._tool_at = now
252
+ elif not self._tool_depth and previous:
253
+ self._close_tool_window(now)
254
+
255
+ def _close_tool_window(self, now: float) -> None:
256
+ """Bank an open tool window, if there is one. Idempotent."""
257
+ if self._tool_at is not None:
258
+ self._tool_total += max(0.0, now - self._tool_at)
259
+ self._tool_at = None
192
260
 
193
261
  def add_output_chars(self, count: int) -> None:
194
- """Characters of answer received. The live rate's numerator.
262
+ """Characters the MODEL GENERATED. The live rate's numerator.
195
263
 
196
264
  Called per token delta, INCLUDING the ones being withheld by the
197
265
  commit cap -- which is the point. The deltas arrive whether or not
198
- the renderer is drawing them, so this is what keeps a figure
199
- moving through a table hold.
266
+ the renderer is drawing them, so this is what keeps a figure moving
267
+ through a table hold.
268
+
269
+ AND INCLUDING THINKING, AND TOOL-CALL ARGUMENTS (batch 118). Both
270
+ are output the provider counts and bills, and neither reached here
271
+ before: thinking arrives as its own `LoopEvent` field, and tool-call
272
+ arguments never stream as a delta at all -- `StreamToken` carries
273
+ text, thinking, or the final response, so the arguments land whole
274
+ in the last one. Counting only the visible answer made the live
275
+ figure read `~0 tok/s` on a turn that was thinking and calling tools,
276
+ which is the turn it was built to describe.
200
277
  """
201
278
  if self._turn_at is not None:
202
279
  self._chars += max(0, int(count or 0))
@@ -218,11 +295,30 @@ class TurnMeter:
218
295
  blocked += max(0.0, now - self._blocked_at)
219
296
  return max(0.0, (now - self._turn_at) - blocked)
220
297
 
298
+ def generating(self, now: float) -> float:
299
+ """Active seconds with a tool NOT running: the rate's denominator.
300
+
301
+ `elapsed` minus tool time. See the module docstring for why these
302
+ are two numbers rather than one -- briefly, a tool running is part
303
+ of how long the turn took and no part of how fast the model writes.
304
+
305
+ Clamped, and the clamp is doing real work rather than being
306
+ defensive: the two pauses are kept disjoint by `set_blocked`, and
307
+ if that ever stopped being true this would go negative rather than
308
+ merely wrong.
309
+ """
310
+ if self._turn_at is None:
311
+ return 0.0
312
+ tool = self._tool_total
313
+ if self._tool_at is not None:
314
+ tool += max(0.0, now - self._tool_at)
315
+ return max(0.0, self.elapsed(now) - tool)
316
+
221
317
  def rate(self, now: float):
222
318
  """Estimated tokens per second, or None when it would be noise."""
223
319
  if self._turn_at is None or not self._chars:
224
320
  return None
225
- active = self.elapsed(now)
321
+ active = self.generating(now)
226
322
  if active < RATE_FLOOR_S:
227
323
  return None
228
324
  return (self._chars / CHARS_PER_TOKEN) / active
@@ -253,7 +349,7 @@ class TurnMeter:
253
349
  `output_tokens` is the provider's own count for the turn, or None
254
350
  where the provider reports none. NO TOKEN CLAIM is made in that
255
351
  case -- not a zero, which would say the model wrote nothing on
256
- sixteen of the nineteen configured providers.
352
+ twelve of the fifteen configured providers.
257
353
 
258
354
  The rate here carries no tilde. It is not the same instrument as
259
355
  the live figure: that one divides characters by a constant, this