@yahyahammad/venastine 0.1.0 → 0.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/.env.example +19 -14
  2. package/CONFIG_ARCHITECTURE.md +2002 -0
  3. package/NOTICE +1 -1
  4. package/README.md +249 -93
  5. package/agents/builtin/build.md +73 -0
  6. package/agents/builtin/general.md +80 -0
  7. package/agents/builtin/plan.md +23 -5
  8. package/agents/builtin/test.md +65 -0
  9. package/agents/builtin/writer.md +67 -0
  10. package/agents/manager.py +261 -195
  11. package/agents/subagent_tool.py +280 -23
  12. package/agents/tui_commands.py +31 -21
  13. package/bin/venastine.mjs +528 -382
  14. package/bin/venastine.py +71 -0
  15. package/cli_commands.py +835 -0
  16. package/config.py +95 -1198
  17. package/config.yaml +1090 -0
  18. package/config_edit.py +1301 -0
  19. package/config_schema.py +966 -0
  20. package/config_update.py +749 -0
  21. package/core/agent_activity.py +256 -0
  22. package/core/ansi.py +452 -0
  23. package/core/approval.py +27 -5
  24. package/core/call_view.py +364 -0
  25. package/core/cancel.py +121 -0
  26. package/core/chatgpt_auth.py +747 -0
  27. package/core/claims_view.py +107 -0
  28. package/core/client.py +1249 -82
  29. package/core/commands.py +454 -0
  30. package/core/compaction.py +1735 -959
  31. package/core/config_loader.py +516 -97
  32. package/core/connect.py +686 -0
  33. package/core/events.py +68 -6
  34. package/core/file_mentions.py +280 -0
  35. package/core/interaction.py +87 -4
  36. package/core/line_pattern.py +128 -0
  37. package/core/loop.py +2566 -1353
  38. package/core/memory.py +482 -370
  39. package/core/pipeline_models.py +23 -9
  40. package/core/provider_errors.py +234 -0
  41. package/core/reasoning/confidence_scoring.py +70 -54
  42. package/core/reasoning/json_retry.py +19 -1
  43. package/core/reasoning/orchestrator.py +232 -53
  44. package/core/reasoning/output_writer.py +90 -31
  45. package/core/reasoning/payload_validation.py +133 -7
  46. package/core/reasoning/pipeline_storage.py +427 -361
  47. package/core/reasoning/review.py +659 -615
  48. package/core/reasoning/scholar.py +38 -14
  49. package/core/reasoning/source_corpus.py +110 -4
  50. package/core/replay.py +315 -167
  51. package/core/session.py +36 -12
  52. package/core/session_view.py +259 -0
  53. package/core/session_wake.py +259 -0
  54. package/core/shell_sessions.py +1553 -0
  55. package/core/workspace_trust.py +1 -1
  56. package/credentials.py +308 -180
  57. package/database.py +82 -1
  58. package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
  59. package/docs/SECURITY.md +108 -0
  60. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  61. package/env_secrets.py +118 -77
  62. package/json_store.py +236 -16
  63. package/logging_setup.py +121 -11
  64. package/main.py +1672 -198
  65. package/mcp_client/client.py +257 -23
  66. package/mcp_client/config.py +484 -379
  67. package/mcp_client/registration.py +163 -2
  68. package/memories/manager.py +15 -0
  69. package/memories/tui_commands.py +21 -14
  70. package/package.json +303 -73
  71. package/project_init/config_files.py +4 -0
  72. package/project_init/generator.py +201 -37
  73. package/project_init/manifest.py +418 -406
  74. package/project_init/tui_commands.py +7 -4
  75. package/prompts/revalidate.md +1 -1
  76. package/prompts/revise.md +17 -15
  77. package/prompts/system_prompts.py +64 -10
  78. package/providers.json.example +94 -92
  79. package/pyproject.toml +149 -100
  80. package/requirements.txt +252 -29
  81. package/safety/policy_enforcement.py +1082 -682
  82. package/security/askpass.py +419 -0
  83. package/security/capability.py +37 -7
  84. package/security/confine.py +484 -0
  85. package/security/hosted.py +182 -0
  86. package/security/permissions.py +54 -7
  87. package/security/posture.py +124 -12
  88. package/security/protected_paths.py +198 -6
  89. package/security/sandbox.py +3262 -139
  90. package/security/secrets.py +855 -0
  91. package/skills/tui_commands.py +3 -4
  92. package/storage.py +1338 -1005
  93. package/tools/base.py +465 -305
  94. package/tools/builtin/_math_common.py +306 -16
  95. package/tools/builtin/_net_common.py +267 -61
  96. package/tools/builtin/arxiv.py +681 -223
  97. package/tools/builtin/ask_user.py +231 -213
  98. package/tools/builtin/discrete_math.py +35 -4
  99. package/tools/builtin/fetch_url.py +239 -21
  100. package/tools/builtin/file_ops.py +341 -33
  101. package/tools/builtin/geometry.py +184 -148
  102. package/tools/builtin/get_time.py +2 -2
  103. package/tools/builtin/linear_algebra.py +66 -23
  104. package/tools/builtin/load_skill.py +10 -0
  105. package/tools/builtin/logic.py +171 -12
  106. package/tools/builtin/pin.py +46 -12
  107. package/tools/builtin/probability_stats.py +224 -228
  108. package/tools/builtin/project_docs.py +80 -21
  109. package/tools/builtin/remember.py +21 -4
  110. package/tools/builtin/shell.py +586 -67
  111. package/tools/builtin/shell_sessions.py +613 -0
  112. package/tools/builtin/symbolic_math.py +80 -10
  113. package/tools/builtin/todo.py +5 -3
  114. package/tools/builtin/web_search.py +329 -151
  115. package/tools/context.py +23 -0
  116. package/tools/isolation.py +387 -314
  117. package/tools/registry.py +374 -29
  118. package/tui/app.py +5281 -397
  119. package/tui/app.tcss +290 -9
  120. package/tui/commands.py +26 -61
  121. package/tui/diffs.py +286 -265
  122. package/tui/history.py +128 -0
  123. package/tui/hosted_driver.py +102 -0
  124. package/tui/markdown.py +1329 -0
  125. package/tui/meters.py +364 -0
  126. package/tui/screens.py +346 -92
  127. package/tui/themes.py +599 -366
  128. package/tui/widgets.py +2673 -116
  129. package/SECURITY.md +0 -88
  130. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -28,12 +28,14 @@ from __future__ import annotations
28
28
  import difflib
29
29
  import logging
30
30
  import os
31
- from typing import Callable, Optional
31
+ from collections.abc import Callable
32
+ from typing import Optional
32
33
 
33
34
  import config
34
35
  from core import config_loader, workspace_trust
35
- from project_init import (config_files, doc_sets,
36
- manifest as manifest_mod)
36
+ from json_store import read_regular_text
37
+ from project_init import config_files, doc_sets
38
+ from project_init import manifest as manifest_mod
37
39
 
38
40
  logger = logging.getLogger(__name__)
39
41
 
@@ -168,13 +170,29 @@ def _shadow_lines() -> list:
168
170
 
169
171
 
170
172
  def _read_existing_context(project_path: str) -> Optional[str]:
173
+ """The current hub, or None to generate a fresh one.
174
+
175
+ BOUNDED SINCE BATCH 221 (#226). It was read, diffed, embedded in the
176
+ model's task and printed whole: measured, a 5.4 MB hub went into a
177
+ 5.4 MB task and a 5.4 MB diff. Head-plus-a-notice was the other shape,
178
+ and it is wrong here: the initializer's reply BECOMES the whole hub, so
179
+ a model shown the head would rewrite the file without its tail. So a
180
+ hub past `init_max_hub_chars` is refused with nothing sent anywhere,
181
+ and the owner sized that at about three times this repository's own
182
+ 347 KB hub. The size is checked before the read, in bytes, where four
183
+ bytes per character makes the comparison safe; the count is checked
184
+ after it. And the read itself is regular-files-only (batch 219).
185
+ """
171
186
  from tools.builtin.project_docs import HUB_FILENAME, doc_path
172
187
  path = doc_path(project_path, HUB_FILENAME)
173
188
  if not os.path.exists(path):
174
189
  return None
190
+ cap = config.INIT_MAX_HUB_CHARS
175
191
  try:
176
- with open(path, "r", encoding="utf-8-sig") as f:
177
- return f.read()
192
+ if os.path.getsize(path) > 4 * cap:
193
+ raise InitError(_too_large(HUB_FILENAME, os.path.getsize(path),
194
+ "bytes", cap))
195
+ text = read_regular_text(path)
178
196
  except (OSError, UnicodeDecodeError) as e:
179
197
  # Degrade like config_loader does with an unreadable hub document.
180
198
  # An undecodable existing file must not stop /init from writing a
@@ -183,6 +201,31 @@ def _read_existing_context(project_path: str) -> Optional[str]:
183
201
  logger.warning("Could not read the existing %s at %s; "
184
202
  "generating a fresh one: %s", HUB_FILENAME, path, e)
185
203
  return None
204
+ if len(text) > cap:
205
+ raise InitError(_too_large(HUB_FILENAME, len(text), "characters",
206
+ cap))
207
+ return text
208
+
209
+
210
+ def _too_large(name: str, size: int, unit: str, cap: int) -> str:
211
+ return (f"{name} is {size:,} {unit}, more than /init revises "
212
+ f"({cap:,} characters, init_max_hub_chars in config.yaml). "
213
+ f"Nothing has been changed: trim it, or edit it by hand.")
214
+
215
+
216
+ def _shown(diff: str) -> str:
217
+ """The diff as the shell shows it: at most `init_diff_shown_lines`
218
+ lines, and a line saying how many more there are (#226). The whole
219
+ diff still decides whether anything changed; only the print is cut,
220
+ because a megabyte-scale row freezes the transcript it is printed to."""
221
+ lines = diff.splitlines(keepends=True)
222
+ cap = config.INIT_DIFF_SHOWN_LINES
223
+ if len(lines) <= cap:
224
+ return diff
225
+ return ("".join(lines[:cap]).rstrip("\n")
226
+ + f"\n... {len(lines) - cap:,} more line(s) of the diff are not "
227
+ f"shown ({cap:,} of {len(lines):,}; init_diff_shown_lines in "
228
+ f"config.yaml).")
186
229
 
187
230
 
188
231
  def _task(manifest_text: str, existing: Optional[str], kind: str) -> str:
@@ -219,9 +262,10 @@ def _task(manifest_text: str, existing: Optional[str], kind: str) -> str:
219
262
 
220
263
 
221
264
  def _run_initializer(project_path: str, kind: str, existing: Optional[str],
222
- model: str, provider_name: str) -> str:
265
+ model: str, provider_name: str, activity=None) -> str:
223
266
  from agents.manager import manager
224
- from core.loop import RunAgentLoop, DEFAULT_SYSTEM_PROMPT
267
+ from core import agent_activity
268
+ from core.loop import DEFAULT_SYSTEM_PROMPT, RunAgentLoop, advertisement_facts
225
269
  from storage import THREAD_KIND_SUBAGENT
226
270
 
227
271
  agent = manager.get(config.INITIALIZER_AGENT)
@@ -231,23 +275,48 @@ def _run_initializer(project_path: str, kind: str, existing: Optional[str],
231
275
  f"there is nothing to generate the hub document with.")
232
276
 
233
277
  context = manager.active_context(agent)
278
+ # #228: the run below is given no authorization, so it is HEADLESS --
279
+ # nothing can answer an approval -- and the prompt says what THAT run
280
+ # can call, the reviewer's rule (core/reasoning/review.py). With
281
+ # system_prompt_for's attended defaults this was right only while
282
+ # initializer.md's allowed_tools kept both catalog tools out.
283
+ callable_only, granted = advertisement_facts()
234
284
  system_prompt = manager.system_prompt_for(
235
- agent, DEFAULT_SYSTEM_PROMPT, context=context)
236
-
237
- response = RunAgentLoop.run_agent_conversation(
238
- user_goal=_task(manifest_mod.build_manifest(project_path),
239
- existing, kind),
240
- model=agent.model or model,
241
- provider_name=agent.provider or provider_name,
242
- max_steps=agent.max_steps or config.INIT_MAX_STEPS,
243
- # #4: no separate /init ceiling any more -- the wrapper resolves the
244
- # configured spend cap, same as every other path.
245
- context=context,
246
- system_prompt=system_prompt,
247
- # §27: the initializer is a SIXTH thread source. Left unlabelled it
248
- # clutters the picker with a thread nobody will ever resume.
249
- thread_kind=THREAD_KIND_SUBAGENT,
250
- )
285
+ agent, DEFAULT_SYSTEM_PROMPT, context=context,
286
+ callable_only=callable_only, granted=granted)
287
+
288
+ # §47, the span batch 59 named as a follow-on and did not build. /init
289
+ # is a long agent-shaped run -- it reads the project and drafts a
290
+ # document -- and until now a shell watching the agent stack saw
291
+ # nothing at all while it happened.
292
+ #
293
+ # `context.subagent_depth + 1` rather than a literal 1, which is
294
+ # core/loop._compactor_depth's rule one command over: the number is
295
+ # derived from the depth the run actually starts at, so a second
296
+ # count cannot disagree with the first.
297
+ with agent_activity.span(activity, agent.name,
298
+ context.subagent_depth + 1):
299
+ response = RunAgentLoop.run_agent_conversation(
300
+ user_goal=_task(manifest_mod.build_manifest(project_path),
301
+ existing, kind),
302
+ model=agent.model or model,
303
+ provider_name=agent.provider or provider_name,
304
+ max_steps=agent.max_steps or config.INIT_MAX_STEPS,
305
+ # #4: no separate /init ceiling any more -- the wrapper
306
+ # resolves the configured spend cap, same as every other path.
307
+ context=context,
308
+ system_prompt=system_prompt,
309
+ # §27: the initializer is a SIXTH thread source. Left
310
+ # unlabelled it clutters the picker with a thread nobody
311
+ # will ever resume.
312
+ thread_kind=THREAD_KIND_SUBAGENT,
313
+ # §47. NO parent thread, deliberately: /init scaffolds a
314
+ # project, it is not a child of the conversation the command
315
+ # was typed in -- and `generate()` is shell-agnostic and has
316
+ # no conversation to name anyway. Reachable from the panel
317
+ # while it runs, which is what the span is for.
318
+ thread_agent=agent.name,
319
+ )
251
320
  body = (response.text or "").strip()
252
321
  if not body:
253
322
  raise InitError(
@@ -279,6 +348,7 @@ def generate(
279
348
  choose_kind: Optional[Callable] = None,
280
349
  scaffold_docs: bool = True,
281
350
  scaffold_config: bool = False,
351
+ activity=None,
282
352
  ) -> dict:
283
353
  """Scaffold this project's documentation set, its configuration, or both.
284
354
 
@@ -303,7 +373,7 @@ def generate(
303
373
  Returns a notice dict in the {"kind", "text"} shape both shells render.
304
374
  """
305
375
  from tools.builtin.project_docs import HUB_FILENAME, doc_path
306
- from tools.registry import registry, ToolCallDenied
376
+ from tools.registry import ToolCallDenied, registry
307
377
 
308
378
  root = project_path or config_loader.get_project_path()
309
379
  if root is None:
@@ -325,8 +395,19 @@ def generate(
325
395
  # than an addition to a run that always costs a model call.
326
396
  if scaffold_docs:
327
397
  if kind is None:
328
- blank = manifest_mod.is_blank(root)
329
- proposal, reason = (None, "") if blank else propose_kind(root)
398
+ # Both walk the project (#255, batch 213). An OSError here --
399
+ # a directory nobody may list, a path that stopped being one --
400
+ # left as itself, and the CLI catches InitError only, so it was
401
+ # a traceback there and an error line in the TUI.
402
+ try:
403
+ blank = manifest_mod.is_blank(root)
404
+ proposal, reason = ((None, "") if blank
405
+ else propose_kind(root))
406
+ except OSError as e:
407
+ raise InitError(
408
+ f"Could not read the project directory to work out its "
409
+ f"type ({e}). Re-run with --software or --research, or "
410
+ f"fix the directory's permissions.") from e
330
411
  if choose_kind is None:
331
412
  # No way to ask. Refusing beats scaffolding seven files of
332
413
  # the wrong kind into someone's project -- V6's rule, and
@@ -346,7 +427,7 @@ def generate(
346
427
  existing_context = _read_existing_context(root)
347
428
  say(f"Reading the project and drafting {HUB_FILENAME}…")
348
429
  body = _run_initializer(root, kind, existing_context, model,
349
- provider_name)
430
+ provider_name, activity)
350
431
 
351
432
  already_there = _existing_documents(root, kind)
352
433
  body = f"{body.rstrip()}\n\n{doc_sets.render_index(kind)}"
@@ -365,7 +446,7 @@ def generate(
365
446
 
366
447
  if scaffold_docs:
367
448
  if diff:
368
- say(diff)
449
+ say(_shown(diff))
369
450
  else:
370
451
  say(f"{HUB_FILENAME} is unchanged.")
371
452
  if to_create:
@@ -398,7 +479,9 @@ def generate(
398
479
  # Captured BEFORE the write (I6). Afterwards the content hash has moved
399
480
  # and is_trusted() answers False for a project that WAS trusted, which
400
481
  # is exactly the state this distinction exists to tell apart.
401
- was_trusted = workspace_trust.is_trusted(root)
482
+ #
483
+ # TOTAL SINCE BATCH 153, because the capture itself reads files (A47).
484
+ was_trusted, trust_problem = _trust_before(root)
402
485
 
403
486
  granted = lambda _name, _params: True # noqa: E731 -- consent is above
404
487
  written = []
@@ -446,7 +529,7 @@ def generate(
446
529
  # they just authored this content, and that is as true after five of
447
530
  # eight writes as after eight. Skipping this made the failure wrong
448
531
  # about two things instead of one, and silent about the second.
449
- trust_note = _settle_trust(root, was_trusted)
532
+ trust_note, trust_ok = _settle_trust(root, was_trusted, trust_problem)
450
533
 
451
534
  if failure is not None:
452
535
  # NAME WHAT WAS CREATED. The success path reports the files so the
@@ -472,8 +555,14 @@ def generate(
472
555
  lines.append(trust_note)
473
556
  raise InitError("\n".join(line for line in lines if line))
474
557
 
475
- lines = [f"Wrote {len(written)} file{'' if len(written) == 1 else 's'}: "
476
- + ", ".join(written)]
558
+ # THE WARNING LEADS, when there is one (A46). The files are on disk and
559
+ # the user can see them; what they cannot see is that the next launch
560
+ # will ask about this project again, so the half that did NOT happen is
561
+ # the half that goes first. A run whose trust settled as intended keeps
562
+ # the note where it has always been -- last, as the footnote it is.
563
+ lines = [] if trust_ok else [trust_note]
564
+ lines.append(f"Wrote {len(written)} file"
565
+ f"{'' if len(written) == 1 else 's'}: " + ", ".join(written))
477
566
  if made:
478
567
  lines.append("Created: " + ", ".join(made))
479
568
  if already_there:
@@ -482,7 +571,8 @@ def generate(
482
571
  lines.append("Left alone: "
483
572
  + ", ".join(".venastine/" + name
484
573
  for name in plan["existing"]))
485
- lines.append(trust_note)
574
+ if trust_ok:
575
+ lines.append(trust_note)
486
576
  return {"kind": "init", "text": "\n".join(line for line in lines if line)}
487
577
 
488
578
 
@@ -540,7 +630,51 @@ def _write(registry, granted, name: str, content: str) -> None:
540
630
  raise InitError(result["error"])
541
631
 
542
632
 
543
- def _settle_trust(project_path: str, was_trusted: bool) -> str:
633
+ def _trust_before(project_path: str) -> tuple:
634
+ """`is_trusted()`, made total. Returns (was_trusted, problem).
635
+
636
+ THE CAPTURE READS FILES, which is the half of I6 nobody was looking at
637
+ (batch 153). `is_trusted` hashes every path `content_files()`
638
+ lists -- the root `AGENTS.md` and everything under `.venastine/` -- and
639
+ `open` on any one of them can fail: a file an editor holds, a sync
640
+ client mid-write, a mode the user changed by hand. That read sits
641
+ before the `try`, so a single unreadable file killed `/init` as a raw
642
+ `PermissionError` before it wrote anything, and on the CLI
643
+ (`except InitError` only) as a traceback.
644
+
645
+ DEGRADES, and the precedent is three functions up in this same module:
646
+ `_read_existing_context` catches `OSError` on the SAME file for the
647
+ SAME reason and says "Degrade like config_loader does with an
648
+ unreadable hub document". One function answering one error two ways is
649
+ what this closes (A47).
650
+
651
+ `False` rather than `True` for an unanswerable question, which is
652
+ `core/workspace_trust.py`'s own posture -- `_load_trust_store` returns
653
+ `{}` for anything unreadable and says it fails CLOSED in both senses.
654
+ Read as untrusted, `_settle_trust` attempts no re-grant, and the
655
+ problem is REPORTED rather than inferred from a trust prompt arriving
656
+ on the next launch.
657
+
658
+ NOT fixed in `is_trusted` itself, deliberately (A48). Its other callers
659
+ are contained -- `main.load_project_config` catches `OSError` and exits
660
+ 1 with one clean line -- and making a security control answer "not
661
+ trusted" where it currently refuses to start is a change to D17's
662
+ posture, not to this command's. `docs/TECHNICAL_DEBT.md` carries it.
663
+ """
664
+ try:
665
+ return workspace_trust.is_trusted(project_path), ""
666
+ except OSError as e:
667
+ logger.warning("Could not read the trust state of %s; proceeding as "
668
+ "untrusted: %s", project_path, e)
669
+ return False, (
670
+ f"This project's trust state could not be read ({e}), so the run "
671
+ f"proceeded as if it were untrusted and no re-grant was "
672
+ f"attempted. You will be asked about its .venastine/ content on "
673
+ f"the next launch.")
674
+
675
+
676
+ def _settle_trust(project_path: str, was_trusted: bool,
677
+ problem: str = "") -> tuple:
544
678
  """I6. Re-grant only for a project that was ALREADY trusted.
545
679
 
546
680
  Writing the hub changes the D17 content hash, so a project the user
@@ -554,11 +688,41 @@ def _settle_trust(project_path: str, was_trusted: bool) -> str:
554
688
  a hole in D17 opened from a direction the trust prompt never sees.
555
689
  The hub is never special-cased out of the hash either; that would be
556
690
  the same hole with a smaller mouth.
691
+
692
+ TOTAL SINCE BATCH 153. Returns (note, settled_as_intended),
693
+ and the note is the whole of what the user is told about trust -- so
694
+ this function raising discarded it, along with everything `generate`
695
+ had assembled around it. #95 put the settle on both paths; the settle
696
+ then sat outside every handler on both of them, so a trust-store I/O
697
+ failure took the success path's notice and the failure path's "wrote N
698
+ files before that" accounting down with it. Measured: both config
699
+ files on disk, `is_trusted` True -> False, and a raw `OSError` out.
700
+
701
+ `grant_trust` raises from either half -- `_content_hash` re-reads the
702
+ files just written, `_save_trust_store` writes the store -- and
703
+ `OSError` is both. `json_store` argues that a trust grant which
704
+ silently did not persist is worse than a crash, and that argument is
705
+ about SILENCE: the grant still either persists or is named, and the
706
+ naming now reaches the user instead of the traceback handler.
707
+
708
+ `False` is returned for the grant failure and for an unreadable
709
+ pre-write state, and `True` for a project that was untrusted and
710
+ stayed that way -- nothing went wrong there, and that note is a
711
+ footnote rather than a warning.
557
712
  """
713
+ if problem:
714
+ return problem, False
558
715
  if not was_trusted:
559
716
  return ("This project is not trusted, so nothing here has changed "
560
717
  "that. You will be asked about its .venastine/ content on "
561
- "the next launch.")
562
- workspace_trust.grant_trust(project_path)
718
+ "the next launch."), True
719
+ try:
720
+ workspace_trust.grant_trust(project_path)
721
+ except OSError as e:
722
+ logger.warning("Could not re-grant workspace trust for %s: %s",
723
+ project_path, e)
724
+ return (f"Workspace trust could NOT be re-granted ({e}), so the next "
725
+ f"launch will ask about this project's .venastine/ content "
726
+ f"again."), False
563
727
  return ("Workspace trust re-granted for the updated .venastine/ content, "
564
- "so the next launch will not re-ask.")
728
+ "so the next launch will not re-ask."), True