@yahyahammad/venastine 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/.env.example +14 -0
  2. package/LICENSE +201 -0
  3. package/NOTICE +14 -0
  4. package/PRIVACY.md +66 -0
  5. package/README.md +878 -0
  6. package/SECURITY.md +88 -0
  7. package/THIRD_PARTY_NOTICES.md +100 -0
  8. package/agents/builtin/compactor.md +75 -0
  9. package/agents/builtin/explore.md +77 -0
  10. package/agents/builtin/grill-me.md +27 -0
  11. package/agents/builtin/initializer.md +94 -0
  12. package/agents/builtin/pipeline-reviewer.md +100 -0
  13. package/agents/builtin/plan.md +88 -0
  14. package/agents/builtin/review.md +74 -0
  15. package/agents/manager.py +195 -0
  16. package/agents/subagent_tool.py +215 -0
  17. package/agents/tui_commands.py +136 -0
  18. package/bin/venastine.mjs +382 -0
  19. package/config.py +1198 -0
  20. package/core/__init__.py +0 -0
  21. package/core/approval.py +115 -0
  22. package/core/client.py +1548 -0
  23. package/core/compaction.py +959 -0
  24. package/core/config_loader.py +1563 -0
  25. package/core/events.py +61 -0
  26. package/core/interaction.py +328 -0
  27. package/core/loop.py +1353 -0
  28. package/core/memory.py +370 -0
  29. package/core/model_windows.py +175 -0
  30. package/core/pipeline_models.py +176 -0
  31. package/core/reasoning/__init__.py +0 -0
  32. package/core/reasoning/authorization.py +130 -0
  33. package/core/reasoning/base.py +219 -0
  34. package/core/reasoning/confidence_scoring.py +468 -0
  35. package/core/reasoning/events.py +149 -0
  36. package/core/reasoning/json_retry.py +219 -0
  37. package/core/reasoning/orchestrator.py +1382 -0
  38. package/core/reasoning/output_writer.py +286 -0
  39. package/core/reasoning/payload_validation.py +398 -0
  40. package/core/reasoning/pipeline_storage.py +361 -0
  41. package/core/reasoning/review.py +615 -0
  42. package/core/reasoning/scholar.py +434 -0
  43. package/core/reasoning/source_corpus.py +327 -0
  44. package/core/reasoning/source_scoring.py +855 -0
  45. package/core/replay.py +167 -0
  46. package/core/session.py +153 -0
  47. package/core/workspace_trust.py +207 -0
  48. package/credentials.py +180 -0
  49. package/database.py +252 -0
  50. package/env_secrets.py +77 -0
  51. package/json_store.py +123 -0
  52. package/logging_setup.py +185 -0
  53. package/main.py +1959 -0
  54. package/mcp_client/__init__.py +15 -0
  55. package/mcp_client/client.py +499 -0
  56. package/mcp_client/config.py +379 -0
  57. package/mcp_client/registration.py +182 -0
  58. package/memories/manager.py +132 -0
  59. package/memories/tui_commands.py +232 -0
  60. package/package.json +73 -0
  61. package/project_init/config_files.py +278 -0
  62. package/project_init/doc_sets.py +330 -0
  63. package/project_init/generator.py +564 -0
  64. package/project_init/manifest.py +406 -0
  65. package/project_init/tui_commands.py +144 -0
  66. package/prompts/__init__.py +0 -0
  67. package/prompts/annotate.md +14 -0
  68. package/prompts/assumption_audit.md +27 -0
  69. package/prompts/claim_extraction.md +34 -0
  70. package/prompts/completeness.md +18 -0
  71. package/prompts/confidence_tiers.md +23 -0
  72. package/prompts/critic_pass.md +24 -0
  73. package/prompts/final_synthesis.md +13 -0
  74. package/prompts/initial_generation.md +14 -0
  75. package/prompts/preliminary_plan.md +23 -0
  76. package/prompts/revalidate.md +25 -0
  77. package/prompts/revise.md +15 -0
  78. package/prompts/source_grounding.md +66 -0
  79. package/prompts/system_prompts.py +262 -0
  80. package/prompts/universal_system_prompt +7 -0
  81. package/prompts/untrusted_content +1 -0
  82. package/providers.json.example +92 -0
  83. package/pyproject.toml +100 -0
  84. package/requirements.txt +72 -0
  85. package/safety/__init__.py +0 -0
  86. package/safety/policy_enforcement.py +682 -0
  87. package/security/__init__.py +0 -0
  88. package/security/capability.py +177 -0
  89. package/security/permissions.py +190 -0
  90. package/security/posture.py +302 -0
  91. package/security/protected_paths.py +309 -0
  92. package/security/sandbox.py +985 -0
  93. package/skills/builtin/crypto/cryptography-verification.md +66 -0
  94. package/skills/builtin/math/formal-specification.md +67 -0
  95. package/skills/builtin/math/numerical-methods.md +66 -0
  96. package/skills/builtin/math/proof-writing.md +61 -0
  97. package/skills/builtin/research/experiment-design.md +62 -0
  98. package/skills/builtin/research/literature-review.md +65 -0
  99. package/skills/builtin/research/reproducibility.md +55 -0
  100. package/skills/builtin/research/source-evaluation.md +70 -0
  101. package/skills/builtin/research/statistical-inference.md +63 -0
  102. package/skills/builtin/research/technical-writing.md +59 -0
  103. package/skills/builtin/security/cybersecurity-research.md +63 -0
  104. package/skills/builtin/software/code-review.md +61 -0
  105. package/skills/builtin/software/debugging.md +61 -0
  106. package/skills/builtin/software/test-design.md +67 -0
  107. package/skills/manager.py +138 -0
  108. package/skills/tui_commands.py +111 -0
  109. package/storage.py +1005 -0
  110. package/tools/base.py +305 -0
  111. package/tools/builtin/__init__.py +0 -0
  112. package/tools/builtin/_math_common.py +377 -0
  113. package/tools/builtin/_net_common.py +61 -0
  114. package/tools/builtin/arxiv.py +223 -0
  115. package/tools/builtin/ask_user.py +213 -0
  116. package/tools/builtin/discrete_math.py +134 -0
  117. package/tools/builtin/fetch_url.py +153 -0
  118. package/tools/builtin/file_ops.py +412 -0
  119. package/tools/builtin/geometry.py +148 -0
  120. package/tools/builtin/get_time.py +20 -0
  121. package/tools/builtin/linear_algebra.py +155 -0
  122. package/tools/builtin/load_skill.py +66 -0
  123. package/tools/builtin/logic.py +96 -0
  124. package/tools/builtin/pin.py +193 -0
  125. package/tools/builtin/probability_stats.py +228 -0
  126. package/tools/builtin/project_docs.py +405 -0
  127. package/tools/builtin/remember.py +158 -0
  128. package/tools/builtin/shell.py +461 -0
  129. package/tools/builtin/symbolic_math.py +119 -0
  130. package/tools/builtin/todo.py +147 -0
  131. package/tools/builtin/web_search.py +151 -0
  132. package/tools/context.py +136 -0
  133. package/tools/isolation.py +314 -0
  134. package/tools/registry.py +897 -0
  135. package/tui/app.py +3423 -0
  136. package/tui/app.tcss +458 -0
  137. package/tui/commands.py +61 -0
  138. package/tui/diffs.py +265 -0
  139. package/tui/preferences.py +182 -0
  140. package/tui/ravens.py +146 -0
  141. package/tui/screens.py +943 -0
  142. package/tui/themes.py +366 -0
  143. package/tui/widgets.py +1239 -0
@@ -0,0 +1,219 @@
1
+ """
2
+ core/reasoning/json_retry.py
3
+
4
+ ROADMAP §3's malformed-JSON recovery, extracted from orchestrator.py so
5
+ that §20's reviewer gets it too.
6
+
7
+ The mechanism: when a model that was asked for JSON returns something
8
+ that does not parse, send a corrective follow-up into the SAME thread it
9
+ just used. The failed assistant turn is already in that thread's history
10
+ (core/loop.py's D20 always-persist behaviour), so the model sees its own
11
+ prior output rather than being asked again from a clean slate. That is
12
+ the whole reason this is a thread continuation and not a fresh call.
13
+
14
+ WHY THIS IS ITS OWN MODULE. The orchestrator's version hardcoded
15
+ stream_deep_research_mode as the first attempt and pass_prompt(pass_id) as
16
+ the system prompt -- correct for the ten passes, wrong for anything else.
17
+ §20's reviewer is agent-shaped: it starts through
18
+ run_agent_conversation with an agent's own system prompt. Copying twenty
19
+ lines of retry loop into review.py would put two copies of the corrective
20
+ wording, the attempt arithmetic and the trace format in one codebase,
21
+ and they would drift. So the loop takes an ALREADY-OBTAINED response and
22
+ the parameters needed to continue its thread; each caller starts its own
23
+ first attempt however it likes.
24
+
25
+ ROADMAP_v2 §30 (B2) added the SECOND thing worth correcting. `validate`
26
+ is an optional callable applied to the parsed payload; if it raises
27
+ PayloadShapeError the loop treats that exactly like a parse failure --
28
+ same thread, same attempt arithmetic -- with one difference that has to
29
+ be there: the corrective WORDING branches. "Your last response did not
30
+ parse as valid JSON" is false about a payload that parsed perfectly and
31
+ was the wrong shape, and telling a model its JSON is malformed when it
32
+ is not is the kind of correction that produces a second wrong answer.
33
+
34
+ `validate=None` is the default and keeps this function byte-identical
35
+ for §20's reviewer, which does its own value-level filtering in
36
+ review.py:_validated for reasons that module states.
37
+
38
+ `attempts = max_retries + 1` (one initial plus max_retries corrective).
39
+ Unrecoverable failure raises ValueError with the final parse error
40
+ attached -- the orchestrator's try/except turns that into a durable
41
+ status='failed' record.
42
+ """
43
+
44
+ from __future__ import annotations
45
+
46
+ import json
47
+ from typing import Any, Callable, Optional
48
+
49
+ import config
50
+ from core.reasoning.payload_validation import PayloadShapeError
51
+
52
+ MAX_JSON_RETRIES = config.MAX_JSON_RETRIES
53
+
54
+
55
+ def parse_json_response(text: str) -> Any:
56
+ """Callers are instructed (via the universal preamble, or an agent's
57
+ own output-format section) to emit clean JSON, optionally in a code
58
+ fence. Strip the fence if present."""
59
+ cleaned = text.strip()
60
+ if cleaned.startswith("```"):
61
+ cleaned = cleaned.split("```")[1]
62
+ if cleaned.lower().startswith("json"):
63
+ cleaned = cleaned[4:]
64
+ cleaned = cleaned.strip()
65
+ try:
66
+ return json.loads(cleaned)
67
+ except json.JSONDecodeError as e:
68
+ # Caller-neutral wording: this module also serves §20's reviewer,
69
+ # which is not a pass (review §19-20 r5-1).
70
+ raise ValueError(f"Response did not return valid JSON: {e}\n\nRaw text:\n{text[:500]}") from e
71
+
72
+
73
+ def retry_until_json(
74
+ response,
75
+ label: str,
76
+ system_prompt: str,
77
+ model: str,
78
+ provider_name: str,
79
+ trace: Optional[list] = None,
80
+ temperature: Optional[float] = None,
81
+ authorization=None,
82
+ max_retries: int = MAX_JSON_RETRIES,
83
+ on_response: Optional[Callable] = None,
84
+ context=None,
85
+ max_total_tokens: Optional[int] = None,
86
+ validate: Optional[Callable] = None,
87
+ effort: Optional[str] = None,
88
+ ) -> str:
89
+ """Returns the first response text that parses as JSON, retrying by
90
+ continuing `response`'s thread.
91
+
92
+ response an already-obtained ModelResponse. Its thread_id is what
93
+ the corrective message is sent into, so the caller's own
94
+ first attempt is what gets corrected.
95
+ label what the trace line names -- a pass id, or "Review".
96
+ on_response called with every response this function obtains. The
97
+ orchestrator hooks §25's granted-call recorder here: a
98
+ retry is the same pass continuing, so its tool calls
99
+ belong on the same audit trail, and a retry that spent a
100
+ grant invisibly is exactly the record §25 exists to
101
+ keep.
102
+ context the ToolContext the caller's first attempt ran under.
103
+ A retry is the SAME run continuing, so it must carry
104
+ the same tool restriction -- defaulting to None would
105
+ re-advertise globally-allowed tools an agent-shaped
106
+ caller excluded on turn 1 (review §19-20 r2-1).
107
+ max_total_tokens
108
+ the caller's own spend ceiling, for exactly the same
109
+ reason. #4: there is no separate pass constant any more
110
+ -- a caller forwarding its ceiling keeps it; None omits
111
+ the kwarg, so the wrapper resolves the configured
112
+ settings.json max_token_budget (uncapped unless set),
113
+ which is right for §20's agent-shaped reviewer too.
114
+ A budget stop returns the last response as it stands,
115
+ which is how an empty pass gets produced in the first
116
+ place -- hence carrying the caller's figure at all.
117
+ effort batch 25 (#139): the caller's effort level. A retry is
118
+ the same pass continuing, so it carries the same level;
119
+ effort_for() re-validates against this thread's model.
120
+ validate §30 (B2). Applied to the PARSED payload; raising
121
+ PayloadShapeError re-enters this same loop. None means
122
+ "parse is the only contract", which is what the reviewer
123
+ wants and what every caller had before §30.
124
+ """
125
+ from core.loop import RunAgentLoop
126
+
127
+ raw_text = response.text
128
+
129
+ for attempt in range(max_retries):
130
+ try:
131
+ _check(raw_text, validate)
132
+ return raw_text
133
+ except ValueError as e:
134
+ shape = isinstance(e, PayloadShapeError)
135
+ if trace is not None:
136
+ # The parse wording is unchanged to the byte. A reader
137
+ # scanning a trace for "JSON parse failed" is scanning for
138
+ # a model that cannot emit JSON, and a shape rejection is
139
+ # a different fact about a different failure.
140
+ trace.append(
141
+ f"{label}: "
142
+ f"{'payload shape rejected' if shape else 'JSON parse failed'}"
143
+ f" on attempt {attempt + 1}/"
144
+ f"{max_retries + 1} ({str(e).splitlines()[0]}), "
145
+ f"retrying with corrective message."
146
+ )
147
+ corrective = (
148
+ _shape_corrective(e, raw_text) if shape
149
+ else _parse_corrective(e, raw_text)
150
+ )
151
+ retry_response = RunAgentLoop.continue_conversation(
152
+ thread_id=response.thread_id,
153
+ message=corrective,
154
+ system_prompt=system_prompt,
155
+ model=model,
156
+ provider_name=provider_name,
157
+ temperature=temperature,
158
+ # A retry is the SAME work continuing. Omitting this would
159
+ # strip the gated tools on exactly the attempt where the
160
+ # model is already struggling, and the only symptom would
161
+ # be a second malformed answer.
162
+ authorization=authorization,
163
+ context=context,
164
+ # Same rule, batch 25 (#139): a retry is the same pass
165
+ # continuing, so it carries the pass's effort too --
166
+ # effort_for re-validates against this thread's model.
167
+ effort=effort,
168
+ **({} if max_total_tokens is None
169
+ else {"max_total_tokens": max_total_tokens}),
170
+ )
171
+ if on_response is not None:
172
+ on_response(retry_response)
173
+ raw_text = retry_response.text
174
+
175
+ # Last attempt: validate one more time and either return or raise.
176
+ _check(raw_text, validate)
177
+ return raw_text
178
+
179
+
180
+ def _check(raw_text: str, validate: Optional[Callable]) -> None:
181
+ """Both contracts, in the order they can be checked: a payload has to
182
+ parse before it can have a shape."""
183
+ payload = parse_json_response(raw_text)
184
+ if validate is not None:
185
+ validate(payload)
186
+
187
+
188
+ def _parse_corrective(error: ValueError, raw_text: str) -> str:
189
+ return (
190
+ f"Your last response did not parse as valid JSON.\n\n"
191
+ f"Parse error:\n{error}\n\n"
192
+ f"Start of your failed output (first 200 chars):\n{raw_text[:200]}\n\n"
193
+ # Caller-neutral: "the JSON your original instructions asked
194
+ # for" is true for a pass (object) and for §20's reviewer
195
+ # (array) alike. Naming a shape the caller never promised
196
+ # could talk the model out of its own output contract.
197
+ f"Respond again with ONLY valid JSON -- no preamble, no prose, "
198
+ f"no code fence, just the JSON your original instructions "
199
+ f"asked for."
200
+ )
201
+
202
+
203
+ def _shape_corrective(error: ValueError, raw_text: str) -> str:
204
+ """§30 (B2). The opening sentence is the whole reason this is a second
205
+ function: a model told its JSON was malformed, when it was not, has
206
+ nothing to fix and will most likely resend the same structure.
207
+
208
+ The problem line is PayloadShapeError's message, which already names
209
+ the pass, the field and the expected shape -- the three things #76
210
+ found the AttributeError naming none of."""
211
+ return (
212
+ f"Your last response was valid JSON, but not the structure this "
213
+ f"step asked for.\n\n"
214
+ f"Problem:\n{error}\n\n"
215
+ f"Start of your output (first 200 chars):\n{raw_text[:200]}\n\n"
216
+ f"Respond again with ONLY valid JSON, in the structure your "
217
+ f"original instructions specified -- no preamble, no prose, no "
218
+ f"code fence."
219
+ )