@yahyahammad/venastine 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +14 -0
- package/LICENSE +201 -0
- package/NOTICE +14 -0
- package/PRIVACY.md +66 -0
- package/README.md +878 -0
- package/SECURITY.md +88 -0
- package/THIRD_PARTY_NOTICES.md +100 -0
- package/agents/builtin/compactor.md +75 -0
- package/agents/builtin/explore.md +77 -0
- package/agents/builtin/grill-me.md +27 -0
- package/agents/builtin/initializer.md +94 -0
- package/agents/builtin/pipeline-reviewer.md +100 -0
- package/agents/builtin/plan.md +88 -0
- package/agents/builtin/review.md +74 -0
- package/agents/manager.py +195 -0
- package/agents/subagent_tool.py +215 -0
- package/agents/tui_commands.py +136 -0
- package/bin/venastine.mjs +382 -0
- package/config.py +1198 -0
- package/core/__init__.py +0 -0
- package/core/approval.py +115 -0
- package/core/client.py +1548 -0
- package/core/compaction.py +959 -0
- package/core/config_loader.py +1563 -0
- package/core/events.py +61 -0
- package/core/interaction.py +328 -0
- package/core/loop.py +1353 -0
- package/core/memory.py +370 -0
- package/core/model_windows.py +175 -0
- package/core/pipeline_models.py +176 -0
- package/core/reasoning/__init__.py +0 -0
- package/core/reasoning/authorization.py +130 -0
- package/core/reasoning/base.py +219 -0
- package/core/reasoning/confidence_scoring.py +468 -0
- package/core/reasoning/events.py +149 -0
- package/core/reasoning/json_retry.py +219 -0
- package/core/reasoning/orchestrator.py +1382 -0
- package/core/reasoning/output_writer.py +286 -0
- package/core/reasoning/payload_validation.py +398 -0
- package/core/reasoning/pipeline_storage.py +361 -0
- package/core/reasoning/review.py +615 -0
- package/core/reasoning/scholar.py +434 -0
- package/core/reasoning/source_corpus.py +327 -0
- package/core/reasoning/source_scoring.py +855 -0
- package/core/replay.py +167 -0
- package/core/session.py +153 -0
- package/core/workspace_trust.py +207 -0
- package/credentials.py +180 -0
- package/database.py +252 -0
- package/env_secrets.py +77 -0
- package/json_store.py +123 -0
- package/logging_setup.py +185 -0
- package/main.py +1959 -0
- package/mcp_client/__init__.py +15 -0
- package/mcp_client/client.py +499 -0
- package/mcp_client/config.py +379 -0
- package/mcp_client/registration.py +182 -0
- package/memories/manager.py +132 -0
- package/memories/tui_commands.py +232 -0
- package/package.json +73 -0
- package/project_init/config_files.py +278 -0
- package/project_init/doc_sets.py +330 -0
- package/project_init/generator.py +564 -0
- package/project_init/manifest.py +406 -0
- package/project_init/tui_commands.py +144 -0
- package/prompts/__init__.py +0 -0
- package/prompts/annotate.md +14 -0
- package/prompts/assumption_audit.md +27 -0
- package/prompts/claim_extraction.md +34 -0
- package/prompts/completeness.md +18 -0
- package/prompts/confidence_tiers.md +23 -0
- package/prompts/critic_pass.md +24 -0
- package/prompts/final_synthesis.md +13 -0
- package/prompts/initial_generation.md +14 -0
- package/prompts/preliminary_plan.md +23 -0
- package/prompts/revalidate.md +25 -0
- package/prompts/revise.md +15 -0
- package/prompts/source_grounding.md +66 -0
- package/prompts/system_prompts.py +262 -0
- package/prompts/universal_system_prompt +7 -0
- package/prompts/untrusted_content +1 -0
- package/providers.json.example +92 -0
- package/pyproject.toml +100 -0
- package/requirements.txt +72 -0
- package/safety/__init__.py +0 -0
- package/safety/policy_enforcement.py +682 -0
- package/security/__init__.py +0 -0
- package/security/capability.py +177 -0
- package/security/permissions.py +190 -0
- package/security/posture.py +302 -0
- package/security/protected_paths.py +309 -0
- package/security/sandbox.py +985 -0
- package/skills/builtin/crypto/cryptography-verification.md +66 -0
- package/skills/builtin/math/formal-specification.md +67 -0
- package/skills/builtin/math/numerical-methods.md +66 -0
- package/skills/builtin/math/proof-writing.md +61 -0
- package/skills/builtin/research/experiment-design.md +62 -0
- package/skills/builtin/research/literature-review.md +65 -0
- package/skills/builtin/research/reproducibility.md +55 -0
- package/skills/builtin/research/source-evaluation.md +70 -0
- package/skills/builtin/research/statistical-inference.md +63 -0
- package/skills/builtin/research/technical-writing.md +59 -0
- package/skills/builtin/security/cybersecurity-research.md +63 -0
- package/skills/builtin/software/code-review.md +61 -0
- package/skills/builtin/software/debugging.md +61 -0
- package/skills/builtin/software/test-design.md +67 -0
- package/skills/manager.py +138 -0
- package/skills/tui_commands.py +111 -0
- package/storage.py +1005 -0
- package/tools/base.py +305 -0
- package/tools/builtin/__init__.py +0 -0
- package/tools/builtin/_math_common.py +377 -0
- package/tools/builtin/_net_common.py +61 -0
- package/tools/builtin/arxiv.py +223 -0
- package/tools/builtin/ask_user.py +213 -0
- package/tools/builtin/discrete_math.py +134 -0
- package/tools/builtin/fetch_url.py +153 -0
- package/tools/builtin/file_ops.py +412 -0
- package/tools/builtin/geometry.py +148 -0
- package/tools/builtin/get_time.py +20 -0
- package/tools/builtin/linear_algebra.py +155 -0
- package/tools/builtin/load_skill.py +66 -0
- package/tools/builtin/logic.py +96 -0
- package/tools/builtin/pin.py +193 -0
- package/tools/builtin/probability_stats.py +228 -0
- package/tools/builtin/project_docs.py +405 -0
- package/tools/builtin/remember.py +158 -0
- package/tools/builtin/shell.py +461 -0
- package/tools/builtin/symbolic_math.py +119 -0
- package/tools/builtin/todo.py +147 -0
- package/tools/builtin/web_search.py +151 -0
- package/tools/context.py +136 -0
- package/tools/isolation.py +314 -0
- package/tools/registry.py +897 -0
- package/tui/app.py +3423 -0
- package/tui/app.tcss +458 -0
- package/tui/commands.py +61 -0
- package/tui/diffs.py +265 -0
- package/tui/preferences.py +182 -0
- package/tui/ravens.py +146 -0
- package/tui/screens.py +943 -0
- package/tui/themes.py +366 -0
- package/tui/widgets.py +1239 -0
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"""
|
|
2
|
+
core/reasoning/json_retry.py
|
|
3
|
+
|
|
4
|
+
ROADMAP §3's malformed-JSON recovery, extracted from orchestrator.py so
|
|
5
|
+
that §20's reviewer gets it too.
|
|
6
|
+
|
|
7
|
+
The mechanism: when a model that was asked for JSON returns something
|
|
8
|
+
that does not parse, send a corrective follow-up into the SAME thread it
|
|
9
|
+
just used. The failed assistant turn is already in that thread's history
|
|
10
|
+
(core/loop.py's D20 always-persist behaviour), so the model sees its own
|
|
11
|
+
prior output rather than being asked again from a clean slate. That is
|
|
12
|
+
the whole reason this is a thread continuation and not a fresh call.
|
|
13
|
+
|
|
14
|
+
WHY THIS IS ITS OWN MODULE. The orchestrator's version hardcoded
|
|
15
|
+
stream_deep_research_mode as the first attempt and pass_prompt(pass_id) as
|
|
16
|
+
the system prompt -- correct for the ten passes, wrong for anything else.
|
|
17
|
+
§20's reviewer is agent-shaped: it starts through
|
|
18
|
+
run_agent_conversation with an agent's own system prompt. Copying twenty
|
|
19
|
+
lines of retry loop into review.py would put two copies of the corrective
|
|
20
|
+
wording, the attempt arithmetic and the trace format in one codebase,
|
|
21
|
+
and they would drift. So the loop takes an ALREADY-OBTAINED response and
|
|
22
|
+
the parameters needed to continue its thread; each caller starts its own
|
|
23
|
+
first attempt however it likes.
|
|
24
|
+
|
|
25
|
+
ROADMAP_v2 §30 (B2) added the SECOND thing worth correcting. `validate`
|
|
26
|
+
is an optional callable applied to the parsed payload; if it raises
|
|
27
|
+
PayloadShapeError the loop treats that exactly like a parse failure --
|
|
28
|
+
same thread, same attempt arithmetic -- with one difference that has to
|
|
29
|
+
be there: the corrective WORDING branches. "Your last response did not
|
|
30
|
+
parse as valid JSON" is false about a payload that parsed perfectly and
|
|
31
|
+
was the wrong shape, and telling a model its JSON is malformed when it
|
|
32
|
+
is not is the kind of correction that produces a second wrong answer.
|
|
33
|
+
|
|
34
|
+
`validate=None` is the default and keeps this function byte-identical
|
|
35
|
+
for §20's reviewer, which does its own value-level filtering in
|
|
36
|
+
review.py:_validated for reasons that module states.
|
|
37
|
+
|
|
38
|
+
`attempts = max_retries + 1` (one initial plus max_retries corrective).
|
|
39
|
+
Unrecoverable failure raises ValueError with the final parse error
|
|
40
|
+
attached -- the orchestrator's try/except turns that into a durable
|
|
41
|
+
status='failed' record.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
from __future__ import annotations
|
|
45
|
+
|
|
46
|
+
import json
|
|
47
|
+
from typing import Any, Callable, Optional
|
|
48
|
+
|
|
49
|
+
import config
|
|
50
|
+
from core.reasoning.payload_validation import PayloadShapeError
|
|
51
|
+
|
|
52
|
+
MAX_JSON_RETRIES = config.MAX_JSON_RETRIES
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def parse_json_response(text: str) -> Any:
|
|
56
|
+
"""Callers are instructed (via the universal preamble, or an agent's
|
|
57
|
+
own output-format section) to emit clean JSON, optionally in a code
|
|
58
|
+
fence. Strip the fence if present."""
|
|
59
|
+
cleaned = text.strip()
|
|
60
|
+
if cleaned.startswith("```"):
|
|
61
|
+
cleaned = cleaned.split("```")[1]
|
|
62
|
+
if cleaned.lower().startswith("json"):
|
|
63
|
+
cleaned = cleaned[4:]
|
|
64
|
+
cleaned = cleaned.strip()
|
|
65
|
+
try:
|
|
66
|
+
return json.loads(cleaned)
|
|
67
|
+
except json.JSONDecodeError as e:
|
|
68
|
+
# Caller-neutral wording: this module also serves §20's reviewer,
|
|
69
|
+
# which is not a pass (review §19-20 r5-1).
|
|
70
|
+
raise ValueError(f"Response did not return valid JSON: {e}\n\nRaw text:\n{text[:500]}") from e
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def retry_until_json(
|
|
74
|
+
response,
|
|
75
|
+
label: str,
|
|
76
|
+
system_prompt: str,
|
|
77
|
+
model: str,
|
|
78
|
+
provider_name: str,
|
|
79
|
+
trace: Optional[list] = None,
|
|
80
|
+
temperature: Optional[float] = None,
|
|
81
|
+
authorization=None,
|
|
82
|
+
max_retries: int = MAX_JSON_RETRIES,
|
|
83
|
+
on_response: Optional[Callable] = None,
|
|
84
|
+
context=None,
|
|
85
|
+
max_total_tokens: Optional[int] = None,
|
|
86
|
+
validate: Optional[Callable] = None,
|
|
87
|
+
effort: Optional[str] = None,
|
|
88
|
+
) -> str:
|
|
89
|
+
"""Returns the first response text that parses as JSON, retrying by
|
|
90
|
+
continuing `response`'s thread.
|
|
91
|
+
|
|
92
|
+
response an already-obtained ModelResponse. Its thread_id is what
|
|
93
|
+
the corrective message is sent into, so the caller's own
|
|
94
|
+
first attempt is what gets corrected.
|
|
95
|
+
label what the trace line names -- a pass id, or "Review".
|
|
96
|
+
on_response called with every response this function obtains. The
|
|
97
|
+
orchestrator hooks §25's granted-call recorder here: a
|
|
98
|
+
retry is the same pass continuing, so its tool calls
|
|
99
|
+
belong on the same audit trail, and a retry that spent a
|
|
100
|
+
grant invisibly is exactly the record §25 exists to
|
|
101
|
+
keep.
|
|
102
|
+
context the ToolContext the caller's first attempt ran under.
|
|
103
|
+
A retry is the SAME run continuing, so it must carry
|
|
104
|
+
the same tool restriction -- defaulting to None would
|
|
105
|
+
re-advertise globally-allowed tools an agent-shaped
|
|
106
|
+
caller excluded on turn 1 (review §19-20 r2-1).
|
|
107
|
+
max_total_tokens
|
|
108
|
+
the caller's own spend ceiling, for exactly the same
|
|
109
|
+
reason. #4: there is no separate pass constant any more
|
|
110
|
+
-- a caller forwarding its ceiling keeps it; None omits
|
|
111
|
+
the kwarg, so the wrapper resolves the configured
|
|
112
|
+
settings.json max_token_budget (uncapped unless set),
|
|
113
|
+
which is right for §20's agent-shaped reviewer too.
|
|
114
|
+
A budget stop returns the last response as it stands,
|
|
115
|
+
which is how an empty pass gets produced in the first
|
|
116
|
+
place -- hence carrying the caller's figure at all.
|
|
117
|
+
effort batch 25 (#139): the caller's effort level. A retry is
|
|
118
|
+
the same pass continuing, so it carries the same level;
|
|
119
|
+
effort_for() re-validates against this thread's model.
|
|
120
|
+
validate §30 (B2). Applied to the PARSED payload; raising
|
|
121
|
+
PayloadShapeError re-enters this same loop. None means
|
|
122
|
+
"parse is the only contract", which is what the reviewer
|
|
123
|
+
wants and what every caller had before §30.
|
|
124
|
+
"""
|
|
125
|
+
from core.loop import RunAgentLoop
|
|
126
|
+
|
|
127
|
+
raw_text = response.text
|
|
128
|
+
|
|
129
|
+
for attempt in range(max_retries):
|
|
130
|
+
try:
|
|
131
|
+
_check(raw_text, validate)
|
|
132
|
+
return raw_text
|
|
133
|
+
except ValueError as e:
|
|
134
|
+
shape = isinstance(e, PayloadShapeError)
|
|
135
|
+
if trace is not None:
|
|
136
|
+
# The parse wording is unchanged to the byte. A reader
|
|
137
|
+
# scanning a trace for "JSON parse failed" is scanning for
|
|
138
|
+
# a model that cannot emit JSON, and a shape rejection is
|
|
139
|
+
# a different fact about a different failure.
|
|
140
|
+
trace.append(
|
|
141
|
+
f"{label}: "
|
|
142
|
+
f"{'payload shape rejected' if shape else 'JSON parse failed'}"
|
|
143
|
+
f" on attempt {attempt + 1}/"
|
|
144
|
+
f"{max_retries + 1} ({str(e).splitlines()[0]}), "
|
|
145
|
+
f"retrying with corrective message."
|
|
146
|
+
)
|
|
147
|
+
corrective = (
|
|
148
|
+
_shape_corrective(e, raw_text) if shape
|
|
149
|
+
else _parse_corrective(e, raw_text)
|
|
150
|
+
)
|
|
151
|
+
retry_response = RunAgentLoop.continue_conversation(
|
|
152
|
+
thread_id=response.thread_id,
|
|
153
|
+
message=corrective,
|
|
154
|
+
system_prompt=system_prompt,
|
|
155
|
+
model=model,
|
|
156
|
+
provider_name=provider_name,
|
|
157
|
+
temperature=temperature,
|
|
158
|
+
# A retry is the SAME work continuing. Omitting this would
|
|
159
|
+
# strip the gated tools on exactly the attempt where the
|
|
160
|
+
# model is already struggling, and the only symptom would
|
|
161
|
+
# be a second malformed answer.
|
|
162
|
+
authorization=authorization,
|
|
163
|
+
context=context,
|
|
164
|
+
# Same rule, batch 25 (#139): a retry is the same pass
|
|
165
|
+
# continuing, so it carries the pass's effort too --
|
|
166
|
+
# effort_for re-validates against this thread's model.
|
|
167
|
+
effort=effort,
|
|
168
|
+
**({} if max_total_tokens is None
|
|
169
|
+
else {"max_total_tokens": max_total_tokens}),
|
|
170
|
+
)
|
|
171
|
+
if on_response is not None:
|
|
172
|
+
on_response(retry_response)
|
|
173
|
+
raw_text = retry_response.text
|
|
174
|
+
|
|
175
|
+
# Last attempt: validate one more time and either return or raise.
|
|
176
|
+
_check(raw_text, validate)
|
|
177
|
+
return raw_text
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _check(raw_text: str, validate: Optional[Callable]) -> None:
|
|
181
|
+
"""Both contracts, in the order they can be checked: a payload has to
|
|
182
|
+
parse before it can have a shape."""
|
|
183
|
+
payload = parse_json_response(raw_text)
|
|
184
|
+
if validate is not None:
|
|
185
|
+
validate(payload)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _parse_corrective(error: ValueError, raw_text: str) -> str:
|
|
189
|
+
return (
|
|
190
|
+
f"Your last response did not parse as valid JSON.\n\n"
|
|
191
|
+
f"Parse error:\n{error}\n\n"
|
|
192
|
+
f"Start of your failed output (first 200 chars):\n{raw_text[:200]}\n\n"
|
|
193
|
+
# Caller-neutral: "the JSON your original instructions asked
|
|
194
|
+
# for" is true for a pass (object) and for §20's reviewer
|
|
195
|
+
# (array) alike. Naming a shape the caller never promised
|
|
196
|
+
# could talk the model out of its own output contract.
|
|
197
|
+
f"Respond again with ONLY valid JSON -- no preamble, no prose, "
|
|
198
|
+
f"no code fence, just the JSON your original instructions "
|
|
199
|
+
f"asked for."
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _shape_corrective(error: ValueError, raw_text: str) -> str:
|
|
204
|
+
"""§30 (B2). The opening sentence is the whole reason this is a second
|
|
205
|
+
function: a model told its JSON was malformed, when it was not, has
|
|
206
|
+
nothing to fix and will most likely resend the same structure.
|
|
207
|
+
|
|
208
|
+
The problem line is PayloadShapeError's message, which already names
|
|
209
|
+
the pass, the field and the expected shape -- the three things #76
|
|
210
|
+
found the AttributeError naming none of."""
|
|
211
|
+
return (
|
|
212
|
+
f"Your last response was valid JSON, but not the structure this "
|
|
213
|
+
f"step asked for.\n\n"
|
|
214
|
+
f"Problem:\n{error}\n\n"
|
|
215
|
+
f"Start of your output (first 200 chars):\n{raw_text[:200]}\n\n"
|
|
216
|
+
f"Respond again with ONLY valid JSON, in the structure your "
|
|
217
|
+
f"original instructions specified -- no preamble, no prose, no "
|
|
218
|
+
f"code fence."
|
|
219
|
+
)
|