@yahyahammad/venastine 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +14 -0
- package/LICENSE +201 -0
- package/NOTICE +14 -0
- package/PRIVACY.md +66 -0
- package/README.md +878 -0
- package/SECURITY.md +88 -0
- package/THIRD_PARTY_NOTICES.md +100 -0
- package/agents/builtin/compactor.md +75 -0
- package/agents/builtin/explore.md +77 -0
- package/agents/builtin/grill-me.md +27 -0
- package/agents/builtin/initializer.md +94 -0
- package/agents/builtin/pipeline-reviewer.md +100 -0
- package/agents/builtin/plan.md +88 -0
- package/agents/builtin/review.md +74 -0
- package/agents/manager.py +195 -0
- package/agents/subagent_tool.py +215 -0
- package/agents/tui_commands.py +136 -0
- package/bin/venastine.mjs +382 -0
- package/config.py +1198 -0
- package/core/__init__.py +0 -0
- package/core/approval.py +115 -0
- package/core/client.py +1548 -0
- package/core/compaction.py +959 -0
- package/core/config_loader.py +1563 -0
- package/core/events.py +61 -0
- package/core/interaction.py +328 -0
- package/core/loop.py +1353 -0
- package/core/memory.py +370 -0
- package/core/model_windows.py +175 -0
- package/core/pipeline_models.py +176 -0
- package/core/reasoning/__init__.py +0 -0
- package/core/reasoning/authorization.py +130 -0
- package/core/reasoning/base.py +219 -0
- package/core/reasoning/confidence_scoring.py +468 -0
- package/core/reasoning/events.py +149 -0
- package/core/reasoning/json_retry.py +219 -0
- package/core/reasoning/orchestrator.py +1382 -0
- package/core/reasoning/output_writer.py +286 -0
- package/core/reasoning/payload_validation.py +398 -0
- package/core/reasoning/pipeline_storage.py +361 -0
- package/core/reasoning/review.py +615 -0
- package/core/reasoning/scholar.py +434 -0
- package/core/reasoning/source_corpus.py +327 -0
- package/core/reasoning/source_scoring.py +855 -0
- package/core/replay.py +167 -0
- package/core/session.py +153 -0
- package/core/workspace_trust.py +207 -0
- package/credentials.py +180 -0
- package/database.py +252 -0
- package/env_secrets.py +77 -0
- package/json_store.py +123 -0
- package/logging_setup.py +185 -0
- package/main.py +1959 -0
- package/mcp_client/__init__.py +15 -0
- package/mcp_client/client.py +499 -0
- package/mcp_client/config.py +379 -0
- package/mcp_client/registration.py +182 -0
- package/memories/manager.py +132 -0
- package/memories/tui_commands.py +232 -0
- package/package.json +73 -0
- package/project_init/config_files.py +278 -0
- package/project_init/doc_sets.py +330 -0
- package/project_init/generator.py +564 -0
- package/project_init/manifest.py +406 -0
- package/project_init/tui_commands.py +144 -0
- package/prompts/__init__.py +0 -0
- package/prompts/annotate.md +14 -0
- package/prompts/assumption_audit.md +27 -0
- package/prompts/claim_extraction.md +34 -0
- package/prompts/completeness.md +18 -0
- package/prompts/confidence_tiers.md +23 -0
- package/prompts/critic_pass.md +24 -0
- package/prompts/final_synthesis.md +13 -0
- package/prompts/initial_generation.md +14 -0
- package/prompts/preliminary_plan.md +23 -0
- package/prompts/revalidate.md +25 -0
- package/prompts/revise.md +15 -0
- package/prompts/source_grounding.md +66 -0
- package/prompts/system_prompts.py +262 -0
- package/prompts/universal_system_prompt +7 -0
- package/prompts/untrusted_content +1 -0
- package/providers.json.example +92 -0
- package/pyproject.toml +100 -0
- package/requirements.txt +72 -0
- package/safety/__init__.py +0 -0
- package/safety/policy_enforcement.py +682 -0
- package/security/__init__.py +0 -0
- package/security/capability.py +177 -0
- package/security/permissions.py +190 -0
- package/security/posture.py +302 -0
- package/security/protected_paths.py +309 -0
- package/security/sandbox.py +985 -0
- package/skills/builtin/crypto/cryptography-verification.md +66 -0
- package/skills/builtin/math/formal-specification.md +67 -0
- package/skills/builtin/math/numerical-methods.md +66 -0
- package/skills/builtin/math/proof-writing.md +61 -0
- package/skills/builtin/research/experiment-design.md +62 -0
- package/skills/builtin/research/literature-review.md +65 -0
- package/skills/builtin/research/reproducibility.md +55 -0
- package/skills/builtin/research/source-evaluation.md +70 -0
- package/skills/builtin/research/statistical-inference.md +63 -0
- package/skills/builtin/research/technical-writing.md +59 -0
- package/skills/builtin/security/cybersecurity-research.md +63 -0
- package/skills/builtin/software/code-review.md +61 -0
- package/skills/builtin/software/debugging.md +61 -0
- package/skills/builtin/software/test-design.md +67 -0
- package/skills/manager.py +138 -0
- package/skills/tui_commands.py +111 -0
- package/storage.py +1005 -0
- package/tools/base.py +305 -0
- package/tools/builtin/__init__.py +0 -0
- package/tools/builtin/_math_common.py +377 -0
- package/tools/builtin/_net_common.py +61 -0
- package/tools/builtin/arxiv.py +223 -0
- package/tools/builtin/ask_user.py +213 -0
- package/tools/builtin/discrete_math.py +134 -0
- package/tools/builtin/fetch_url.py +153 -0
- package/tools/builtin/file_ops.py +412 -0
- package/tools/builtin/geometry.py +148 -0
- package/tools/builtin/get_time.py +20 -0
- package/tools/builtin/linear_algebra.py +155 -0
- package/tools/builtin/load_skill.py +66 -0
- package/tools/builtin/logic.py +96 -0
- package/tools/builtin/pin.py +193 -0
- package/tools/builtin/probability_stats.py +228 -0
- package/tools/builtin/project_docs.py +405 -0
- package/tools/builtin/remember.py +158 -0
- package/tools/builtin/shell.py +461 -0
- package/tools/builtin/symbolic_math.py +119 -0
- package/tools/builtin/todo.py +147 -0
- package/tools/builtin/web_search.py +151 -0
- package/tools/context.py +136 -0
- package/tools/isolation.py +314 -0
- package/tools/registry.py +897 -0
- package/tui/app.py +3423 -0
- package/tui/app.tcss +458 -0
- package/tui/commands.py +61 -0
- package/tui/diffs.py +265 -0
- package/tui/preferences.py +182 -0
- package/tui/ravens.py +146 -0
- package/tui/screens.py +943 -0
- package/tui/themes.py +366 -0
- package/tui/widgets.py +1239 -0
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
from pydantic import BaseModel, Field
|
|
7
|
+
|
|
8
|
+
from safety.policy_enforcement import is_url_permitted
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
_TOOL_DESCRIPTION = "Retrieve the contents of a web page using a URL."
|
|
13
|
+
MAX_CONTENT_CHARS = 5000
|
|
14
|
+
|
|
15
|
+
# ROADMAP_v2 §31 (H7), #55. How many BYTES may be pulled off the wire
|
|
16
|
+
# before the connection is dropped, as distinct from how many characters
|
|
17
|
+
# are returned. Both are needed and they bound different things:
|
|
18
|
+
# MAX_CONTENT_CHARS is what the model reads, this is what the process
|
|
19
|
+
# holds.
|
|
20
|
+
#
|
|
21
|
+
# Generous next to 5000 characters -- a multi-byte encoding can spend
|
|
22
|
+
# four bytes on one of them -- because the point is not to be tight. It
|
|
23
|
+
# is that `response.text` had no ceiling AT ALL: httpx.get buffers the
|
|
24
|
+
# entire body before the slice happens, and the body is chosen by
|
|
25
|
+
# whatever the URL points at. A Content-Length check is not the fix
|
|
26
|
+
# either; a hostile server can omit it or lie about it, so the only
|
|
27
|
+
# thing that actually bounds this is refusing to read past a cap.
|
|
28
|
+
MAX_CONTENT_BYTES = 65_536
|
|
29
|
+
REQUEST_TIMEOUT_S = 8.0
|
|
30
|
+
|
|
31
|
+
# Redirects are followed BY HAND (see run()), so this is the loop bound,
|
|
32
|
+
# not httpx's. httpx's own default is 20; this is lower because a
|
|
33
|
+
# research fetch that needs more than five hops is not a fetch worth
|
|
34
|
+
# making, and every hop costs a policy check and a DNS resolution.
|
|
35
|
+
MAX_REDIRECTS = 5
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class FetchURLParams(BaseModel):
|
|
39
|
+
url: str = Field(..., description="URL to be fetched", min_length=1, max_length=400)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
TOOL_SCHEMA = {
|
|
43
|
+
"name": "fetch_url",
|
|
44
|
+
"description": _TOOL_DESCRIPTION,
|
|
45
|
+
"input_schema": FetchURLParams.model_json_schema(),
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _bounded_body(response) -> tuple:
|
|
50
|
+
"""(decoded text, whether the server still had more to send).
|
|
51
|
+
|
|
52
|
+
Stops after MAX_CONTENT_BYTES rather than reading to the end. The
|
|
53
|
+
extra chunk beyond the cap is what makes `more` honest: without it,
|
|
54
|
+
a body that happens to end exactly at the cap is indistinguishable
|
|
55
|
+
from one that was cut off.
|
|
56
|
+
"""
|
|
57
|
+
chunks, total, more = [], 0, False
|
|
58
|
+
for chunk in response.iter_bytes():
|
|
59
|
+
chunks.append(chunk)
|
|
60
|
+
total += len(chunk)
|
|
61
|
+
if total > MAX_CONTENT_BYTES:
|
|
62
|
+
more = True
|
|
63
|
+
break
|
|
64
|
+
raw = b"".join(chunks)[:MAX_CONTENT_BYTES]
|
|
65
|
+
encoding = getattr(response, "encoding", None) or "utf-8"
|
|
66
|
+
try:
|
|
67
|
+
return raw.decode(encoding, errors="replace"), more
|
|
68
|
+
except LookupError:
|
|
69
|
+
# A server may name an encoding this build of Python does not
|
|
70
|
+
# have. That is not a reason to lose the page.
|
|
71
|
+
return raw.decode("utf-8", errors="replace"), more
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def run(params: dict) -> dict:
|
|
75
|
+
"""Fetch one URL, checking policy BEFORE every request it issues.
|
|
76
|
+
|
|
77
|
+
Redirects are followed manually (#53, #54). The version this replaced
|
|
78
|
+
passed `follow_redirects=True` and checked the blocklist once, against
|
|
79
|
+
the argument -- so one 302 walked past the control, and the blocked
|
|
80
|
+
host had already served its body by the time anything could object.
|
|
81
|
+
§25 R5's dispatch-wide check has the same blind spot for the same
|
|
82
|
+
reason: it scans the argument, and the argument was clean.
|
|
83
|
+
|
|
84
|
+
Checking each hop BEFORE issuing it, rather than checking the history
|
|
85
|
+
afterwards, is the whole point. For a malware host the difference is
|
|
86
|
+
whether it was contacted; for `169.254.169.254` the request IS the
|
|
87
|
+
disclosure, so a post-hoc check does not fix #54 at all.
|
|
88
|
+
|
|
89
|
+
The returned `url` is the one that ANSWERED, not the one that was
|
|
90
|
+
asked for. That half is a correctness fix as much as a security one:
|
|
91
|
+
the grounding passes attribute fetched text to the URL in this field,
|
|
92
|
+
and output_writer builds each run's sources/ directory from it -- so
|
|
93
|
+
a redirect chain used to silently rewrite what a claim was grounded
|
|
94
|
+
in, with nothing in the result, the transcript or the MessageLog
|
|
95
|
+
recording where the content actually came from.
|
|
96
|
+
"""
|
|
97
|
+
parsed = FetchURLParams(**params)
|
|
98
|
+
|
|
99
|
+
url = parsed.url
|
|
100
|
+
for _ in range(MAX_REDIRECTS):
|
|
101
|
+
refusal = is_url_permitted(url)
|
|
102
|
+
if refusal:
|
|
103
|
+
return {"error": refusal}
|
|
104
|
+
|
|
105
|
+
try:
|
|
106
|
+
# §31 (H7): streamed, so the headers are available before
|
|
107
|
+
# any body is pulled. Two consequences, both wanted -- a
|
|
108
|
+
# redirect now costs no body at all (the old path downloaded
|
|
109
|
+
# every hop's), and the body that IS wanted stops at
|
|
110
|
+
# MAX_CONTENT_BYTES instead of being buffered whole and then
|
|
111
|
+
# sliced. read_run's guard is the model: bound before, not
|
|
112
|
+
# after.
|
|
113
|
+
with httpx.stream("GET", url, timeout=REQUEST_TIMEOUT_S,
|
|
114
|
+
follow_redirects=False) as response:
|
|
115
|
+
if response.is_redirect:
|
|
116
|
+
# .next_request is None on a redirect with no usable
|
|
117
|
+
# Location; treat that as the end of the chain rather
|
|
118
|
+
# than following nothing.
|
|
119
|
+
if response.next_request is None:
|
|
120
|
+
break
|
|
121
|
+
url = str(response.next_request.url)
|
|
122
|
+
continue
|
|
123
|
+
response.raise_for_status()
|
|
124
|
+
body, more = _bounded_body(response)
|
|
125
|
+
except httpx.HTTPError as e:
|
|
126
|
+
# Interpolated, NOT extra={}: the default formatter renders only
|
|
127
|
+
# %(message)s, so every field passed via extra was silently
|
|
128
|
+
# dropped -- fifteen consecutive "fetch_url failed" lines with
|
|
129
|
+
# no URL and no reason, which is what made a run of ordinary
|
|
130
|
+
# 404s indistinguishable from a broken tool.
|
|
131
|
+
logger.warning("fetch_url failed for %s: %s", url, e)
|
|
132
|
+
return {"error": f"Could not fetch URL: {e}"}
|
|
133
|
+
|
|
134
|
+
return {
|
|
135
|
+
"url": url,
|
|
136
|
+
"content": body[:MAX_CONTENT_CHARS],
|
|
137
|
+
# True when EITHER bound bit: more characters were decoded
|
|
138
|
+
# than are returned, or the byte cap stopped the read with
|
|
139
|
+
# the server still sending. A silently short page is how a
|
|
140
|
+
# grounding pass concludes a source does not say something.
|
|
141
|
+
#
|
|
142
|
+
# `or more` CANNOT FIRE with the shipped constants, and that
|
|
143
|
+
# is deliberate rather than an oversight: 65,536 bytes is at
|
|
144
|
+
# least 16,384 characters even at four bytes each, so the
|
|
145
|
+
# first term is always already true when the byte cap bites.
|
|
146
|
+
# It is here for the day someone raises MAX_CONTENT_CHARS or
|
|
147
|
+
# lowers MAX_CONTENT_BYTES, which is exactly when a silent
|
|
148
|
+
# under-report would appear. A mutation deleting it survived
|
|
149
|
+
# until a test moved the constants to reach it.
|
|
150
|
+
"truncated": len(body) > MAX_CONTENT_CHARS or more,
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
return {"error": f"Too many redirects (more than {MAX_REDIRECTS}): {parsed.url}"}
|
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
"""
|
|
2
|
+
tools/builtin/file_ops.py
|
|
3
|
+
|
|
4
|
+
Three file-operation tools — read, write, edit — confined to a
|
|
5
|
+
configurable workspace directory. Registered as three separate tools
|
|
6
|
+
to match config.ToolPermissions' three separate boolean fields (read /
|
|
7
|
+
write / edit), so review/audit agents can be given read access without
|
|
8
|
+
write or edit.
|
|
9
|
+
|
|
10
|
+
SECURITY MODEL (ROADMAP §6, deviated per user decision):
|
|
11
|
+
Paths within WORKSPACE_DIR are auto-approved (unless the matching
|
|
12
|
+
ToolApprovals boolean is True, which forces approval even inside the
|
|
13
|
+
workspace). Paths outside WORKSPACE_DIR always require user approval
|
|
14
|
+
— they are NOT hard-rejected, so an operator can grant one-off access
|
|
15
|
+
to files outside the workspace. Symlinks and '..' segments are always
|
|
16
|
+
resolved via os.path.realpath before any check, so a path that looks
|
|
17
|
+
inside the workspace but resolves outside it is treated as outside.
|
|
18
|
+
|
|
19
|
+
PROTECTED SEGMENTS (security/protected_paths.py) are the one hard
|
|
20
|
+
deny, and the exception to both rules above: any path -- inside or
|
|
21
|
+
outside the workspace -- that resolves into `.venastine/` is refused
|
|
22
|
+
by the refusal_check before approval and re-tested in all three
|
|
23
|
+
handlers, whatever the ToolApprovals booleans say.
|
|
24
|
+
|
|
25
|
+
READ TOOL PAGINATION (deviated from ROADMAP §6):
|
|
26
|
+
The read tool accepts offset (lines to skip) and count (lines to
|
|
27
|
+
read). count is clamped to config.MAX_READ_LINES with a note in the
|
|
28
|
+
response (not a hard error). Output is also truncated at
|
|
29
|
+
config.MAX_READ_CHARS with a note. Files larger than
|
|
30
|
+
config.MAX_FILE_SIZE_BYTES are rejected before opening to prevent
|
|
31
|
+
memory exhaustion (especially on the markitdown path, which loads
|
|
32
|
+
the whole file).
|
|
33
|
+
|
|
34
|
+
RICH FORMAT SUPPORT (deviated from ROADMAP §6):
|
|
35
|
+
For binary/rich extensions (.pdf, .docx, .pptx, etc.) the read tool
|
|
36
|
+
uses markitdown to convert to markdown before applying line/char
|
|
37
|
+
truncation. Plain-text extensions use open() directly.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
import logging
|
|
43
|
+
import os
|
|
44
|
+
from typing import Optional
|
|
45
|
+
|
|
46
|
+
from pydantic import BaseModel, Field
|
|
47
|
+
|
|
48
|
+
import config
|
|
49
|
+
|
|
50
|
+
from security.protected_paths import protected_segment
|
|
51
|
+
|
|
52
|
+
logger = logging.getLogger(__name__)
|
|
53
|
+
|
|
54
|
+
# ---------------------------------------------------------------------------
|
|
55
|
+
# ---- Path helpers ---------------------------------------------------------
|
|
56
|
+
# ---------------------------------------------------------------------------
|
|
57
|
+
|
|
58
|
+
WORKSPACE_ROOT = os.path.realpath(config.WORKSPACE_DIR)
|
|
59
|
+
|
|
60
|
+
_RICH_EXTENSIONS = frozenset({
|
|
61
|
+
".pdf", ".docx", ".pptx", ".xlsx", ".xls",
|
|
62
|
+
".html", ".htm", ".epub", ".ipynb",
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def resolve_path(user_path: str) -> str:
|
|
67
|
+
"""Resolve *user_path* relative to WORKSPACE_ROOT via realpath.
|
|
68
|
+
|
|
69
|
+
Handles ``..`` segments and symlinks. Does NOT reject paths that
|
|
70
|
+
land outside the workspace — that decision belongs to the approval
|
|
71
|
+
layer (``_file_approval_check``), not to path resolution.
|
|
72
|
+
|
|
73
|
+
PUBLIC since batch 41 (X4). The TUI has to read the file a `write`
|
|
74
|
+
or an `edit` is about to change in order to show the diff, and it
|
|
75
|
+
must land on the same file the tool will — a second
|
|
76
|
+
``os.path.join(WORKSPACE_ROOT, ...)`` at the call site is this
|
|
77
|
+
project's canonical producer/consumer bug, and here it would mean a
|
|
78
|
+
diff of one file beside an edit to another.
|
|
79
|
+
"""
|
|
80
|
+
return os.path.realpath(os.path.join(WORKSPACE_ROOT, user_path))
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _is_within_workspace(user_path: str) -> bool:
|
|
84
|
+
resolved = resolve_path(user_path)
|
|
85
|
+
return resolved == WORKSPACE_ROOT or resolved.startswith(WORKSPACE_ROOT + os.sep)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _file_approval_check(tool_name: str, params: dict) -> bool:
|
|
89
|
+
"""Path-dependent approval policy for read / write / edit.
|
|
90
|
+
|
|
91
|
+
Outside workspace → always True (ask user).
|
|
92
|
+
Within workspace → defer to the matching ToolApprovals boolean
|
|
93
|
+
(False = auto-approved, True = still ask).
|
|
94
|
+
"""
|
|
95
|
+
path = params.get("path", "")
|
|
96
|
+
if not _is_within_workspace(path):
|
|
97
|
+
return True
|
|
98
|
+
approvals = config.ToolApprovals()
|
|
99
|
+
return getattr(approvals, tool_name, False)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
# ===========================================================================
|
|
103
|
+
# ---- Protected segments (.venastine/) -------------------------------------
|
|
104
|
+
# ===========================================================================
|
|
105
|
+
|
|
106
|
+
def _protected_path_error(user_path: str) -> Optional[str]:
|
|
107
|
+
"""Why *user_path* must be refused outright, or None.
|
|
108
|
+
|
|
109
|
+
Shared by the refusal_check and all three handlers so the decision
|
|
110
|
+
and its wording cannot drift between the pre-approval answer and the
|
|
111
|
+
backstop. resolve_path BEFORE the segment test, so `..` travel and a
|
|
112
|
+
symlink are caught the same way _is_within_workspace catches them.
|
|
113
|
+
|
|
114
|
+
Unconditional on workspace membership, like project_docs'
|
|
115
|
+
`_is_readable_doc`: the segment is denied wherever the path lands,
|
|
116
|
+
because `.venastine/` holds mcp.json -- which names commands the
|
|
117
|
+
harness spawns at startup -- and skills/ and agents/ content that is
|
|
118
|
+
injected into system prompts. Workspace membership is the wrong
|
|
119
|
+
question here: with AGENT_WORKSPACE pointed at the project the
|
|
120
|
+
directory is INSIDE the workspace and _file_approval_check would
|
|
121
|
+
auto-approve every call against it.
|
|
122
|
+
"""
|
|
123
|
+
segment = protected_segment(resolve_path(user_path))
|
|
124
|
+
if segment is None:
|
|
125
|
+
return None
|
|
126
|
+
return (
|
|
127
|
+
f"Refused: the path resolves into '{segment}/', which holds this "
|
|
128
|
+
"harness's own configuration and trust-gated content (settings.json, "
|
|
129
|
+
"mcp.json, skills, agents). It is denied however the file tools are "
|
|
130
|
+
"called."
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _protected_refusal(params: dict, _context=None) -> Optional[str]:
|
|
135
|
+
"""ToolSpec.refusal_check for read / write / edit (§32 A7's shape).
|
|
136
|
+
|
|
137
|
+
dispatch() consults this BEFORE the approval gate and returns the
|
|
138
|
+
reason as the result, and core/loop.py suppresses the prompt for a
|
|
139
|
+
refused call -- nobody is asked a question whose answer cannot
|
|
140
|
+
change the outcome. NOT the enforcement (base.py's own words): the
|
|
141
|
+
three handlers re-test below, so a caller that reaches them
|
|
142
|
+
directly is refused too.
|
|
143
|
+
"""
|
|
144
|
+
return _protected_path_error(params.get("path", ""))
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
# ===========================================================================
|
|
148
|
+
# ---- READ -----------------------------------------------------------------
|
|
149
|
+
# ===========================================================================
|
|
150
|
+
|
|
151
|
+
class ReadFileParams(BaseModel):
|
|
152
|
+
path: str = Field(..., description="Path relative to the agent's workspace directory, or an absolute path.")
|
|
153
|
+
offset: int = Field(0, ge=0, description="Number of lines to skip before reading (0-based).")
|
|
154
|
+
count: int = Field(
|
|
155
|
+
config.MAX_READ_LINES,
|
|
156
|
+
ge=1,
|
|
157
|
+
description=f"Number of lines to read. Clamped to {config.MAX_READ_LINES} if higher.",
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
READ_TOOL_SCHEMA = {
|
|
162
|
+
"name": "read",
|
|
163
|
+
"description": (
|
|
164
|
+
"Read the contents of a file. For rich formats (PDF, DOCX, PPTX, "
|
|
165
|
+
"XLSX, HTML, etc.) the content is converted to markdown first. "
|
|
166
|
+
"Use offset and count to page through large files. Output is "
|
|
167
|
+
f"limited to {config.MAX_READ_LINES} lines or {config.MAX_READ_CHARS} "
|
|
168
|
+
"characters, whichever comes first."
|
|
169
|
+
),
|
|
170
|
+
"input_schema": ReadFileParams.model_json_schema(),
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _read_rich(resolved_path: str) -> str:
|
|
175
|
+
"""Convert a rich-format file to markdown via markitdown.
|
|
176
|
+
|
|
177
|
+
markitdown is an OPTIONAL extra since #144: this function is its only
|
|
178
|
+
consumer, it sits behind `read`, and `read` is globally denied and
|
|
179
|
+
cannot be re-enabled at runtime -- so a default install carries neither
|
|
180
|
+
it nor its eleven transitive distributions. Anyone who reaches here has
|
|
181
|
+
enabled `read` in config.py, and the bare ModuleNotFoundError would name
|
|
182
|
+
a package they never asked for and cannot place.
|
|
183
|
+
"""
|
|
184
|
+
try:
|
|
185
|
+
from markitdown import MarkItDown
|
|
186
|
+
except ImportError as e:
|
|
187
|
+
raise RuntimeError(
|
|
188
|
+
f"Reading {os.path.basename(resolved_path)} needs the optional "
|
|
189
|
+
f"`markitdown` dependency, which a default install does not "
|
|
190
|
+
f"carry (it serves only this path, behind the globally-denied "
|
|
191
|
+
f"`read` tool). Install it with: pip install -e \".[documents]\""
|
|
192
|
+
) from e
|
|
193
|
+
md = MarkItDown()
|
|
194
|
+
result = md.convert(resolved_path)
|
|
195
|
+
return result.text_content
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _truncate_lines(
|
|
199
|
+
lines: list[str],
|
|
200
|
+
offset: int,
|
|
201
|
+
count: int,
|
|
202
|
+
) -> tuple[str, int, bool, Optional[str]]:
|
|
203
|
+
"""Apply offset, clamp count, join, then char-truncate.
|
|
204
|
+
|
|
205
|
+
Returns (content, lines_returned, truncated, message).
|
|
206
|
+
"""
|
|
207
|
+
clamped = count > config.MAX_READ_LINES
|
|
208
|
+
effective_count = min(count, config.MAX_READ_LINES)
|
|
209
|
+
|
|
210
|
+
selected = lines[offset:offset + effective_count]
|
|
211
|
+
lines_returned = len(selected)
|
|
212
|
+
content = "\n".join(selected)
|
|
213
|
+
|
|
214
|
+
truncated = False
|
|
215
|
+
message_parts: list[str] = []
|
|
216
|
+
|
|
217
|
+
if clamped:
|
|
218
|
+
truncated = True
|
|
219
|
+
message_parts.append(
|
|
220
|
+
f"Requested {count} lines, returned {effective_count} (maximum). "
|
|
221
|
+
f"Use offset={offset + effective_count} to read further."
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
if len(content) > config.MAX_READ_CHARS:
|
|
225
|
+
truncated = True
|
|
226
|
+
content = content[:config.MAX_READ_CHARS]
|
|
227
|
+
message_parts.append(
|
|
228
|
+
f"Content truncated at {config.MAX_READ_CHARS} characters."
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
message = " ".join(message_parts) if message_parts else None
|
|
232
|
+
return content, lines_returned, truncated, message
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def read_run(params: dict) -> dict:
|
|
236
|
+
parsed = ReadFileParams(**params)
|
|
237
|
+
refused = _protected_path_error(parsed.path)
|
|
238
|
+
if refused is not None:
|
|
239
|
+
return {"error": refused}
|
|
240
|
+
resolved = resolve_path(parsed.path)
|
|
241
|
+
|
|
242
|
+
if not os.path.exists(resolved):
|
|
243
|
+
return {"error": f"No such file or directory: {parsed.path}"}
|
|
244
|
+
if not os.path.isfile(resolved):
|
|
245
|
+
return {"error": f"Not a regular file: {parsed.path}"}
|
|
246
|
+
|
|
247
|
+
file_size = os.path.getsize(resolved)
|
|
248
|
+
if file_size > config.MAX_FILE_SIZE_BYTES:
|
|
249
|
+
return {
|
|
250
|
+
"error": (
|
|
251
|
+
f"File size ({file_size} bytes) exceeds the maximum "
|
|
252
|
+
f"readable size ({config.MAX_FILE_SIZE_BYTES} bytes)."
|
|
253
|
+
),
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
ext = os.path.splitext(resolved)[1].lower()
|
|
257
|
+
|
|
258
|
+
try:
|
|
259
|
+
if ext in _RICH_EXTENSIONS:
|
|
260
|
+
raw_text = _read_rich(resolved)
|
|
261
|
+
all_lines = raw_text.splitlines()
|
|
262
|
+
else:
|
|
263
|
+
with open(resolved, "r", encoding="utf-8", errors="replace") as f:
|
|
264
|
+
all_lines = f.read().splitlines()
|
|
265
|
+
except Exception as e:
|
|
266
|
+
logger.exception("read tool failed on %s", parsed.path)
|
|
267
|
+
return {"error": f"Could not read file: {e}"}
|
|
268
|
+
|
|
269
|
+
content, lines_returned, truncated, message = _truncate_lines(
|
|
270
|
+
all_lines, parsed.offset, parsed.count,
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
result: dict = {
|
|
274
|
+
"content": content,
|
|
275
|
+
"lines_returned": lines_returned,
|
|
276
|
+
"offset": parsed.offset,
|
|
277
|
+
"truncated": truncated,
|
|
278
|
+
}
|
|
279
|
+
if message:
|
|
280
|
+
result["message"] = message
|
|
281
|
+
return result
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
# ===========================================================================
|
|
285
|
+
# ---- WRITE ----------------------------------------------------------------
|
|
286
|
+
# ===========================================================================
|
|
287
|
+
|
|
288
|
+
class WriteFileParams(BaseModel):
|
|
289
|
+
path: str = Field(..., description="Path relative to the agent's workspace directory, or an absolute path.")
|
|
290
|
+
content: str = Field(..., description="Full content to write — overwrites any existing file.")
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
WRITE_TOOL_SCHEMA = {
|
|
294
|
+
"name": "write",
|
|
295
|
+
"description": (
|
|
296
|
+
"Create or overwrite a file. Parent directories are created "
|
|
297
|
+
"automatically. Content size is limited to "
|
|
298
|
+
f"{config.MAX_FILE_SIZE_BYTES} bytes."
|
|
299
|
+
),
|
|
300
|
+
"input_schema": WriteFileParams.model_json_schema(),
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def write_run(params: dict) -> dict:
|
|
305
|
+
parsed = WriteFileParams(**params)
|
|
306
|
+
|
|
307
|
+
# BEFORE the makedirs below: a refused write must not conjure the
|
|
308
|
+
# protected directory into existence on its way to being refused.
|
|
309
|
+
refused = _protected_path_error(parsed.path)
|
|
310
|
+
if refused is not None:
|
|
311
|
+
return {"error": refused}
|
|
312
|
+
|
|
313
|
+
content_bytes = parsed.content.encode("utf-8")
|
|
314
|
+
if len(content_bytes) > config.MAX_FILE_SIZE_BYTES:
|
|
315
|
+
return {
|
|
316
|
+
"error": (
|
|
317
|
+
f"Content size ({len(content_bytes)} bytes) exceeds the "
|
|
318
|
+
f"maximum writable size ({config.MAX_FILE_SIZE_BYTES} bytes)."
|
|
319
|
+
),
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
resolved = resolve_path(parsed.path)
|
|
323
|
+
os.makedirs(os.path.dirname(resolved), exist_ok=True)
|
|
324
|
+
|
|
325
|
+
try:
|
|
326
|
+
with open(resolved, "w", encoding="utf-8") as f:
|
|
327
|
+
f.write(parsed.content)
|
|
328
|
+
except Exception as e:
|
|
329
|
+
logger.exception("write tool failed on %s", parsed.path)
|
|
330
|
+
return {"error": f"Could not write file: {e}"}
|
|
331
|
+
|
|
332
|
+
return {"result": f"Wrote {len(parsed.content)} characters to {parsed.path}"}
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
# ===========================================================================
|
|
336
|
+
# ---- EDIT -----------------------------------------------------------------
|
|
337
|
+
# ===========================================================================
|
|
338
|
+
|
|
339
|
+
class EditFileParams(BaseModel):
|
|
340
|
+
path: str = Field(..., description="Path relative to the agent's workspace directory, or an absolute path.")
|
|
341
|
+
old_text: str = Field(..., description="Exact text to replace — must appear EXACTLY ONCE in the file.")
|
|
342
|
+
new_text: str = Field(..., description="Replacement text.")
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
EDIT_TOOL_SCHEMA = {
|
|
346
|
+
"name": "edit",
|
|
347
|
+
"description": (
|
|
348
|
+
"Replace an exact, uniquely-matching span of text within an "
|
|
349
|
+
"existing file. old_text must appear exactly once; if it appears "
|
|
350
|
+
"zero or more than one time the edit is rejected with an error."
|
|
351
|
+
),
|
|
352
|
+
"input_schema": EditFileParams.model_json_schema(),
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def edit_run(params: dict) -> dict:
|
|
357
|
+
parsed = EditFileParams(**params)
|
|
358
|
+
refused = _protected_path_error(parsed.path)
|
|
359
|
+
if refused is not None:
|
|
360
|
+
return {"error": refused}
|
|
361
|
+
resolved = resolve_path(parsed.path)
|
|
362
|
+
|
|
363
|
+
if not os.path.isfile(resolved):
|
|
364
|
+
return {"error": f"No such file: {parsed.path}"}
|
|
365
|
+
|
|
366
|
+
# ROADMAP_v2 §31 (H7), #55. read_run has had this check since the
|
|
367
|
+
# beginning and this module's own docstring gives the reason --
|
|
368
|
+
# "rejected before opening to prevent memory exhaustion" -- while
|
|
369
|
+
# the sibling twenty lines down read whatever it was pointed at.
|
|
370
|
+
# Measured on a 30 MB file: 60 MB of peak traced memory to answer
|
|
371
|
+
# "old_text not found in file."
|
|
372
|
+
#
|
|
373
|
+
# Copied rather than reinvented, down to the wording, so the two
|
|
374
|
+
# paths refuse identically. A caller that learns what `read` rejects
|
|
375
|
+
# now knows what `edit` rejects.
|
|
376
|
+
file_size = os.path.getsize(resolved)
|
|
377
|
+
if file_size > config.MAX_FILE_SIZE_BYTES:
|
|
378
|
+
return {
|
|
379
|
+
"error": (
|
|
380
|
+
f"File size ({file_size} bytes) exceeds the maximum "
|
|
381
|
+
f"editable size ({config.MAX_FILE_SIZE_BYTES} bytes)."
|
|
382
|
+
),
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
try:
|
|
386
|
+
with open(resolved, "r", encoding="utf-8") as f:
|
|
387
|
+
content = f.read()
|
|
388
|
+
except Exception as e:
|
|
389
|
+
logger.exception("edit tool read failed on %s", parsed.path)
|
|
390
|
+
return {"error": f"Could not read file: {e}"}
|
|
391
|
+
|
|
392
|
+
match_count = content.count(parsed.old_text)
|
|
393
|
+
if match_count == 0:
|
|
394
|
+
return {"error": "old_text not found in file."}
|
|
395
|
+
if match_count > 1:
|
|
396
|
+
return {
|
|
397
|
+
"error": (
|
|
398
|
+
f"old_text appears {match_count} times — must be unique. "
|
|
399
|
+
"Include more surrounding context to narrow the match."
|
|
400
|
+
),
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
content = content.replace(parsed.old_text, parsed.new_text)
|
|
404
|
+
|
|
405
|
+
try:
|
|
406
|
+
with open(resolved, "w", encoding="utf-8") as f:
|
|
407
|
+
f.write(content)
|
|
408
|
+
except Exception as e:
|
|
409
|
+
logger.exception("edit tool write failed on %s", parsed.path)
|
|
410
|
+
return {"error": f"Could not write file: {e}"}
|
|
411
|
+
|
|
412
|
+
return {"result": f"Replaced 1 occurrence in {parsed.path}"}
|