@yahyahammad/venastine 0.1.0 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -14
- package/CONFIG_ARCHITECTURE.md +2002 -0
- package/NOTICE +1 -1
- package/README.md +249 -93
- package/agents/builtin/build.md +73 -0
- package/agents/builtin/general.md +80 -0
- package/agents/builtin/plan.md +23 -5
- package/agents/builtin/test.md +65 -0
- package/agents/builtin/writer.md +67 -0
- package/agents/manager.py +261 -195
- package/agents/subagent_tool.py +280 -23
- package/agents/tui_commands.py +31 -21
- package/bin/venastine.mjs +528 -382
- package/bin/venastine.py +71 -0
- package/cli_commands.py +835 -0
- package/config.py +95 -1198
- package/config.yaml +1090 -0
- package/config_edit.py +1301 -0
- package/config_schema.py +966 -0
- package/config_update.py +749 -0
- package/core/agent_activity.py +256 -0
- package/core/ansi.py +452 -0
- package/core/approval.py +27 -5
- package/core/call_view.py +364 -0
- package/core/cancel.py +121 -0
- package/core/chatgpt_auth.py +747 -0
- package/core/claims_view.py +107 -0
- package/core/client.py +1249 -82
- package/core/commands.py +454 -0
- package/core/compaction.py +1735 -959
- package/core/config_loader.py +516 -97
- package/core/connect.py +686 -0
- package/core/events.py +68 -6
- package/core/file_mentions.py +280 -0
- package/core/interaction.py +87 -4
- package/core/line_pattern.py +128 -0
- package/core/loop.py +2566 -1353
- package/core/memory.py +482 -370
- package/core/pipeline_models.py +23 -9
- package/core/provider_errors.py +234 -0
- package/core/reasoning/confidence_scoring.py +70 -54
- package/core/reasoning/json_retry.py +19 -1
- package/core/reasoning/orchestrator.py +232 -53
- package/core/reasoning/output_writer.py +90 -31
- package/core/reasoning/payload_validation.py +133 -7
- package/core/reasoning/pipeline_storage.py +427 -361
- package/core/reasoning/review.py +659 -615
- package/core/reasoning/scholar.py +38 -14
- package/core/reasoning/source_corpus.py +110 -4
- package/core/replay.py +315 -167
- package/core/session.py +36 -12
- package/core/session_view.py +259 -0
- package/core/session_wake.py +259 -0
- package/core/shell_sessions.py +1553 -0
- package/core/workspace_trust.py +1 -1
- package/credentials.py +308 -180
- package/database.py +82 -1
- package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
- package/docs/SECURITY.md +108 -0
- package/docs/THIRD_PARTY_NOTICES.md +148 -0
- package/env_secrets.py +118 -77
- package/json_store.py +236 -16
- package/logging_setup.py +121 -11
- package/main.py +1672 -198
- package/mcp_client/client.py +257 -23
- package/mcp_client/config.py +484 -379
- package/mcp_client/registration.py +163 -2
- package/memories/manager.py +15 -0
- package/memories/tui_commands.py +21 -14
- package/package.json +303 -73
- package/project_init/config_files.py +4 -0
- package/project_init/generator.py +201 -37
- package/project_init/manifest.py +418 -406
- package/project_init/tui_commands.py +7 -4
- package/prompts/revalidate.md +1 -1
- package/prompts/revise.md +17 -15
- package/prompts/system_prompts.py +64 -10
- package/providers.json.example +94 -92
- package/pyproject.toml +149 -100
- package/requirements.txt +252 -29
- package/safety/policy_enforcement.py +1082 -682
- package/security/askpass.py +419 -0
- package/security/capability.py +37 -7
- package/security/confine.py +484 -0
- package/security/hosted.py +182 -0
- package/security/permissions.py +54 -7
- package/security/posture.py +124 -12
- package/security/protected_paths.py +198 -6
- package/security/sandbox.py +3262 -139
- package/security/secrets.py +855 -0
- package/skills/tui_commands.py +3 -4
- package/storage.py +1338 -1005
- package/tools/base.py +465 -305
- package/tools/builtin/_math_common.py +306 -16
- package/tools/builtin/_net_common.py +267 -61
- package/tools/builtin/arxiv.py +681 -223
- package/tools/builtin/ask_user.py +231 -213
- package/tools/builtin/discrete_math.py +35 -4
- package/tools/builtin/fetch_url.py +239 -21
- package/tools/builtin/file_ops.py +341 -33
- package/tools/builtin/geometry.py +184 -148
- package/tools/builtin/get_time.py +2 -2
- package/tools/builtin/linear_algebra.py +66 -23
- package/tools/builtin/load_skill.py +10 -0
- package/tools/builtin/logic.py +171 -12
- package/tools/builtin/pin.py +46 -12
- package/tools/builtin/probability_stats.py +224 -228
- package/tools/builtin/project_docs.py +80 -21
- package/tools/builtin/remember.py +21 -4
- package/tools/builtin/shell.py +586 -67
- package/tools/builtin/shell_sessions.py +613 -0
- package/tools/builtin/symbolic_math.py +80 -10
- package/tools/builtin/todo.py +5 -3
- package/tools/builtin/web_search.py +329 -151
- package/tools/context.py +23 -0
- package/tools/isolation.py +387 -314
- package/tools/registry.py +374 -29
- package/tui/app.py +5281 -397
- package/tui/app.tcss +290 -9
- package/tui/commands.py +26 -61
- package/tui/diffs.py +286 -265
- package/tui/history.py +128 -0
- package/tui/hosted_driver.py +102 -0
- package/tui/markdown.py +1329 -0
- package/tui/meters.py +364 -0
- package/tui/screens.py +346 -92
- package/tui/themes.py +599 -366
- package/tui/widgets.py +2673 -116
- package/SECURITY.md +0 -88
- package/THIRD_PARTY_NOTICES.md +0 -100
package/core/events.py
CHANGED
|
@@ -18,10 +18,19 @@ raise) depends on a real exception propagating -- converting failures
|
|
|
18
18
|
into data-shaped events would require every consumer to know to unpack
|
|
19
19
|
and re-raise. Consumers that want graceful error display (the TUI) wrap
|
|
20
20
|
their own consumption loop in try/except.
|
|
21
|
+
|
|
22
|
+
`retract` is NOT an error variant, and batch 113 added it without
|
|
23
|
+
reopening that decision. The rule above is about where a failure is
|
|
24
|
+
HANDLED: a failure must stay an exception so that the persistence which
|
|
25
|
+
depends on one still fires. A retraction says nothing about whether the
|
|
26
|
+
call failed for good -- it says the rows drawn so far are being replaced,
|
|
27
|
+
and the exception still propagates, unchanged, when the retries run out.
|
|
28
|
+
The two coexist because they answer different questions, and a consumer
|
|
29
|
+
that ignores `retract` entirely is still correct about failures.
|
|
21
30
|
"""
|
|
22
31
|
|
|
23
32
|
from dataclasses import dataclass
|
|
24
|
-
from typing import
|
|
33
|
+
from typing import Any, Optional
|
|
25
34
|
|
|
26
35
|
|
|
27
36
|
@dataclass
|
|
@@ -52,10 +61,63 @@ class LoopEvent:
|
|
|
52
61
|
#
|
|
53
62
|
# ALSO carried on the ModelResponse, because run_to_completion()
|
|
54
63
|
# discards every non-final event -- so a notice delivered only here
|
|
55
|
-
# would be invisible to the
|
|
56
|
-
#
|
|
57
|
-
#
|
|
58
|
-
#
|
|
64
|
+
# would be invisible to the pipeline, which drains the generator
|
|
65
|
+
# rather than watching it. That is the same defect §20 and §25 each hit
|
|
66
|
+
# once; the event is for live display, the response field is for
|
|
67
|
+
# everyone else.
|
|
68
|
+
#
|
|
69
|
+
# The CLI drained too until ROADMAP_v3 §86 (A81). It now hands
|
|
70
|
+
# run_to_completion() an observer, prints this event when it arrives,
|
|
71
|
+
# and prints from the response only a notice that never arrived. That
|
|
72
|
+
# is also how a TOOL's notice (§23 J10) reaches it: those ride only
|
|
73
|
+
# this event, never the response.
|
|
59
74
|
notice: Optional[dict] = None
|
|
75
|
+
# Batch 113 (TECHNICAL_DEBT 25). THE NINTH FIELD, and it argues for
|
|
76
|
+
# itself where test_pipeline_events.py says it must. This is not a
|
|
77
|
+
# notice ABOUT the run -- it is an instruction to a drawing surface:
|
|
78
|
+
# take back everything you have drawn since the last flush, because
|
|
79
|
+
# the model call that produced it failed and is being tried again.
|
|
80
|
+
# thinking_delta's family exactly, and P1's rule was never about this
|
|
81
|
+
# case either: it describes one model call's progress and lives for a
|
|
82
|
+
# turn.
|
|
83
|
+
#
|
|
84
|
+
# {"text", "attempt", "attempts"} -- the sentence to show, and the
|
|
85
|
+
# numbers behind it for a consumer that would rather phrase its own.
|
|
86
|
+
#
|
|
87
|
+
# DISPLAY-ONLY, and unlike `notice` it is deliberately NOT mirrored
|
|
88
|
+
# onto ModelResponse. A consumer that drains has drawn nothing and so
|
|
89
|
+
# has nothing to retract; handing it a retraction in the response
|
|
90
|
+
# would be asking it to undo something that never happened. A
|
|
91
|
+
# consumer with no screen correctly ignores this field, which is why
|
|
92
|
+
# the research pipeline translates it into nothing at all.
|
|
93
|
+
#
|
|
94
|
+
# The CLI (ROADMAP_v3 §86, A81) has a screen and still draws nothing
|
|
95
|
+
# a retraction could reach: it prints a step only once the step has
|
|
96
|
+
# finished, which is after the last moment one can arrive. It honours
|
|
97
|
+
# this field by dropping what it was holding.
|
|
98
|
+
retract: Optional[dict] = None
|
|
60
99
|
final_response: Optional[Any] = None # ModelResponse on terminal event
|
|
61
|
-
|
|
100
|
+
# "complete" | "max_steps_reached" | "token_budget_exceeded" |
|
|
101
|
+
# "cancelled"
|
|
102
|
+
#
|
|
103
|
+
# ROADMAP_v3 §61 (CN3) added the fourth, and added NO field: this
|
|
104
|
+
# comment is the only place the set is written down, so a value
|
|
105
|
+
# missing from here is a reader's silent gap -- which is why
|
|
106
|
+
# tests/test_cancel.py asserts all four appear in this file.
|
|
107
|
+
#
|
|
108
|
+
# "cancelled" is NEVER "failed". A failed run is recorded on the
|
|
109
|
+
# thread by mark_turn_failed and reads as something broken; a
|
|
110
|
+
# cancelled one is a person changing their mind. The loop RETURNS on
|
|
111
|
+
# it rather than raising, for the reason the module docstring above
|
|
112
|
+
# gives about exceptions -- an exception here would reach _run's
|
|
113
|
+
# frame and stamp a failure on a turn nobody saw fail, which is the
|
|
114
|
+
# same argument that frame already makes for GeneratorExit.
|
|
115
|
+
#
|
|
116
|
+
# A CONSUMER THAT TREATS EVERY NON-"complete" VALUE AS TRUNCATION IS
|
|
117
|
+
# NOW WRONG, and there is one: core/reasoning/orchestrator.py's
|
|
118
|
+
# _check_truncation continues the pipeline on a truncated pass with
|
|
119
|
+
# text. No research pass can return "cancelled" today (§61 CN12
|
|
120
|
+
# defers the pipeline deliberately), so that reader is correct as it
|
|
121
|
+
# stands -- and it has to be taught the difference before the
|
|
122
|
+
# deferral is lifted.
|
|
123
|
+
stop_reason: Optional[str] = None
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
"""Project path completion and mention grammar (ROADMAP_v3 §54).
|
|
2
|
+
|
|
3
|
+
Only names are indexed. No file contents or access decisions belong here.
|
|
4
|
+
Git is optional; its ignore semantics are used when available. Snapshots are
|
|
5
|
+
bounded and replaced by a worker, never walked by the keystroke handler.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import shutil
|
|
14
|
+
|
|
15
|
+
# Optional git listing uses fixed argv and no shell.
|
|
16
|
+
import subprocess # nosec B404
|
|
17
|
+
import threading
|
|
18
|
+
import time
|
|
19
|
+
import weakref
|
|
20
|
+
from collections import deque
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
|
|
23
|
+
INDEX_LIMIT = 50_000
|
|
24
|
+
MATCH_LIMIT = 200
|
|
25
|
+
SCAN_SECONDS = 5
|
|
26
|
+
GIT_BYTE_LIMIT = 16 * 1024 * 1024
|
|
27
|
+
SKIP_DIRS = frozenset({
|
|
28
|
+
"__pycache__", ".venv", "venv", "node_modules", ".pytest_cache",
|
|
29
|
+
".mypy_cache", ".ruff_cache", "dist", "build",
|
|
30
|
+
})
|
|
31
|
+
_START = re.compile(r'(?<![\w@/\\])@(?=\S|$)')
|
|
32
|
+
_BARE = re.compile(r'[\w./\\:-]+', re.UNICODE)
|
|
33
|
+
_indexes: weakref.WeakSet = weakref.WeakSet()
|
|
34
|
+
_indexes_lock = threading.Lock()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True)
|
|
38
|
+
class Mention:
|
|
39
|
+
start: int
|
|
40
|
+
end: int
|
|
41
|
+
path: str
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def mentions(text: str, *, incomplete: bool = False) -> list[Mention]:
|
|
45
|
+
"""Parse standalone @paths and JSON-quoted @\"paths with spaces\".
|
|
46
|
+
|
|
47
|
+
Bare email addresses and package scopes embedded in a path do not
|
|
48
|
+
trigger. Quoting also permits punctuation in a filename, unambiguously.
|
|
49
|
+
The incomplete form is used only by completion, never by the prompt.
|
|
50
|
+
"""
|
|
51
|
+
found = []
|
|
52
|
+
consumed = 0
|
|
53
|
+
for match in _START.finditer(text):
|
|
54
|
+
if match.start() < consumed:
|
|
55
|
+
continue
|
|
56
|
+
begin = match.end()
|
|
57
|
+
if text[begin:begin + 1] == '"':
|
|
58
|
+
try:
|
|
59
|
+
path, length = json.JSONDecoder().raw_decode(text[begin:])
|
|
60
|
+
end = begin + length
|
|
61
|
+
except ValueError:
|
|
62
|
+
if not incomplete:
|
|
63
|
+
continue
|
|
64
|
+
end = text.find("\n", begin)
|
|
65
|
+
end = len(text) if end < 0 else end
|
|
66
|
+
path = text[begin + 1:end]
|
|
67
|
+
else:
|
|
68
|
+
token = _BARE.match(text, begin)
|
|
69
|
+
end = token.end() if token else begin
|
|
70
|
+
path = text[begin:end]
|
|
71
|
+
if isinstance(path, str) and (path or incomplete):
|
|
72
|
+
found.append(Mention(match.start(), end, path))
|
|
73
|
+
consumed = end
|
|
74
|
+
return found
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def active_mention(text: str, cursor: int) -> Mention | None:
|
|
78
|
+
for item in mentions(text, incomplete=True):
|
|
79
|
+
if item.start < cursor <= item.end:
|
|
80
|
+
return item
|
|
81
|
+
return None
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def quote_path(path: str) -> str:
|
|
85
|
+
return path if _BARE.fullmatch(path) else json.dumps(path, ensure_ascii=False)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@dataclass(frozen=True)
|
|
89
|
+
class FileSuggestion:
|
|
90
|
+
path: str
|
|
91
|
+
prefix: str = "@"
|
|
92
|
+
usage: str | None = None
|
|
93
|
+
|
|
94
|
+
@property
|
|
95
|
+
def name(self) -> str:
|
|
96
|
+
return quote_path(self.path)
|
|
97
|
+
|
|
98
|
+
@property
|
|
99
|
+
def summary(self) -> str:
|
|
100
|
+
return "folder" if self.path.endswith("/") else "file"
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@dataclass(frozen=True)
|
|
104
|
+
class Snapshot:
|
|
105
|
+
visible: tuple[str, ...] = ()
|
|
106
|
+
ignored: tuple[str, ...] = ()
|
|
107
|
+
source: str = "indexing"
|
|
108
|
+
limited: bool = False
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _git_paths(root: str, ignored: bool):
|
|
112
|
+
"""Stream bounded NUL records; fixed argv, no shell or fsmonitor hook."""
|
|
113
|
+
git = shutil.which("git")
|
|
114
|
+
if not git:
|
|
115
|
+
raise OSError("git unavailable")
|
|
116
|
+
args = [git, "-c", "core.fsmonitor=false", "ls-files", "-z",
|
|
117
|
+
"--others", "--exclude-standard"]
|
|
118
|
+
args += ["--ignored"] if ignored else ["--cached"]
|
|
119
|
+
# B603: executable from PATH, fixed options; root is cwd, never argv.
|
|
120
|
+
process = subprocess.Popen( # nosec B603
|
|
121
|
+
args, cwd=root, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
|
|
122
|
+
expired = threading.Event()
|
|
123
|
+
|
|
124
|
+
def stop():
|
|
125
|
+
expired.set()
|
|
126
|
+
try:
|
|
127
|
+
process.kill()
|
|
128
|
+
except OSError: # process exited as the timer fired
|
|
129
|
+
pass
|
|
130
|
+
|
|
131
|
+
timer = threading.Timer(SCAN_SECONDS, stop)
|
|
132
|
+
timer.daemon = True
|
|
133
|
+
timer.start()
|
|
134
|
+
try:
|
|
135
|
+
pending = b""
|
|
136
|
+
received = 0
|
|
137
|
+
while chunk := process.stdout.read1(65536):
|
|
138
|
+
received += len(chunk)
|
|
139
|
+
if received > GIT_BYTE_LIMIT:
|
|
140
|
+
raise TimeoutError("git listing byte limit")
|
|
141
|
+
records = (pending + chunk).split(b"\0")
|
|
142
|
+
pending = records.pop()
|
|
143
|
+
for raw in records:
|
|
144
|
+
yield os.fsdecode(raw)
|
|
145
|
+
if process.wait() or expired.is_set():
|
|
146
|
+
raise OSError("git listing failed or timed out")
|
|
147
|
+
finally:
|
|
148
|
+
timer.cancel()
|
|
149
|
+
if process.poll() is None:
|
|
150
|
+
process.kill()
|
|
151
|
+
process.wait()
|
|
152
|
+
process.stdout.close()
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _safe_name(path: str) -> bool:
|
|
156
|
+
return (bool(path) and not os.path.isabs(path)
|
|
157
|
+
and not any(part in ("..", ".git") for part in path.split("/"))
|
|
158
|
+
and not any(ord(char) < 32 or 127 <= ord(char) < 160
|
|
159
|
+
or 0xD800 <= ord(char) < 0xE000 for char in path))
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def scan(root: str) -> Snapshot:
|
|
163
|
+
visible, ignored = set(), set()
|
|
164
|
+
limited = False
|
|
165
|
+
|
|
166
|
+
def add(path, hidden):
|
|
167
|
+
nonlocal limited
|
|
168
|
+
path = path.replace(os.sep, "/")
|
|
169
|
+
if not _safe_name(path):
|
|
170
|
+
return
|
|
171
|
+
target = ignored if hidden else visible
|
|
172
|
+
parts = path.rstrip("/").split("/")
|
|
173
|
+
entries = ["/".join(parts[:n]) + "/" for n in range(1, len(parts))]
|
|
174
|
+
entries.append(path)
|
|
175
|
+
for entry in entries:
|
|
176
|
+
if entry in visible or entry in ignored:
|
|
177
|
+
continue
|
|
178
|
+
if len(visible) + len(ignored) >= INDEX_LIMIT:
|
|
179
|
+
limited = True
|
|
180
|
+
return
|
|
181
|
+
target.add(entry)
|
|
182
|
+
|
|
183
|
+
try:
|
|
184
|
+
for hidden in (False, True):
|
|
185
|
+
listing = _git_paths(root, hidden)
|
|
186
|
+
try:
|
|
187
|
+
for path in listing:
|
|
188
|
+
add(path, hidden)
|
|
189
|
+
if limited:
|
|
190
|
+
break
|
|
191
|
+
finally:
|
|
192
|
+
listing.close()
|
|
193
|
+
if limited:
|
|
194
|
+
break
|
|
195
|
+
source = "git ignores"
|
|
196
|
+
except (OSError, TimeoutError):
|
|
197
|
+
visible.clear()
|
|
198
|
+
ignored.clear()
|
|
199
|
+
limited = False
|
|
200
|
+
source = "fallback skips"
|
|
201
|
+
pending = deque([(root, False)])
|
|
202
|
+
deadline = time.monotonic() + SCAN_SECONDS
|
|
203
|
+
visited = 0
|
|
204
|
+
while pending and not limited:
|
|
205
|
+
directory, hidden = pending.popleft()
|
|
206
|
+
try:
|
|
207
|
+
with os.scandir(directory) as entries:
|
|
208
|
+
for entry in entries:
|
|
209
|
+
visited += 1
|
|
210
|
+
if visited > INDEX_LIMIT or time.monotonic() >= deadline:
|
|
211
|
+
limited = True
|
|
212
|
+
break
|
|
213
|
+
if entry.name.casefold() == ".git":
|
|
214
|
+
continue
|
|
215
|
+
is_dir = entry.is_dir(follow_symlinks=False)
|
|
216
|
+
child_hidden = hidden or (is_dir and entry.name in SKIP_DIRS)
|
|
217
|
+
rel = os.path.relpath(entry.path, root).replace(os.sep, "/")
|
|
218
|
+
add(rel + ("/" if is_dir else ""), child_hidden)
|
|
219
|
+
if is_dir:
|
|
220
|
+
pending.append((entry.path, child_hidden))
|
|
221
|
+
if limited:
|
|
222
|
+
break
|
|
223
|
+
except OSError:
|
|
224
|
+
# An unreadable subtree makes the snapshot partial, visibly.
|
|
225
|
+
limited = True
|
|
226
|
+
return Snapshot(tuple(sorted(visible)), tuple(sorted(ignored)), source, limited)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
class FileIndex:
|
|
230
|
+
def __init__(self, root: str | None):
|
|
231
|
+
self.root = os.path.realpath(root) if root else None
|
|
232
|
+
self.snapshot = Snapshot(source="indexing" if root else "no project")
|
|
233
|
+
self.show_ignored = False
|
|
234
|
+
self.dirty = threading.Event()
|
|
235
|
+
if root:
|
|
236
|
+
self.dirty.set()
|
|
237
|
+
with _indexes_lock:
|
|
238
|
+
_indexes.add(self)
|
|
239
|
+
|
|
240
|
+
def refresh(self):
|
|
241
|
+
self.dirty.clear()
|
|
242
|
+
if self.root:
|
|
243
|
+
self.snapshot = scan(self.root)
|
|
244
|
+
|
|
245
|
+
def matching(self, query: str) -> tuple[list[FileSuggestion], str]:
|
|
246
|
+
snapshot = self.snapshot
|
|
247
|
+
query = query.replace("\\", "/").casefold()
|
|
248
|
+
paths = snapshot.visible + (snapshot.ignored if self.show_ignored else ())
|
|
249
|
+
# Keep bounded prefix/substring buckets instead of sorting 50,000
|
|
250
|
+
# rows at every keystroke. Snapshot order breaks ties predictably.
|
|
251
|
+
prefixes, others = [], []
|
|
252
|
+
total = 0
|
|
253
|
+
for path in paths:
|
|
254
|
+
folded = path.casefold()
|
|
255
|
+
if query not in folded:
|
|
256
|
+
continue
|
|
257
|
+
total += 1
|
|
258
|
+
starts = folded.startswith(query) or folded.rstrip("/").rsplit("/", 1)[-1].startswith(query)
|
|
259
|
+
bucket = prefixes if starts else others
|
|
260
|
+
if len(bucket) < MATCH_LIMIT:
|
|
261
|
+
bucket.append(FileSuggestion(path))
|
|
262
|
+
notices = []
|
|
263
|
+
if total > MATCH_LIMIT:
|
|
264
|
+
notices.append(f"first {MATCH_LIMIT} of {total} matches")
|
|
265
|
+
if snapshot.limited:
|
|
266
|
+
notices.append(f"partial index (limit {INDEX_LIMIT:,})")
|
|
267
|
+
notices += [snapshot.source, f"ignored {'shown' if self.show_ignored else 'hidden'}"]
|
|
268
|
+
return (prefixes + others)[:MATCH_LIMIT], " · ".join(notices)
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def note_file_created(path: str) -> None:
|
|
272
|
+
"""Called only after write succeeds; subagents use the same producer."""
|
|
273
|
+
with _indexes_lock:
|
|
274
|
+
indexes = list(_indexes)
|
|
275
|
+
for index in indexes:
|
|
276
|
+
try:
|
|
277
|
+
if os.path.commonpath((index.root, path)) == index.root:
|
|
278
|
+
index.dirty.set()
|
|
279
|
+
except ValueError: # different drives
|
|
280
|
+
continue
|
package/core/interaction.py
CHANGED
|
@@ -39,13 +39,58 @@ Deliberately a leaf module with no project imports, matching core/approval.py
|
|
|
39
39
|
it depends on none of them.
|
|
40
40
|
"""
|
|
41
41
|
|
|
42
|
-
from dataclasses import dataclass, field
|
|
43
|
-
from typing import Any, Callable, Optional
|
|
44
|
-
|
|
45
42
|
import logging
|
|
43
|
+
import threading
|
|
44
|
+
from collections.abc import Callable
|
|
45
|
+
from dataclasses import dataclass, field
|
|
46
|
+
from typing import Any, Optional
|
|
46
47
|
|
|
47
48
|
logger = logging.getLogger(__name__)
|
|
48
49
|
|
|
50
|
+
# ROADMAP_v2 §47 slice 8 (NA12). ONE QUESTION ON SCREEN AT A TIME, and it
|
|
51
|
+
# lives here because this module is already the one entry point every
|
|
52
|
+
# question goes through.
|
|
53
|
+
#
|
|
54
|
+
# Until slice 8 nothing could ask twice at once, so no shell needed to cope
|
|
55
|
+
# with it and none does. Both of ours break in the same way and for the
|
|
56
|
+
# same reason -- a single slot for the pending answer. `tui/app.py` keeps
|
|
57
|
+
# one `_permission_channel` so shutdown can release a parked worker: a
|
|
58
|
+
# second ask overwrites the field and orphans the first, and the first ask
|
|
59
|
+
# to FINISH nulls it in its `finally`, so a still-open question becomes
|
|
60
|
+
# unreleasable. `main.py` has one `_StdinReader`, and two readers of one
|
|
61
|
+
# stdin is the defect §29's N1 exists to prevent, arrived at from the other
|
|
62
|
+
# direction. Two consent surfaces at once is also what §42 refuses on its
|
|
63
|
+
# own terms.
|
|
64
|
+
#
|
|
65
|
+
# HERE RATHER THAN IN EACH SHELL, which is this project's canonical rule --
|
|
66
|
+
# fix at the producer. A lock around `tui/app.py`'s `_blocking_modal` would
|
|
67
|
+
# leave the CLI unprotected and would have to be written again for the
|
|
68
|
+
# third shell.
|
|
69
|
+
#
|
|
70
|
+
# NO TIMEOUT, deliberately. Every waiter is bounded by the holder's own
|
|
71
|
+
# deadline ONCE THE HOLDER IS PARKED: the TUI parks for at most
|
|
72
|
+
# ATTENDED_APPROVAL_TIMEOUT_S and a run-backed CLI prompt carries the same,
|
|
73
|
+
# so the queue drains on its own. The step before the TUI's park has no
|
|
74
|
+
# deadline of its own, and until #223 this paragraph read as if it had:
|
|
75
|
+
# `_blocking_modal` first marshals its screen onto the UI thread, which is
|
|
76
|
+
# bounded only by that thread staying responsive -- the bound every other
|
|
77
|
+
# marshal in the shell lives with. Measured under textual 8.2.8 (#220), a
|
|
78
|
+
# marshal after the app exits returns or raises at once rather than
|
|
79
|
+
# parking, so what is left unbounded is a UI thread that is alive and
|
|
80
|
+
# wedged, where nothing else in the shell answers either.
|
|
81
|
+
# A deadline here would need this leaf module to know a number that
|
|
82
|
+
# belongs to config.py, and the failure it would prevent is one no shell
|
|
83
|
+
# can currently produce. Shutdown does not wait either: the TUI's release
|
|
84
|
+
# unblocks the holder, and every queued worker then finds `_shutting_down`
|
|
85
|
+
# already true and returns its declining default without pushing a screen.
|
|
86
|
+
#
|
|
87
|
+
# NOT FIFO, and nothing here claims it is -- a Lock makes no ordering
|
|
88
|
+
# promise, so which of two simultaneous questions is put first is
|
|
89
|
+
# undefined. What IS guaranteed is that each answer reaches the asker that
|
|
90
|
+
# holds it, because the request travels with the caller and never through
|
|
91
|
+
# this module's state.
|
|
92
|
+
_ask_lock = threading.Lock()
|
|
93
|
+
|
|
49
94
|
|
|
50
95
|
# ---------------------------------------------------------------------------
|
|
51
96
|
# ---- Kinds ----------------------------------------------------------------
|
|
@@ -66,6 +111,16 @@ CHOICE = "choice"
|
|
|
66
111
|
# four affordances the spec asks for by name, and a shape CHOICE's decoder
|
|
67
112
|
# cannot express without becoming two decoders wearing one name.
|
|
68
113
|
QUESTION = "question"
|
|
114
|
+
# ROADMAP_v3 §49 slice 5a (SS59). One value, typed by a human, that the
|
|
115
|
+
# caller needs and this module must never see printed.
|
|
116
|
+
#
|
|
117
|
+
# Its own kind rather than a QUESTION with a flag, because the KIND is what
|
|
118
|
+
# tells a shell how to RENDER -- and the rendering is the whole point here:
|
|
119
|
+
# the field is masked, the answer is not echoed, and it does not reach the
|
|
120
|
+
# transcript, the log or the scrollback. A flag on an existing kind would
|
|
121
|
+
# make a shell that ignored the flag look wired up while showing the
|
|
122
|
+
# passphrase in clear.
|
|
123
|
+
SECRET = "secret"
|
|
69
124
|
|
|
70
125
|
# Review decisions, moved here from core/reasoning/review.py so the decoder
|
|
71
126
|
# and the pipeline cannot disagree about what a valid answer is.
|
|
@@ -105,6 +160,14 @@ SAFE_DEFAULTS = {
|
|
|
105
160
|
# answer carrying defer=True, because a user who says that has told the
|
|
106
161
|
# model something, and a user who closed the modal has not.
|
|
107
162
|
QUESTION: None,
|
|
163
|
+
# None, for CHOICE's reason and one more that is specific to this kind:
|
|
164
|
+
# the empty string is a value a user could legitimately mean to type,
|
|
165
|
+
# and it is also what a torn-down modal and a shell with no branch for
|
|
166
|
+
# SECRET both produce. Defaulting to "" would hand `sudo` an empty
|
|
167
|
+
# password -- an authentication ATTEMPT made on nobody's behalf, which
|
|
168
|
+
# on a remote account is a step towards a lockout. None is not a
|
|
169
|
+
# password, so every consumer has to notice.
|
|
170
|
+
SECRET: None,
|
|
108
171
|
}
|
|
109
172
|
|
|
110
173
|
|
|
@@ -191,6 +254,21 @@ def decode(request: "Request", raw: Any) -> Any:
|
|
|
191
254
|
# third state to confuse it with.
|
|
192
255
|
return bool(raw)
|
|
193
256
|
|
|
257
|
+
if kind == SECRET:
|
|
258
|
+
# A string, or nothing. NOT `str(raw)`: a shell that returned a
|
|
259
|
+
# widget, a dict or a bool would be coerced into a "password" that
|
|
260
|
+
# is really a repr, and it would then be TRIED against a real
|
|
261
|
+
# account. And not `raw or None` either -- an empty string is the
|
|
262
|
+
# shape a dismissed modal produces, and it must decode as "nobody
|
|
263
|
+
# answered" rather than as a password of length zero.
|
|
264
|
+
#
|
|
265
|
+
# The value is returned raw and is NOT logged, here or anywhere on
|
|
266
|
+
# this path. This is the one decode branch whose input must not
|
|
267
|
+
# appear in a warning about its own shape.
|
|
268
|
+
if not isinstance(raw, str) or raw == "":
|
|
269
|
+
return None
|
|
270
|
+
return raw
|
|
271
|
+
|
|
194
272
|
if kind == CHOICE:
|
|
195
273
|
# Valid ONLY against what was offered. A shell returning something
|
|
196
274
|
# nobody proposed is answering a different question, and the
|
|
@@ -314,7 +392,12 @@ def ask(channel: Optional[ResponseChannel], request: Request) -> Any:
|
|
|
314
392
|
if channel is None:
|
|
315
393
|
return decode(request, None)
|
|
316
394
|
try:
|
|
317
|
-
|
|
395
|
+
# NA12. Serialised, so a shell that can only hold one pending
|
|
396
|
+
# answer is never handed two. The lock covers the shell's call and
|
|
397
|
+
# nothing else: `decode` is pure, and holding a lock across it
|
|
398
|
+
# would make one shell's slowness another's wait for no reason.
|
|
399
|
+
with _ask_lock:
|
|
400
|
+
raw = channel.ask(request)
|
|
318
401
|
except Exception: # noqa: BLE001 -- a shell's failure is not the run's
|
|
319
402
|
# Logged with the traceback, because a shell raising here is a real
|
|
320
403
|
# bug worth finding, and at ERROR because the user's answer was
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""
|
|
2
|
+
core/line_pattern.py
|
|
3
|
+
|
|
4
|
+
The one place a model-supplied pattern is compiled and matched (ROADMAP_v3
|
|
5
|
+
§49, SS8). A monitor session wakes the agent when a line of a program's
|
|
6
|
+
output matches a pattern the MODEL wrote -- two untrusted inputs meeting
|
|
7
|
+
inside the harness's own process, on a thread beside the TUI's.
|
|
8
|
+
|
|
9
|
+
WHY NOT `re`. Measured on CPython 3.13 (batch 93): `re.match(r"(a+)+$",
|
|
10
|
+
"a" * 25 + "b")` took 1.88 s, and a sibling thread's ticker stalled for
|
|
11
|
+
1.98 s of it. `re` holds the GIL for the whole match, so a backtracking
|
|
12
|
+
pattern freezes EVERY thread, and each extra character doubles the time. No
|
|
13
|
+
per-line budget enforced from inside the same process can bound that.
|
|
14
|
+
|
|
15
|
+
re2 matches in time linear in the input, releases the GIL while it runs,
|
|
16
|
+
and refuses the constructs that make backtracking possible: a backreference
|
|
17
|
+
or a lookaround is a compile error, and the error is what the agent is told.
|
|
18
|
+
|
|
19
|
+
THREE GUARDS, each measured against google-re2 1.1.20251105:
|
|
20
|
+
|
|
21
|
+
provenance another package ships a module named `re2` that falls back to
|
|
22
|
+
`re` for what it cannot handle, which would bring the freeze
|
|
23
|
+
back under a name that looks safe. The module must be the one
|
|
24
|
+
the `google-re2` distribution installed, or this module
|
|
25
|
+
refuses to import. Checked by locating that distribution's
|
|
26
|
+
own `re2/__init__.py` (0.6 ms), not by
|
|
27
|
+
`packages_distributions()`, which scans every installed
|
|
28
|
+
package (593 ms, on a path every launch takes).
|
|
29
|
+
log_errors OFF. re2's C++ side reports every compile error on stderr
|
|
30
|
+
through Abseil, and anything written to stderr while Textual
|
|
31
|
+
is up paints over the screen.
|
|
32
|
+
max_mem set explicitly to re2's own 8 MiB default, so the bound on
|
|
33
|
+
what one compiled pattern may cost is a decision rather than
|
|
34
|
+
a library default that can move. A pattern past it is refused
|
|
35
|
+
("pattern too large").
|
|
36
|
+
|
|
37
|
+
Imports nothing first-party, so a session backend or a tool can use it
|
|
38
|
+
without an import cycle.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
from __future__ import annotations
|
|
42
|
+
|
|
43
|
+
import importlib.metadata
|
|
44
|
+
import os
|
|
45
|
+
|
|
46
|
+
import re2
|
|
47
|
+
|
|
48
|
+
_DISTRIBUTION = "google-re2"
|
|
49
|
+
_MAX_MEM = 8 << 20
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class PatternError(ValueError):
|
|
53
|
+
"""A pattern re2 will not compile, carrying re2's own reason."""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _assert_provenance() -> None:
|
|
57
|
+
"""Refuse a `re2` module that the `google-re2` distribution did not
|
|
58
|
+
install. Raises ImportError, because a module that cannot vouch for its
|
|
59
|
+
engine should fail where it is imported rather than at the first match."""
|
|
60
|
+
try:
|
|
61
|
+
dist = importlib.metadata.distribution(_DISTRIBUTION)
|
|
62
|
+
expected = os.path.realpath(str(dist.locate_file("re2/__init__.py")))
|
|
63
|
+
except importlib.metadata.PackageNotFoundError:
|
|
64
|
+
expected = None
|
|
65
|
+
actual = os.path.realpath(getattr(re2, "__file__", "") or "")
|
|
66
|
+
if expected is None or expected != actual:
|
|
67
|
+
raise ImportError(
|
|
68
|
+
f"core/line_pattern.py needs the `re2` module from the "
|
|
69
|
+
f"{_DISTRIBUTION} distribution, and the one imported is "
|
|
70
|
+
f"{actual or 'unknown'}. Another package's `re2` can fall back "
|
|
71
|
+
f"to Python's `re`, which freezes every thread on a backtracking "
|
|
72
|
+
f"pattern. Install it with: pip install {_DISTRIBUTION}")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
_assert_provenance()
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _options():
|
|
79
|
+
options = re2.Options()
|
|
80
|
+
options.log_errors = False
|
|
81
|
+
options.max_mem = _MAX_MEM
|
|
82
|
+
return options
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _reason(error: Exception) -> str:
|
|
86
|
+
"""re2's message as text. The wrapper raises it as bytes."""
|
|
87
|
+
detail = error.args[0] if error.args else ""
|
|
88
|
+
if isinstance(detail, bytes):
|
|
89
|
+
detail = detail.decode("utf-8", "replace")
|
|
90
|
+
return str(detail) or "the pattern could not be compiled"
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class LinePattern:
|
|
94
|
+
"""A compiled monitor pattern. `search` is the only question asked."""
|
|
95
|
+
|
|
96
|
+
__slots__ = ("source", "_compiled")
|
|
97
|
+
|
|
98
|
+
def __init__(self, source: str, compiled) -> None:
|
|
99
|
+
self.source = source
|
|
100
|
+
self._compiled = compiled
|
|
101
|
+
|
|
102
|
+
def search(self, line: str) -> bool:
|
|
103
|
+
"""Whether the pattern occurs anywhere in *line*."""
|
|
104
|
+
return self._compiled.search(line) is not None
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def compile_pattern(pattern) -> LinePattern:
|
|
108
|
+
"""Compile *pattern* for line matching, or raise PatternError.
|
|
109
|
+
|
|
110
|
+
Total over whatever the model sent: the value arrives from a tool call's
|
|
111
|
+
params, so a non-string is refused here rather than raising a TypeError
|
|
112
|
+
out of re2.
|
|
113
|
+
"""
|
|
114
|
+
if not isinstance(pattern, str) or not pattern:
|
|
115
|
+
raise PatternError("a pattern must be a non-empty string")
|
|
116
|
+
try:
|
|
117
|
+
return LinePattern(pattern, re2.compile(pattern, _options()))
|
|
118
|
+
except re2.error as e:
|
|
119
|
+
raise PatternError(_reason(e)) from None
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def validate(pattern) -> str | None:
|
|
123
|
+
"""Why *pattern* cannot be a monitor pattern, or None when it can."""
|
|
124
|
+
try:
|
|
125
|
+
compile_pattern(pattern)
|
|
126
|
+
except PatternError as e:
|
|
127
|
+
return str(e)
|
|
128
|
+
return None
|