@yahyahammad/venastine 0.2.0 → 0.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/.env.example +19 -14
  2. package/CONFIG_ARCHITECTURE.md +2002 -0
  3. package/NOTICE +1 -1
  4. package/README.md +233 -94
  5. package/agents/builtin/build.md +73 -0
  6. package/agents/builtin/general.md +80 -0
  7. package/agents/builtin/plan.md +23 -5
  8. package/agents/builtin/test.md +65 -0
  9. package/agents/builtin/writer.md +67 -0
  10. package/agents/manager.py +261 -195
  11. package/agents/subagent_tool.py +277 -43
  12. package/agents/tui_commands.py +7 -12
  13. package/bin/venastine.mjs +528 -382
  14. package/bin/venastine.py +71 -0
  15. package/cli_commands.py +835 -0
  16. package/config.py +95 -1198
  17. package/config.yaml +1090 -0
  18. package/config_edit.py +1301 -0
  19. package/config_schema.py +966 -0
  20. package/config_update.py +749 -0
  21. package/core/agent_activity.py +158 -20
  22. package/core/ansi.py +452 -0
  23. package/core/approval.py +27 -5
  24. package/core/call_view.py +364 -0
  25. package/core/cancel.py +121 -0
  26. package/core/chatgpt_auth.py +747 -0
  27. package/core/claims_view.py +107 -0
  28. package/core/client.py +1238 -82
  29. package/core/commands.py +454 -0
  30. package/core/compaction.py +830 -66
  31. package/core/config_loader.py +516 -97
  32. package/core/connect.py +686 -0
  33. package/core/events.py +68 -6
  34. package/core/file_mentions.py +280 -0
  35. package/core/interaction.py +87 -4
  36. package/core/line_pattern.py +128 -0
  37. package/core/loop.py +1516 -440
  38. package/core/memory.py +120 -8
  39. package/core/pipeline_models.py +23 -9
  40. package/core/provider_errors.py +234 -0
  41. package/core/reasoning/confidence_scoring.py +70 -54
  42. package/core/reasoning/json_retry.py +19 -1
  43. package/core/reasoning/orchestrator.py +232 -53
  44. package/core/reasoning/output_writer.py +90 -31
  45. package/core/reasoning/payload_validation.py +133 -7
  46. package/core/reasoning/pipeline_storage.py +427 -361
  47. package/core/reasoning/review.py +659 -615
  48. package/core/reasoning/scholar.py +38 -14
  49. package/core/reasoning/source_corpus.py +110 -4
  50. package/core/replay.py +158 -26
  51. package/core/session.py +36 -12
  52. package/core/session_view.py +259 -0
  53. package/core/session_wake.py +259 -0
  54. package/core/shell_sessions.py +1553 -0
  55. package/core/workspace_trust.py +1 -1
  56. package/credentials.py +308 -180
  57. package/database.py +82 -1
  58. package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
  59. package/docs/SECURITY.md +108 -0
  60. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  61. package/env_secrets.py +118 -77
  62. package/json_store.py +236 -16
  63. package/logging_setup.py +121 -11
  64. package/main.py +1671 -199
  65. package/mcp_client/client.py +257 -23
  66. package/mcp_client/config.py +484 -379
  67. package/mcp_client/registration.py +163 -2
  68. package/memories/manager.py +15 -0
  69. package/memories/tui_commands.py +7 -12
  70. package/package.json +234 -4
  71. package/project_init/config_files.py +4 -0
  72. package/project_init/generator.py +201 -37
  73. package/project_init/manifest.py +418 -406
  74. package/project_init/tui_commands.py +7 -4
  75. package/prompts/revalidate.md +1 -1
  76. package/prompts/revise.md +17 -15
  77. package/prompts/system_prompts.py +64 -10
  78. package/providers.json.example +94 -92
  79. package/pyproject.toml +51 -2
  80. package/requirements.txt +252 -29
  81. package/safety/policy_enforcement.py +353 -17
  82. package/security/askpass.py +419 -0
  83. package/security/capability.py +37 -7
  84. package/security/confine.py +484 -0
  85. package/security/hosted.py +182 -0
  86. package/security/permissions.py +54 -7
  87. package/security/posture.py +124 -12
  88. package/security/protected_paths.py +198 -6
  89. package/security/sandbox.py +3262 -139
  90. package/security/secrets.py +855 -0
  91. package/skills/tui_commands.py +3 -4
  92. package/storage.py +368 -35
  93. package/tools/base.py +465 -305
  94. package/tools/builtin/_math_common.py +306 -16
  95. package/tools/builtin/_net_common.py +267 -61
  96. package/tools/builtin/arxiv.py +681 -223
  97. package/tools/builtin/ask_user.py +231 -213
  98. package/tools/builtin/discrete_math.py +35 -4
  99. package/tools/builtin/fetch_url.py +239 -21
  100. package/tools/builtin/file_ops.py +341 -33
  101. package/tools/builtin/geometry.py +184 -148
  102. package/tools/builtin/get_time.py +2 -2
  103. package/tools/builtin/linear_algebra.py +66 -23
  104. package/tools/builtin/load_skill.py +10 -0
  105. package/tools/builtin/logic.py +171 -12
  106. package/tools/builtin/pin.py +46 -12
  107. package/tools/builtin/probability_stats.py +224 -228
  108. package/tools/builtin/project_docs.py +80 -21
  109. package/tools/builtin/remember.py +21 -4
  110. package/tools/builtin/shell.py +586 -67
  111. package/tools/builtin/shell_sessions.py +613 -0
  112. package/tools/builtin/symbolic_math.py +80 -10
  113. package/tools/builtin/todo.py +5 -3
  114. package/tools/builtin/web_search.py +329 -151
  115. package/tools/context.py +23 -0
  116. package/tools/isolation.py +387 -314
  117. package/tools/registry.py +347 -30
  118. package/tui/app.py +4645 -438
  119. package/tui/app.tcss +185 -2
  120. package/tui/commands.py +10 -154
  121. package/tui/diffs.py +286 -265
  122. package/tui/hosted_driver.py +102 -0
  123. package/tui/markdown.py +60 -10
  124. package/tui/meters.py +103 -7
  125. package/tui/screens.py +346 -92
  126. package/tui/themes.py +59 -19
  127. package/tui/widgets.py +1575 -111
  128. package/SECURITY.md +0 -88
  129. package/THIRD_PARTY_NOTICES.md +0 -100
package/core/events.py CHANGED
@@ -18,10 +18,19 @@ raise) depends on a real exception propagating -- converting failures
18
18
  into data-shaped events would require every consumer to know to unpack
19
19
  and re-raise. Consumers that want graceful error display (the TUI) wrap
20
20
  their own consumption loop in try/except.
21
+
22
+ `retract` is NOT an error variant, and batch 113 added it without
23
+ reopening that decision. The rule above is about where a failure is
24
+ HANDLED: a failure must stay an exception so that the persistence which
25
+ depends on one still fires. A retraction says nothing about whether the
26
+ call failed for good -- it says the rows drawn so far are being replaced,
27
+ and the exception still propagates, unchanged, when the retries run out.
28
+ The two coexist because they answer different questions, and a consumer
29
+ that ignores `retract` entirely is still correct about failures.
21
30
  """
22
31
 
23
32
  from dataclasses import dataclass
24
- from typing import Optional, Any
33
+ from typing import Any, Optional
25
34
 
26
35
 
27
36
  @dataclass
@@ -52,10 +61,63 @@ class LoopEvent:
52
61
  #
53
62
  # ALSO carried on the ModelResponse, because run_to_completion()
54
63
  # discards every non-final event -- so a notice delivered only here
55
- # would be invisible to the CLI and the pipeline, which drain the
56
- # generator rather than watching it. That is the same defect §20 and
57
- # §25 each hit once; the event is for live display, the response field
58
- # is for everyone else.
64
+ # would be invisible to the pipeline, which drains the generator
65
+ # rather than watching it. That is the same defect §20 and §25 each hit
66
+ # once; the event is for live display, the response field is for
67
+ # everyone else.
68
+ #
69
+ # The CLI drained too until ROADMAP_v3 §86 (A81). It now hands
70
+ # run_to_completion() an observer, prints this event when it arrives,
71
+ # and prints from the response only a notice that never arrived. That
72
+ # is also how a TOOL's notice (§23 J10) reaches it: those ride only
73
+ # this event, never the response.
59
74
  notice: Optional[dict] = None
75
+ # Batch 113 (TECHNICAL_DEBT 25). THE NINTH FIELD, and it argues for
76
+ # itself where test_pipeline_events.py says it must. This is not a
77
+ # notice ABOUT the run -- it is an instruction to a drawing surface:
78
+ # take back everything you have drawn since the last flush, because
79
+ # the model call that produced it failed and is being tried again.
80
+ # thinking_delta's family exactly, and P1's rule was never about this
81
+ # case either: it describes one model call's progress and lives for a
82
+ # turn.
83
+ #
84
+ # {"text", "attempt", "attempts"} -- the sentence to show, and the
85
+ # numbers behind it for a consumer that would rather phrase its own.
86
+ #
87
+ # DISPLAY-ONLY, and unlike `notice` it is deliberately NOT mirrored
88
+ # onto ModelResponse. A consumer that drains has drawn nothing and so
89
+ # has nothing to retract; handing it a retraction in the response
90
+ # would be asking it to undo something that never happened. A
91
+ # consumer with no screen correctly ignores this field, which is why
92
+ # the research pipeline translates it into nothing at all.
93
+ #
94
+ # The CLI (ROADMAP_v3 §86, A81) has a screen and still draws nothing
95
+ # a retraction could reach: it prints a step only once the step has
96
+ # finished, which is after the last moment one can arrive. It honours
97
+ # this field by dropping what it was holding.
98
+ retract: Optional[dict] = None
60
99
  final_response: Optional[Any] = None # ModelResponse on terminal event
61
- stop_reason: Optional[str] = None # "complete" | "max_steps_reached" | "token_budget_exceeded"
100
+ # "complete" | "max_steps_reached" | "token_budget_exceeded" |
101
+ # "cancelled"
102
+ #
103
+ # ROADMAP_v3 §61 (CN3) added the fourth, and added NO field: this
104
+ # comment is the only place the set is written down, so a value
105
+ # missing from here is a reader's silent gap -- which is why
106
+ # tests/test_cancel.py asserts all four appear in this file.
107
+ #
108
+ # "cancelled" is NEVER "failed". A failed run is recorded on the
109
+ # thread by mark_turn_failed and reads as something broken; a
110
+ # cancelled one is a person changing their mind. The loop RETURNS on
111
+ # it rather than raising, for the reason the module docstring above
112
+ # gives about exceptions -- an exception here would reach _run's
113
+ # frame and stamp a failure on a turn nobody saw fail, which is the
114
+ # same argument that frame already makes for GeneratorExit.
115
+ #
116
+ # A CONSUMER THAT TREATS EVERY NON-"complete" VALUE AS TRUNCATION IS
117
+ # NOW WRONG, and there is one: core/reasoning/orchestrator.py's
118
+ # _check_truncation continues the pipeline on a truncated pass with
119
+ # text. No research pass can return "cancelled" today (§61 CN12
120
+ # defers the pipeline deliberately), so that reader is correct as it
121
+ # stands -- and it has to be taught the difference before the
122
+ # deferral is lifted.
123
+ stop_reason: Optional[str] = None
@@ -0,0 +1,280 @@
1
+ """Project path completion and mention grammar (ROADMAP_v3 §54).
2
+
3
+ Only names are indexed. No file contents or access decisions belong here.
4
+ Git is optional; its ignore semantics are used when available. Snapshots are
5
+ bounded and replaced by a worker, never walked by the keystroke handler.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ import os
12
+ import re
13
+ import shutil
14
+
15
+ # Optional git listing uses fixed argv and no shell.
16
+ import subprocess # nosec B404
17
+ import threading
18
+ import time
19
+ import weakref
20
+ from collections import deque
21
+ from dataclasses import dataclass
22
+
23
+ INDEX_LIMIT = 50_000
24
+ MATCH_LIMIT = 200
25
+ SCAN_SECONDS = 5
26
+ GIT_BYTE_LIMIT = 16 * 1024 * 1024
27
+ SKIP_DIRS = frozenset({
28
+ "__pycache__", ".venv", "venv", "node_modules", ".pytest_cache",
29
+ ".mypy_cache", ".ruff_cache", "dist", "build",
30
+ })
31
+ _START = re.compile(r'(?<![\w@/\\])@(?=\S|$)')
32
+ _BARE = re.compile(r'[\w./\\:-]+', re.UNICODE)
33
+ _indexes: weakref.WeakSet = weakref.WeakSet()
34
+ _indexes_lock = threading.Lock()
35
+
36
+
37
+ @dataclass(frozen=True)
38
+ class Mention:
39
+ start: int
40
+ end: int
41
+ path: str
42
+
43
+
44
+ def mentions(text: str, *, incomplete: bool = False) -> list[Mention]:
45
+ """Parse standalone @paths and JSON-quoted @\"paths with spaces\".
46
+
47
+ Bare email addresses and package scopes embedded in a path do not
48
+ trigger. Quoting also permits punctuation in a filename, unambiguously.
49
+ The incomplete form is used only by completion, never by the prompt.
50
+ """
51
+ found = []
52
+ consumed = 0
53
+ for match in _START.finditer(text):
54
+ if match.start() < consumed:
55
+ continue
56
+ begin = match.end()
57
+ if text[begin:begin + 1] == '"':
58
+ try:
59
+ path, length = json.JSONDecoder().raw_decode(text[begin:])
60
+ end = begin + length
61
+ except ValueError:
62
+ if not incomplete:
63
+ continue
64
+ end = text.find("\n", begin)
65
+ end = len(text) if end < 0 else end
66
+ path = text[begin + 1:end]
67
+ else:
68
+ token = _BARE.match(text, begin)
69
+ end = token.end() if token else begin
70
+ path = text[begin:end]
71
+ if isinstance(path, str) and (path or incomplete):
72
+ found.append(Mention(match.start(), end, path))
73
+ consumed = end
74
+ return found
75
+
76
+
77
+ def active_mention(text: str, cursor: int) -> Mention | None:
78
+ for item in mentions(text, incomplete=True):
79
+ if item.start < cursor <= item.end:
80
+ return item
81
+ return None
82
+
83
+
84
+ def quote_path(path: str) -> str:
85
+ return path if _BARE.fullmatch(path) else json.dumps(path, ensure_ascii=False)
86
+
87
+
88
+ @dataclass(frozen=True)
89
+ class FileSuggestion:
90
+ path: str
91
+ prefix: str = "@"
92
+ usage: str | None = None
93
+
94
+ @property
95
+ def name(self) -> str:
96
+ return quote_path(self.path)
97
+
98
+ @property
99
+ def summary(self) -> str:
100
+ return "folder" if self.path.endswith("/") else "file"
101
+
102
+
103
+ @dataclass(frozen=True)
104
+ class Snapshot:
105
+ visible: tuple[str, ...] = ()
106
+ ignored: tuple[str, ...] = ()
107
+ source: str = "indexing"
108
+ limited: bool = False
109
+
110
+
111
+ def _git_paths(root: str, ignored: bool):
112
+ """Stream bounded NUL records; fixed argv, no shell or fsmonitor hook."""
113
+ git = shutil.which("git")
114
+ if not git:
115
+ raise OSError("git unavailable")
116
+ args = [git, "-c", "core.fsmonitor=false", "ls-files", "-z",
117
+ "--others", "--exclude-standard"]
118
+ args += ["--ignored"] if ignored else ["--cached"]
119
+ # B603: executable from PATH, fixed options; root is cwd, never argv.
120
+ process = subprocess.Popen( # nosec B603
121
+ args, cwd=root, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
122
+ expired = threading.Event()
123
+
124
+ def stop():
125
+ expired.set()
126
+ try:
127
+ process.kill()
128
+ except OSError: # process exited as the timer fired
129
+ pass
130
+
131
+ timer = threading.Timer(SCAN_SECONDS, stop)
132
+ timer.daemon = True
133
+ timer.start()
134
+ try:
135
+ pending = b""
136
+ received = 0
137
+ while chunk := process.stdout.read1(65536):
138
+ received += len(chunk)
139
+ if received > GIT_BYTE_LIMIT:
140
+ raise TimeoutError("git listing byte limit")
141
+ records = (pending + chunk).split(b"\0")
142
+ pending = records.pop()
143
+ for raw in records:
144
+ yield os.fsdecode(raw)
145
+ if process.wait() or expired.is_set():
146
+ raise OSError("git listing failed or timed out")
147
+ finally:
148
+ timer.cancel()
149
+ if process.poll() is None:
150
+ process.kill()
151
+ process.wait()
152
+ process.stdout.close()
153
+
154
+
155
+ def _safe_name(path: str) -> bool:
156
+ return (bool(path) and not os.path.isabs(path)
157
+ and not any(part in ("..", ".git") for part in path.split("/"))
158
+ and not any(ord(char) < 32 or 127 <= ord(char) < 160
159
+ or 0xD800 <= ord(char) < 0xE000 for char in path))
160
+
161
+
162
+ def scan(root: str) -> Snapshot:
163
+ visible, ignored = set(), set()
164
+ limited = False
165
+
166
+ def add(path, hidden):
167
+ nonlocal limited
168
+ path = path.replace(os.sep, "/")
169
+ if not _safe_name(path):
170
+ return
171
+ target = ignored if hidden else visible
172
+ parts = path.rstrip("/").split("/")
173
+ entries = ["/".join(parts[:n]) + "/" for n in range(1, len(parts))]
174
+ entries.append(path)
175
+ for entry in entries:
176
+ if entry in visible or entry in ignored:
177
+ continue
178
+ if len(visible) + len(ignored) >= INDEX_LIMIT:
179
+ limited = True
180
+ return
181
+ target.add(entry)
182
+
183
+ try:
184
+ for hidden in (False, True):
185
+ listing = _git_paths(root, hidden)
186
+ try:
187
+ for path in listing:
188
+ add(path, hidden)
189
+ if limited:
190
+ break
191
+ finally:
192
+ listing.close()
193
+ if limited:
194
+ break
195
+ source = "git ignores"
196
+ except (OSError, TimeoutError):
197
+ visible.clear()
198
+ ignored.clear()
199
+ limited = False
200
+ source = "fallback skips"
201
+ pending = deque([(root, False)])
202
+ deadline = time.monotonic() + SCAN_SECONDS
203
+ visited = 0
204
+ while pending and not limited:
205
+ directory, hidden = pending.popleft()
206
+ try:
207
+ with os.scandir(directory) as entries:
208
+ for entry in entries:
209
+ visited += 1
210
+ if visited > INDEX_LIMIT or time.monotonic() >= deadline:
211
+ limited = True
212
+ break
213
+ if entry.name.casefold() == ".git":
214
+ continue
215
+ is_dir = entry.is_dir(follow_symlinks=False)
216
+ child_hidden = hidden or (is_dir and entry.name in SKIP_DIRS)
217
+ rel = os.path.relpath(entry.path, root).replace(os.sep, "/")
218
+ add(rel + ("/" if is_dir else ""), child_hidden)
219
+ if is_dir:
220
+ pending.append((entry.path, child_hidden))
221
+ if limited:
222
+ break
223
+ except OSError:
224
+ # An unreadable subtree makes the snapshot partial, visibly.
225
+ limited = True
226
+ return Snapshot(tuple(sorted(visible)), tuple(sorted(ignored)), source, limited)
227
+
228
+
229
+ class FileIndex:
230
+ def __init__(self, root: str | None):
231
+ self.root = os.path.realpath(root) if root else None
232
+ self.snapshot = Snapshot(source="indexing" if root else "no project")
233
+ self.show_ignored = False
234
+ self.dirty = threading.Event()
235
+ if root:
236
+ self.dirty.set()
237
+ with _indexes_lock:
238
+ _indexes.add(self)
239
+
240
+ def refresh(self):
241
+ self.dirty.clear()
242
+ if self.root:
243
+ self.snapshot = scan(self.root)
244
+
245
+ def matching(self, query: str) -> tuple[list[FileSuggestion], str]:
246
+ snapshot = self.snapshot
247
+ query = query.replace("\\", "/").casefold()
248
+ paths = snapshot.visible + (snapshot.ignored if self.show_ignored else ())
249
+ # Keep bounded prefix/substring buckets instead of sorting 50,000
250
+ # rows at every keystroke. Snapshot order breaks ties predictably.
251
+ prefixes, others = [], []
252
+ total = 0
253
+ for path in paths:
254
+ folded = path.casefold()
255
+ if query not in folded:
256
+ continue
257
+ total += 1
258
+ starts = folded.startswith(query) or folded.rstrip("/").rsplit("/", 1)[-1].startswith(query)
259
+ bucket = prefixes if starts else others
260
+ if len(bucket) < MATCH_LIMIT:
261
+ bucket.append(FileSuggestion(path))
262
+ notices = []
263
+ if total > MATCH_LIMIT:
264
+ notices.append(f"first {MATCH_LIMIT} of {total} matches")
265
+ if snapshot.limited:
266
+ notices.append(f"partial index (limit {INDEX_LIMIT:,})")
267
+ notices += [snapshot.source, f"ignored {'shown' if self.show_ignored else 'hidden'}"]
268
+ return (prefixes + others)[:MATCH_LIMIT], " · ".join(notices)
269
+
270
+
271
+ def note_file_created(path: str) -> None:
272
+ """Called only after write succeeds; subagents use the same producer."""
273
+ with _indexes_lock:
274
+ indexes = list(_indexes)
275
+ for index in indexes:
276
+ try:
277
+ if os.path.commonpath((index.root, path)) == index.root:
278
+ index.dirty.set()
279
+ except ValueError: # different drives
280
+ continue
@@ -39,13 +39,58 @@ Deliberately a leaf module with no project imports, matching core/approval.py
39
39
  it depends on none of them.
40
40
  """
41
41
 
42
- from dataclasses import dataclass, field
43
- from typing import Any, Callable, Optional
44
-
45
42
  import logging
43
+ import threading
44
+ from collections.abc import Callable
45
+ from dataclasses import dataclass, field
46
+ from typing import Any, Optional
46
47
 
47
48
  logger = logging.getLogger(__name__)
48
49
 
50
+ # ROADMAP_v2 §47 slice 8 (NA12). ONE QUESTION ON SCREEN AT A TIME, and it
51
+ # lives here because this module is already the one entry point every
52
+ # question goes through.
53
+ #
54
+ # Until slice 8 nothing could ask twice at once, so no shell needed to cope
55
+ # with it and none does. Both of ours break in the same way and for the
56
+ # same reason -- a single slot for the pending answer. `tui/app.py` keeps
57
+ # one `_permission_channel` so shutdown can release a parked worker: a
58
+ # second ask overwrites the field and orphans the first, and the first ask
59
+ # to FINISH nulls it in its `finally`, so a still-open question becomes
60
+ # unreleasable. `main.py` has one `_StdinReader`, and two readers of one
61
+ # stdin is the defect §29's N1 exists to prevent, arrived at from the other
62
+ # direction. Two consent surfaces at once is also what §42 refuses on its
63
+ # own terms.
64
+ #
65
+ # HERE RATHER THAN IN EACH SHELL, which is this project's canonical rule --
66
+ # fix at the producer. A lock around `tui/app.py`'s `_blocking_modal` would
67
+ # leave the CLI unprotected and would have to be written again for the
68
+ # third shell.
69
+ #
70
+ # NO TIMEOUT, deliberately. Every waiter is bounded by the holder's own
71
+ # deadline ONCE THE HOLDER IS PARKED: the TUI parks for at most
72
+ # ATTENDED_APPROVAL_TIMEOUT_S and a run-backed CLI prompt carries the same,
73
+ # so the queue drains on its own. The step before the TUI's park has no
74
+ # deadline of its own, and until #223 this paragraph read as if it had:
75
+ # `_blocking_modal` first marshals its screen onto the UI thread, which is
76
+ # bounded only by that thread staying responsive -- the bound every other
77
+ # marshal in the shell lives with. Measured under textual 8.2.8 (#220), a
78
+ # marshal after the app exits returns or raises at once rather than
79
+ # parking, so what is left unbounded is a UI thread that is alive and
80
+ # wedged, where nothing else in the shell answers either.
81
+ # A deadline here would need this leaf module to know a number that
82
+ # belongs to config.py, and the failure it would prevent is one no shell
83
+ # can currently produce. Shutdown does not wait either: the TUI's release
84
+ # unblocks the holder, and every queued worker then finds `_shutting_down`
85
+ # already true and returns its declining default without pushing a screen.
86
+ #
87
+ # NOT FIFO, and nothing here claims it is -- a Lock makes no ordering
88
+ # promise, so which of two simultaneous questions is put first is
89
+ # undefined. What IS guaranteed is that each answer reaches the asker that
90
+ # holds it, because the request travels with the caller and never through
91
+ # this module's state.
92
+ _ask_lock = threading.Lock()
93
+
49
94
 
50
95
  # ---------------------------------------------------------------------------
51
96
  # ---- Kinds ----------------------------------------------------------------
@@ -66,6 +111,16 @@ CHOICE = "choice"
66
111
  # four affordances the spec asks for by name, and a shape CHOICE's decoder
67
112
  # cannot express without becoming two decoders wearing one name.
68
113
  QUESTION = "question"
114
+ # ROADMAP_v3 §49 slice 5a (SS59). One value, typed by a human, that the
115
+ # caller needs and this module must never see printed.
116
+ #
117
+ # Its own kind rather than a QUESTION with a flag, because the KIND is what
118
+ # tells a shell how to RENDER -- and the rendering is the whole point here:
119
+ # the field is masked, the answer is not echoed, and it does not reach the
120
+ # transcript, the log or the scrollback. A flag on an existing kind would
121
+ # make a shell that ignored the flag look wired up while showing the
122
+ # passphrase in clear.
123
+ SECRET = "secret"
69
124
 
70
125
  # Review decisions, moved here from core/reasoning/review.py so the decoder
71
126
  # and the pipeline cannot disagree about what a valid answer is.
@@ -105,6 +160,14 @@ SAFE_DEFAULTS = {
105
160
  # answer carrying defer=True, because a user who says that has told the
106
161
  # model something, and a user who closed the modal has not.
107
162
  QUESTION: None,
163
+ # None, for CHOICE's reason and one more that is specific to this kind:
164
+ # the empty string is a value a user could legitimately mean to type,
165
+ # and it is also what a torn-down modal and a shell with no branch for
166
+ # SECRET both produce. Defaulting to "" would hand `sudo` an empty
167
+ # password -- an authentication ATTEMPT made on nobody's behalf, which
168
+ # on a remote account is a step towards a lockout. None is not a
169
+ # password, so every consumer has to notice.
170
+ SECRET: None,
108
171
  }
109
172
 
110
173
 
@@ -191,6 +254,21 @@ def decode(request: "Request", raw: Any) -> Any:
191
254
  # third state to confuse it with.
192
255
  return bool(raw)
193
256
 
257
+ if kind == SECRET:
258
+ # A string, or nothing. NOT `str(raw)`: a shell that returned a
259
+ # widget, a dict or a bool would be coerced into a "password" that
260
+ # is really a repr, and it would then be TRIED against a real
261
+ # account. And not `raw or None` either -- an empty string is the
262
+ # shape a dismissed modal produces, and it must decode as "nobody
263
+ # answered" rather than as a password of length zero.
264
+ #
265
+ # The value is returned raw and is NOT logged, here or anywhere on
266
+ # this path. This is the one decode branch whose input must not
267
+ # appear in a warning about its own shape.
268
+ if not isinstance(raw, str) or raw == "":
269
+ return None
270
+ return raw
271
+
194
272
  if kind == CHOICE:
195
273
  # Valid ONLY against what was offered. A shell returning something
196
274
  # nobody proposed is answering a different question, and the
@@ -314,7 +392,12 @@ def ask(channel: Optional[ResponseChannel], request: Request) -> Any:
314
392
  if channel is None:
315
393
  return decode(request, None)
316
394
  try:
317
- raw = channel.ask(request)
395
+ # NA12. Serialised, so a shell that can only hold one pending
396
+ # answer is never handed two. The lock covers the shell's call and
397
+ # nothing else: `decode` is pure, and holding a lock across it
398
+ # would make one shell's slowness another's wait for no reason.
399
+ with _ask_lock:
400
+ raw = channel.ask(request)
318
401
  except Exception: # noqa: BLE001 -- a shell's failure is not the run's
319
402
  # Logged with the traceback, because a shell raising here is a real
320
403
  # bug worth finding, and at ERROR because the user's answer was
@@ -0,0 +1,128 @@
1
+ """
2
+ core/line_pattern.py
3
+
4
+ The one place a model-supplied pattern is compiled and matched (ROADMAP_v3
5
+ §49, SS8). A monitor session wakes the agent when a line of a program's
6
+ output matches a pattern the MODEL wrote -- two untrusted inputs meeting
7
+ inside the harness's own process, on a thread beside the TUI's.
8
+
9
+ WHY NOT `re`. Measured on CPython 3.13 (batch 93): `re.match(r"(a+)+$",
10
+ "a" * 25 + "b")` took 1.88 s, and a sibling thread's ticker stalled for
11
+ 1.98 s of it. `re` holds the GIL for the whole match, so a backtracking
12
+ pattern freezes EVERY thread, and each extra character doubles the time. No
13
+ per-line budget enforced from inside the same process can bound that.
14
+
15
+ re2 matches in time linear in the input, releases the GIL while it runs,
16
+ and refuses the constructs that make backtracking possible: a backreference
17
+ or a lookaround is a compile error, and the error is what the agent is told.
18
+
19
+ THREE GUARDS, each measured against google-re2 1.1.20251105:
20
+
21
+ provenance another package ships a module named `re2` that falls back to
22
+ `re` for what it cannot handle, which would bring the freeze
23
+ back under a name that looks safe. The module must be the one
24
+ the `google-re2` distribution installed, or this module
25
+ refuses to import. Checked by locating that distribution's
26
+ own `re2/__init__.py` (0.6 ms), not by
27
+ `packages_distributions()`, which scans every installed
28
+ package (593 ms, on a path every launch takes).
29
+ log_errors OFF. re2's C++ side reports every compile error on stderr
30
+ through Abseil, and anything written to stderr while Textual
31
+ is up paints over the screen.
32
+ max_mem set explicitly to re2's own 8 MiB default, so the bound on
33
+ what one compiled pattern may cost is a decision rather than
34
+ a library default that can move. A pattern past it is refused
35
+ ("pattern too large").
36
+
37
+ Imports nothing first-party, so a session backend or a tool can use it
38
+ without an import cycle.
39
+ """
40
+
41
+ from __future__ import annotations
42
+
43
+ import importlib.metadata
44
+ import os
45
+
46
+ import re2
47
+
48
+ _DISTRIBUTION = "google-re2"
49
+ _MAX_MEM = 8 << 20
50
+
51
+
52
+ class PatternError(ValueError):
53
+ """A pattern re2 will not compile, carrying re2's own reason."""
54
+
55
+
56
+ def _assert_provenance() -> None:
57
+ """Refuse a `re2` module that the `google-re2` distribution did not
58
+ install. Raises ImportError, because a module that cannot vouch for its
59
+ engine should fail where it is imported rather than at the first match."""
60
+ try:
61
+ dist = importlib.metadata.distribution(_DISTRIBUTION)
62
+ expected = os.path.realpath(str(dist.locate_file("re2/__init__.py")))
63
+ except importlib.metadata.PackageNotFoundError:
64
+ expected = None
65
+ actual = os.path.realpath(getattr(re2, "__file__", "") or "")
66
+ if expected is None or expected != actual:
67
+ raise ImportError(
68
+ f"core/line_pattern.py needs the `re2` module from the "
69
+ f"{_DISTRIBUTION} distribution, and the one imported is "
70
+ f"{actual or 'unknown'}. Another package's `re2` can fall back "
71
+ f"to Python's `re`, which freezes every thread on a backtracking "
72
+ f"pattern. Install it with: pip install {_DISTRIBUTION}")
73
+
74
+
75
+ _assert_provenance()
76
+
77
+
78
+ def _options():
79
+ options = re2.Options()
80
+ options.log_errors = False
81
+ options.max_mem = _MAX_MEM
82
+ return options
83
+
84
+
85
+ def _reason(error: Exception) -> str:
86
+ """re2's message as text. The wrapper raises it as bytes."""
87
+ detail = error.args[0] if error.args else ""
88
+ if isinstance(detail, bytes):
89
+ detail = detail.decode("utf-8", "replace")
90
+ return str(detail) or "the pattern could not be compiled"
91
+
92
+
93
+ class LinePattern:
94
+ """A compiled monitor pattern. `search` is the only question asked."""
95
+
96
+ __slots__ = ("source", "_compiled")
97
+
98
+ def __init__(self, source: str, compiled) -> None:
99
+ self.source = source
100
+ self._compiled = compiled
101
+
102
+ def search(self, line: str) -> bool:
103
+ """Whether the pattern occurs anywhere in *line*."""
104
+ return self._compiled.search(line) is not None
105
+
106
+
107
+ def compile_pattern(pattern) -> LinePattern:
108
+ """Compile *pattern* for line matching, or raise PatternError.
109
+
110
+ Total over whatever the model sent: the value arrives from a tool call's
111
+ params, so a non-string is refused here rather than raising a TypeError
112
+ out of re2.
113
+ """
114
+ if not isinstance(pattern, str) or not pattern:
115
+ raise PatternError("a pattern must be a non-empty string")
116
+ try:
117
+ return LinePattern(pattern, re2.compile(pattern, _options()))
118
+ except re2.error as e:
119
+ raise PatternError(_reason(e)) from None
120
+
121
+
122
+ def validate(pattern) -> str | None:
123
+ """Why *pattern* cannot be a monitor pattern, or None when it can."""
124
+ try:
125
+ compile_pattern(pattern)
126
+ except PatternError as e:
127
+ return str(e)
128
+ return None