@yahyahammad/venastine 0.2.0 → 0.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/.env.example +19 -14
  2. package/CONFIG_ARCHITECTURE.md +2002 -0
  3. package/NOTICE +1 -1
  4. package/README.md +233 -94
  5. package/agents/builtin/build.md +73 -0
  6. package/agents/builtin/general.md +80 -0
  7. package/agents/builtin/plan.md +23 -5
  8. package/agents/builtin/test.md +65 -0
  9. package/agents/builtin/writer.md +67 -0
  10. package/agents/manager.py +261 -195
  11. package/agents/subagent_tool.py +277 -43
  12. package/agents/tui_commands.py +7 -12
  13. package/bin/venastine.mjs +528 -382
  14. package/bin/venastine.py +71 -0
  15. package/cli_commands.py +835 -0
  16. package/config.py +95 -1198
  17. package/config.yaml +1090 -0
  18. package/config_edit.py +1301 -0
  19. package/config_schema.py +966 -0
  20. package/config_update.py +749 -0
  21. package/core/agent_activity.py +158 -20
  22. package/core/ansi.py +452 -0
  23. package/core/approval.py +27 -5
  24. package/core/call_view.py +364 -0
  25. package/core/cancel.py +121 -0
  26. package/core/chatgpt_auth.py +747 -0
  27. package/core/claims_view.py +107 -0
  28. package/core/client.py +1238 -82
  29. package/core/commands.py +454 -0
  30. package/core/compaction.py +830 -66
  31. package/core/config_loader.py +516 -97
  32. package/core/connect.py +686 -0
  33. package/core/events.py +68 -6
  34. package/core/file_mentions.py +280 -0
  35. package/core/interaction.py +87 -4
  36. package/core/line_pattern.py +128 -0
  37. package/core/loop.py +1516 -440
  38. package/core/memory.py +120 -8
  39. package/core/pipeline_models.py +23 -9
  40. package/core/provider_errors.py +234 -0
  41. package/core/reasoning/confidence_scoring.py +70 -54
  42. package/core/reasoning/json_retry.py +19 -1
  43. package/core/reasoning/orchestrator.py +232 -53
  44. package/core/reasoning/output_writer.py +90 -31
  45. package/core/reasoning/payload_validation.py +133 -7
  46. package/core/reasoning/pipeline_storage.py +427 -361
  47. package/core/reasoning/review.py +659 -615
  48. package/core/reasoning/scholar.py +38 -14
  49. package/core/reasoning/source_corpus.py +110 -4
  50. package/core/replay.py +158 -26
  51. package/core/session.py +36 -12
  52. package/core/session_view.py +259 -0
  53. package/core/session_wake.py +259 -0
  54. package/core/shell_sessions.py +1553 -0
  55. package/core/workspace_trust.py +1 -1
  56. package/credentials.py +308 -180
  57. package/database.py +82 -1
  58. package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
  59. package/docs/SECURITY.md +108 -0
  60. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  61. package/env_secrets.py +118 -77
  62. package/json_store.py +236 -16
  63. package/logging_setup.py +121 -11
  64. package/main.py +1671 -199
  65. package/mcp_client/client.py +257 -23
  66. package/mcp_client/config.py +484 -379
  67. package/mcp_client/registration.py +163 -2
  68. package/memories/manager.py +15 -0
  69. package/memories/tui_commands.py +7 -12
  70. package/package.json +234 -4
  71. package/project_init/config_files.py +4 -0
  72. package/project_init/generator.py +201 -37
  73. package/project_init/manifest.py +418 -406
  74. package/project_init/tui_commands.py +7 -4
  75. package/prompts/revalidate.md +1 -1
  76. package/prompts/revise.md +17 -15
  77. package/prompts/system_prompts.py +64 -10
  78. package/providers.json.example +94 -92
  79. package/pyproject.toml +51 -2
  80. package/requirements.txt +252 -29
  81. package/safety/policy_enforcement.py +353 -17
  82. package/security/askpass.py +419 -0
  83. package/security/capability.py +37 -7
  84. package/security/confine.py +484 -0
  85. package/security/hosted.py +182 -0
  86. package/security/permissions.py +54 -7
  87. package/security/posture.py +124 -12
  88. package/security/protected_paths.py +198 -6
  89. package/security/sandbox.py +3262 -139
  90. package/security/secrets.py +855 -0
  91. package/skills/tui_commands.py +3 -4
  92. package/storage.py +368 -35
  93. package/tools/base.py +465 -305
  94. package/tools/builtin/_math_common.py +306 -16
  95. package/tools/builtin/_net_common.py +267 -61
  96. package/tools/builtin/arxiv.py +681 -223
  97. package/tools/builtin/ask_user.py +231 -213
  98. package/tools/builtin/discrete_math.py +35 -4
  99. package/tools/builtin/fetch_url.py +239 -21
  100. package/tools/builtin/file_ops.py +341 -33
  101. package/tools/builtin/geometry.py +184 -148
  102. package/tools/builtin/get_time.py +2 -2
  103. package/tools/builtin/linear_algebra.py +66 -23
  104. package/tools/builtin/load_skill.py +10 -0
  105. package/tools/builtin/logic.py +171 -12
  106. package/tools/builtin/pin.py +46 -12
  107. package/tools/builtin/probability_stats.py +224 -228
  108. package/tools/builtin/project_docs.py +80 -21
  109. package/tools/builtin/remember.py +21 -4
  110. package/tools/builtin/shell.py +586 -67
  111. package/tools/builtin/shell_sessions.py +613 -0
  112. package/tools/builtin/symbolic_math.py +80 -10
  113. package/tools/builtin/todo.py +5 -3
  114. package/tools/builtin/web_search.py +329 -151
  115. package/tools/context.py +23 -0
  116. package/tools/isolation.py +387 -314
  117. package/tools/registry.py +347 -30
  118. package/tui/app.py +4645 -438
  119. package/tui/app.tcss +185 -2
  120. package/tui/commands.py +10 -154
  121. package/tui/diffs.py +286 -265
  122. package/tui/hosted_driver.py +102 -0
  123. package/tui/markdown.py +60 -10
  124. package/tui/meters.py +103 -7
  125. package/tui/screens.py +346 -92
  126. package/tui/themes.py +59 -19
  127. package/tui/widgets.py +1575 -111
  128. package/SECURITY.md +0 -88
  129. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -1,406 +1,418 @@
1
- """
2
- project_init/manifest.py
3
-
4
- ROADMAP_v2 §24 (I3): what `/init` shows the initializer agent before it
5
- reads anything, and the deterministic facts the stubs are built from.
6
-
7
- NO MODEL CALL AND NO TOOL CALL. Discovery is a directory walk, so two runs
8
- against an unchanged project produce byte-identical input -- which is what
9
- makes /init's output diffable in the first place, and what keeps the agent
10
- spending its bounded read budget on documents rather than on navigation.
11
-
12
- WHY A MANIFEST RATHER THAN A LISTING TOOL. The agent needs to choose what to
13
- read, and choosing needs sizes: this repo's own root markdown is 721KB, with
14
- DEVLOG.md at 226KB against a 20KB per-read cap. A path list without sizes
15
- invites it to start reading the largest file first and exhaust INIT_MAX_STEPS
16
- inside one document.
17
-
18
- THE MANIFEST GOES IN THE USER MESSAGE, NOT THE SYSTEM PROMPT. §19's K6 has
19
- been hit four times -- K6 itself, §21b's M13, §21c's M19 and §23's with_todos
20
- -- by appending a tier inside with_catalogs(), which feeds pass_prompt().
21
- (This said three; §23 slice 2 shipped before §24, so the fourth already
22
- existed when the sentence was written -- audit #97. Counting instances is
23
- the kind of claim that only stays right if someone increments it, and audit
24
- #146 has since turned K6 into a parametrised rule so a new tier is covered
25
- without anyone counting.) It does not apply here only
26
- because this is the TASK -- the same way _summarize passes segment_text as
27
- user_goal. Putting a project file listing in the system prompt would deliver
28
- it to all ten research passes.
29
- """
30
-
31
- from __future__ import annotations
32
-
33
- import json
34
- import logging
35
- import os
36
- import re
37
- from typing import NamedTuple, Optional
38
-
39
- logger = logging.getLogger(__name__)
40
-
41
- # Pruned from the walk entirely. Not merely uninteresting -- `.venastine`
42
- # holds mcp.json and settings.json, and node_modules can be tens of
43
- # thousands of files, which turns a listing into a denial of service on the
44
- # context window.
45
- _PRUNED = frozenset({
46
- ".git", ".venastine", "node_modules", ".venv", "venv", "env",
47
- "__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache",
48
- "dist", "build", ".tox", ".idea", ".vscode", "target",
49
- "site-packages", ".next", ".cache",
50
- })
51
-
52
- class _Manifest(NamedTuple):
53
- """What one manifest filename tells us.
54
-
55
- `stack` is None for a manifest that says a project BUILDS without
56
- saying what it is written in.
57
-
58
- `test_command` is a fixed default, or a callable that reads one off
59
- the project, or None.
60
- """
61
- stack: Optional[str]
62
- test_command: object
63
-
64
-
65
- def _rspec_if_spec(project_path: str) -> Optional[str]:
66
- """Ruby's test runner is a choice rather than a given -- minitest and
67
- rspec are both ordinary. `spec/` is the checkable signal that this
68
- project made the second one, and without it nothing is claimed."""
69
- return ("bundle exec rspec"
70
- if os.path.isdir(os.path.join(project_path, "spec")) else None)
71
-
72
-
73
- def _gradle_test(project_path: str) -> str:
74
- """The wrapper when the project ships one, which is the entire point of
75
- shipping it: `gradle test` against whatever Gradle happens to be on the
76
- machine is how a build that passes in CI fails on a laptop."""
77
- return ("./gradlew test"
78
- if os.path.isfile(os.path.join(project_path, "gradlew"))
79
- else "gradle test")
80
-
81
-
82
- # `test:` at the start of a line, and not `tests:` -- a target name is
83
- # exact. `.PHONY: test` does not match, which is right: it declares the
84
- # target's kind and is not itself the declaration.
85
- _MAKE_TEST_TARGET = re.compile(r"^test\s*:", re.M)
86
-
87
-
88
- def _make_test(project_path: str) -> Optional[str]:
89
- """`make test` only when the Makefile declares that target."""
90
- try:
91
- with open(os.path.join(project_path, "Makefile"), "r",
92
- encoding="utf-8", errors="replace") as f:
93
- head = f.read(64_000)
94
- except OSError:
95
- return None
96
- return "make test" if _MAKE_TEST_TARGET.search(head) else None
97
-
98
-
99
- # THE ONE TABLE (#96, #94). Every question this module answers about a code
100
- # manifest is answered from here: which filenames count as one, what stack
101
- # each implies, and how this project runs its tests.
102
- #
103
- # It is one table because it used to be four, and no two agreed. This tuple
104
- # listed twelve names; `detect_facts` branched on seven of them;
105
- # `_root_documents` kept a third test of what is interesting; and
106
- # `tools/builtin/project_docs` a fourth of what is readable. The five
107
- # filenames no branch read -- Gemfile, pom.xml, build.gradle, Makefile,
108
- # Dockerfile -- were exactly the five projects proposed as `research` with
109
- # "no code manifests found" printed as the reason, while the manifest shown
110
- # to the agent listed the manifest (#96). Two lists that happen to overlap
111
- # is how that happens. The fix is that there is nothing to keep in step.
112
- #
113
- # I10 governs the third column. A fact in a committed document is read as
114
- # established and a wrong one is worse than an absent one, so `make test`
115
- # is claimed only when the Makefile declares that target, and `bundle exec
116
- # rspec` only when there is a spec/ directory to run.
117
- _MANIFEST_TABLE = {
118
- "pyproject.toml": _Manifest("Python", None),
119
- "setup.py": _Manifest("Python", None),
120
- "setup.cfg": _Manifest("Python", None),
121
- "requirements.txt": _Manifest("Python", None),
122
- "package.json": _Manifest("Node/JavaScript", None),
123
- "Cargo.toml": _Manifest("Rust", "cargo test"),
124
- "go.mod": _Manifest("Go", "go test ./..."),
125
- "Gemfile": _Manifest("Ruby", _rspec_if_spec),
126
- "pom.xml": _Manifest("Java/Maven", "mvn test"),
127
- "build.gradle": _Manifest("Java/Gradle", _gradle_test),
128
- # A build, and no claim about the language. C, Go, LaTeX and this
129
- # repository all have a Makefile; a Dockerfile says even less about
130
- # what is inside the image it builds. Both are strong evidence that
131
- # this is a codebase, which is the question propose_kind asks, and
132
- # terrible evidence about the stack, which is the question it does not.
133
- "Makefile": _Manifest(None, _make_test),
134
- "Dockerfile": _Manifest(None, None),
135
- }
136
-
137
- # Read off the filesystem for a stack that has conventional ones. Nothing
138
- # is inferred from a file's CONTENTS here -- that is I10's line.
139
- _ENTRY_POINTS = {
140
- "Python": ("main.py", "app.py", "cli.py", "__main__.py"),
141
- "Node/JavaScript": ("index.js", "index.ts", "src/index.js",
142
- "src/index.ts"),
143
- }
144
-
145
- _TEST_DIRS = ("tests", "test", "__tests__", "spec")
146
-
147
- # Never listed, by name or in the layout. The read tool refuses these
148
- # anyway, so this is not the control -- it stops the agent spending one of
149
- # its bounded steps discovering that, and stops a credential FILENAME (which
150
- # can name a service, an account or an environment) reaching the model at
151
- # all. Dotfiles are already excluded from the walk; this covers the rest.
152
- _SECRET_FILENAMES = frozenset({
153
- "providers.json", "credentials.json", "secrets.json", "secrets.yaml",
154
- "secrets.yml",
155
- })
156
-
157
-
158
- def _is_secret(name: str) -> bool:
159
- lower = name.lower()
160
- return lower in _SECRET_FILENAMES or lower.startswith(".env")
161
-
162
-
163
- _TREE_MAX_DEPTH = 2
164
- _TREE_MAX_ENTRIES = 120
165
-
166
-
167
- def is_blank(project_path: str) -> bool:
168
- """True when there is nothing to infer a project type from (I13).
169
-
170
- A folder holding only `.git` and `.venastine` is a project someone is
171
- about to start, not one /init can characterise. Proposing a type from
172
- an empty directory would be a guess presented as a finding, so the
173
- caller asks outright instead.
174
- """
175
- try:
176
- entries = os.listdir(project_path)
177
- except OSError:
178
- return True
179
- return not [e for e in entries
180
- if e not in _PRUNED and not e.startswith(".")]
181
-
182
-
183
- def _first_heading(path: str) -> str:
184
- """The document's first markdown heading, or its first non-empty line.
185
-
186
- Read a few kilobytes, not the file: this runs over every root document
187
- before the agent has chosen anything, and DEVLOG.md is 226KB in this
188
- repo alone.
189
- """
190
- try:
191
- with open(path, "r", encoding="utf-8", errors="replace") as f:
192
- head = f.read(4096)
193
- except OSError:
194
- return ""
195
- for line in head.splitlines():
196
- stripped = line.strip()
197
- if stripped.startswith("#"):
198
- return stripped.lstrip("#").strip()
199
- if stripped:
200
- return stripped[:100]
201
- return ""
202
-
203
-
204
- def _root_documents(project_path: str) -> list:
205
- """(relative path, size, first heading) for every root-level document
206
- and manifest, sorted so the listing is stable across runs."""
207
- out = []
208
- try:
209
- names = sorted(os.listdir(project_path))
210
- except OSError:
211
- return out
212
- for name in names:
213
- full = os.path.join(project_path, name)
214
- if not os.path.isfile(full):
215
- continue
216
- # ONE PRODUCER for "what is readable" (#94). This asked its own
217
- # question -- .md/.markdown/.rst, anything starting with `readme`,
218
- # or a manifest filename -- and the line printed above this listing
219
- # says "Paths below are exactly what read_project_doc expects".
220
- # They disagreed in both directions: `readme.bin` and five build
221
- # manifests were advertised and refused, and every `.txt` document
222
- # in a project was readable and invisible. The tool's test is the
223
- # one that decides, so the sentence is true by construction.
224
- #
225
- # Imported here rather than at module scope: tools/registry.py
226
- # imports project_docs at import time and project_docs reaches
227
- # config, so a top-level import would tie this module's import
228
- # order to the tool layer's for no gain. generator.py does the
229
- # same for doc_path.
230
- from tools.builtin.project_docs import is_readable_doc
231
- # `_is_secret` is belt-and-braces HERE and load-bearing in _tree,
232
- # which is where the mutation actually turns a test red. It is
233
- # unreachable today for a NEW reason: the tool denies
234
- # providers.json by name, and credentials.json / secrets.* are not
235
- # readable under any rule it has. It stays because both lists grow
236
- # -- one more entry on either and the reachability flips, silently.
237
- if not is_readable_doc(project_path, name) or _is_secret(name):
238
- continue
239
- try:
240
- size = os.path.getsize(full)
241
- except OSError:
242
- continue
243
- out.append((name, size, _first_heading(full)))
244
- return out
245
-
246
-
247
- def _tree(project_path: str) -> list:
248
- """A depth-limited directory listing, so the agent can see the shape of
249
- the codebase without a file-by-file enumeration of it."""
250
- lines = []
251
- for dirpath, dirs, files in os.walk(project_path):
252
- dirs[:] = sorted(d for d in dirs
253
- if d not in _PRUNED and not d.startswith("."))
254
- rel = os.path.relpath(dirpath, project_path)
255
- depth = 0 if rel == "." else rel.count(os.sep) + 1
256
- if depth > _TREE_MAX_DEPTH:
257
- dirs[:] = []
258
- continue
259
- visible = sorted(f for f in files
260
- if not f.startswith(".") and not _is_secret(f))
261
- prefix = "" if rel == "." else f"{rel.replace(os.sep, '/')}/"
262
- if prefix:
263
- lines.append(f"{prefix} ({len(visible)} files)")
264
- for fname in visible[:12]:
265
- lines.append(f" {prefix}{fname}")
266
- if len(visible) > 12:
267
- lines.append(f" {prefix}... and {len(visible) - 12} more")
268
- if len(lines) > _TREE_MAX_ENTRIES:
269
- lines.append(" ... listing truncated")
270
- break
271
- return lines
272
-
273
-
274
- def detect_facts(project_path: str) -> dict:
275
- """The deterministic facts the stubs interpolate (I10).
276
-
277
- Everything here is read off a manifest file or the presence of a
278
- directory. Nothing is inferred from source contents, because a fact in
279
- a committed document is read as established -- and a wrong one is worse
280
- than an absent one.
281
- """
282
- facts: dict = {"project_name": os.path.basename(os.path.realpath(project_path))}
283
- # A LIST in table order, not a set: it is reported to the user as the
284
- # working shown behind a proposal (#96), and a set would print in
285
- # whatever order the interpreter felt like.
286
- present = [name for name in _MANIFEST_TABLE
287
- if os.path.isfile(os.path.join(project_path, name))]
288
-
289
- # Deduplicated in table order: four filenames say Python, and the stack
290
- # is named once.
291
- stacks = list(dict.fromkeys(
292
- _MANIFEST_TABLE[name].stack for name in present
293
- if _MANIFEST_TABLE[name].stack))
294
-
295
- entry_points = []
296
- for stack in stacks:
297
- for candidate in _ENTRY_POINTS.get(stack, ()):
298
- if os.path.isfile(os.path.join(project_path, candidate)):
299
- entry_points.append(candidate)
300
-
301
- # PRECEDENCE, unchanged by the table: a command read out of the project
302
- # beats a default, and package.json's `scripts.test` beats pytest
303
- # because it is the more specific statement of the two -- a Python
304
- # project with a package.json has decided something. Everything else
305
- # fills an empty slot only, in table order.
306
- test_command = None
307
- if "Python" in stacks and _mentions(
308
- project_path, ("requirements.txt", "pyproject.toml"), "pytest"):
309
- test_command = "pytest"
310
- if "Node/JavaScript" in stacks:
311
- test_command = _package_json_test(project_path) or test_command
312
- for name in present:
313
- if test_command is not None:
314
- break
315
- declared = _MANIFEST_TABLE[name].test_command
316
- test_command = (declared(project_path) if callable(declared)
317
- else declared)
318
-
319
- test_dirs = [d for d in _TEST_DIRS
320
- if os.path.isdir(os.path.join(project_path, d))]
321
-
322
- # The presence set is a FACT, and the only producer of it. propose_kind
323
- # reads this rather than keeping its own idea of what a manifest is.
324
- if present:
325
- facts["code_manifests"] = present
326
- if stacks:
327
- facts["stack"] = ", ".join(stacks)
328
- if entry_points:
329
- facts["entry_points"] = entry_points
330
- if test_command:
331
- facts["test_command"] = test_command
332
- if test_dirs:
333
- facts["test_dirs"] = test_dirs
334
- return facts
335
-
336
-
337
- def _mentions(project_path: str, filenames, needle: str) -> bool:
338
- for name in filenames:
339
- path = os.path.join(project_path, name)
340
- if not os.path.isfile(path):
341
- continue
342
- try:
343
- with open(path, "r", encoding="utf-8", errors="replace") as f:
344
- if needle in f.read(64_000):
345
- return True
346
- except OSError:
347
- continue
348
- return False
349
-
350
-
351
- def _package_json_test(project_path: str) -> Optional[str]:
352
- path = os.path.join(project_path, "package.json")
353
- try:
354
- with open(path, "r", encoding="utf-8-sig") as f:
355
- data = json.load(f)
356
- except (OSError, ValueError):
357
- # A malformed package.json is the project's problem, not /init's.
358
- # Degrade like config_loader does with an unreadable hub document.
359
- logger.debug("Could not parse package.json at %s", path, exc_info=True)
360
- return None
361
- if isinstance(data, dict) and isinstance(data.get("scripts"), dict):
362
- if data["scripts"].get("test"):
363
- return "npm test"
364
- return None
365
-
366
-
367
- def build_manifest(project_path: str) -> str:
368
- """The listing the agent is shown, as one block of text."""
369
- documents = _root_documents(project_path)
370
- facts = detect_facts(project_path)
371
-
372
- lines = [f"# Project: {facts['project_name']}",
373
- f"Path: {project_path}", ""]
374
-
375
- if facts.get("code_manifests"):
376
- # The signal propose_kind acted on, shown to the agent as well as
377
- # to the user. A Makefile or a Dockerfile appears here and in no
378
- # stack line, which is the whole distinction #96 turned on.
379
- lines.append("Code manifests present: "
380
- + ", ".join(facts["code_manifests"]))
381
- if facts.get("stack"):
382
- lines.append(f"Detected stack: {facts['stack']}")
383
- if facts.get("entry_points"):
384
- lines.append(f"Candidate entry points: {', '.join(facts['entry_points'])}")
385
- if facts.get("test_command"):
386
- lines.append(f"Detected test command: {facts['test_command']}")
387
- if facts.get("test_dirs"):
388
- lines.append(f"Test directories: {', '.join(facts['test_dirs'])}")
389
- lines.append("")
390
-
391
- lines.append("## Readable documents")
392
- if documents:
393
- lines.append("Paths below are exactly what read_project_doc expects.")
394
- lines.append("")
395
- for name, size, heading in documents:
396
- suffix = f" — {heading}" if heading else ""
397
- lines.append(f"- {name} ({size:,} bytes){suffix}")
398
- else:
399
- lines.append("None. This project has no documentation yet.")
400
- lines.append("")
401
-
402
- tree = _tree(project_path)
403
- lines.append("## Layout")
404
- lines += tree if tree else ["(empty)"]
405
- lines.append("")
406
- return "\n".join(lines)
1
+ """
2
+ project_init/manifest.py
3
+
4
+ ROADMAP_v2 §24 (I3): what `/init` shows the initializer agent before it
5
+ reads anything, and the deterministic facts the stubs are built from.
6
+
7
+ NO MODEL CALL AND NO TOOL CALL. Discovery is a directory walk, so two runs
8
+ against an unchanged project produce byte-identical input -- which is what
9
+ makes /init's output diffable in the first place, and what keeps the agent
10
+ spending its bounded read budget on documents rather than on navigation.
11
+
12
+ WHY A MANIFEST RATHER THAN A LISTING TOOL. The agent needs to choose what to
13
+ read, and choosing needs sizes: this repo's own root markdown is 721KB, with
14
+ DEVLOG.md at 226KB against a 20KB per-read cap. A path list without sizes
15
+ invites it to start reading the largest file first and exhaust INIT_MAX_STEPS
16
+ inside one document.
17
+
18
+ THE MANIFEST GOES IN THE USER MESSAGE, NOT THE SYSTEM PROMPT. §19's K6 has
19
+ been hit four times -- K6 itself, §21b's M13, §21c's M19 and §23's with_todos
20
+ -- by appending a tier inside with_catalogs(), which feeds pass_prompt().
21
+ (This said three; §23 slice 2 shipped before §24, so the fourth already
22
+ existed when the sentence was written -- audit #97. Counting instances is
23
+ the kind of claim that only stays right if someone increments it, and audit
24
+ #146 has since turned K6 into a parametrised rule so a new tier is covered
25
+ without anyone counting.) It does not apply here only
26
+ because this is the TASK -- the same way _summarize passes segment_text as
27
+ user_goal. Putting a project file listing in the system prompt would deliver
28
+ it to all ten research passes.
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import json
34
+ import logging
35
+ import os
36
+ import re
37
+ from typing import NamedTuple, Optional
38
+
39
+ logger = logging.getLogger(__name__)
40
+
41
+ # Pruned from the walk entirely. Not merely uninteresting -- `.venastine`
42
+ # holds mcp.json and settings.json, and node_modules can be tens of
43
+ # thousands of files, which turns a listing into a denial of service on the
44
+ # context window.
45
+ _PRUNED = frozenset({
46
+ ".git", ".venastine", "node_modules", ".venv", "venv", "env",
47
+ "__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache",
48
+ "dist", "build", ".tox", ".idea", ".vscode", "target",
49
+ "site-packages", ".next", ".cache",
50
+ })
51
+
52
+ class _Manifest(NamedTuple):
53
+ """What one manifest filename tells us.
54
+
55
+ `stack` is None for a manifest that says a project BUILDS without
56
+ saying what it is written in.
57
+
58
+ `test_command` is a fixed default, or a callable that reads one off
59
+ the project, or None.
60
+ """
61
+ stack: Optional[str]
62
+ test_command: object
63
+
64
+
65
+ def _rspec_if_spec(project_path: str) -> Optional[str]:
66
+ """Ruby's test runner is a choice rather than a given -- minitest and
67
+ rspec are both ordinary. `spec/` is the checkable signal that this
68
+ project made the second one, and without it nothing is claimed."""
69
+ return ("bundle exec rspec"
70
+ if os.path.isdir(os.path.join(project_path, "spec")) else None)
71
+
72
+
73
+ def _gradle_test(project_path: str) -> str:
74
+ """The wrapper when the project ships one, which is the entire point of
75
+ shipping it: `gradle test` against whatever Gradle happens to be on the
76
+ machine is how a build that passes in CI fails on a laptop."""
77
+ return ("./gradlew test"
78
+ if os.path.isfile(os.path.join(project_path, "gradlew"))
79
+ else "gradle test")
80
+
81
+
82
+ # `test:` at the start of a line, and not `tests:` -- a target name is
83
+ # exact. `.PHONY: test` does not match, which is right: it declares the
84
+ # target's kind and is not itself the declaration.
85
+ _MAKE_TEST_TARGET = re.compile(r"^test\s*:", re.M)
86
+
87
+
88
+ def _make_test(project_path: str) -> Optional[str]:
89
+ """`make test` only when the Makefile declares that target."""
90
+ try:
91
+ with open(os.path.join(project_path, "Makefile"),
92
+ encoding="utf-8", errors="replace") as f:
93
+ head = f.read(64_000)
94
+ except OSError:
95
+ return None
96
+ return "make test" if _MAKE_TEST_TARGET.search(head) else None
97
+
98
+
99
+ # THE ONE TABLE (#96, #94). Every question this module answers about a code
100
+ # manifest is answered from here: which filenames count as one, what stack
101
+ # each implies, and how this project runs its tests.
102
+ #
103
+ # It is one table because it used to be four, and no two agreed. This tuple
104
+ # listed twelve names; `detect_facts` branched on seven of them;
105
+ # `_root_documents` kept a third test of what is interesting; and
106
+ # `tools/builtin/project_docs` a fourth of what is readable. The five
107
+ # filenames no branch read -- Gemfile, pom.xml, build.gradle, Makefile,
108
+ # Dockerfile -- were exactly the five projects proposed as `research` with
109
+ # "no code manifests found" printed as the reason, while the manifest shown
110
+ # to the agent listed the manifest (#96). Two lists that happen to overlap
111
+ # is how that happens. The fix is that there is nothing to keep in step.
112
+ #
113
+ # I10 governs the third column. A fact in a committed document is read as
114
+ # established and a wrong one is worse than an absent one, so `make test`
115
+ # is claimed only when the Makefile declares that target, and `bundle exec
116
+ # rspec` only when there is a spec/ directory to run.
117
+ _MANIFEST_TABLE = {
118
+ "pyproject.toml": _Manifest("Python", None),
119
+ "setup.py": _Manifest("Python", None),
120
+ "setup.cfg": _Manifest("Python", None),
121
+ "requirements.txt": _Manifest("Python", None),
122
+ "package.json": _Manifest("Node/JavaScript", None),
123
+ "Cargo.toml": _Manifest("Rust", "cargo test"),
124
+ "go.mod": _Manifest("Go", "go test ./..."),
125
+ "Gemfile": _Manifest("Ruby", _rspec_if_spec),
126
+ "pom.xml": _Manifest("Java/Maven", "mvn test"),
127
+ "build.gradle": _Manifest("Java/Gradle", _gradle_test),
128
+ # A build, and no claim about the language. C, Go, LaTeX and this
129
+ # repository all have a Makefile; a Dockerfile says even less about
130
+ # what is inside the image it builds. Both are strong evidence that
131
+ # this is a codebase, which is the question propose_kind asks, and
132
+ # terrible evidence about the stack, which is the question it does not.
133
+ "Makefile": _Manifest(None, _make_test),
134
+ "Dockerfile": _Manifest(None, None),
135
+ }
136
+
137
+ # Read off the filesystem for a stack that has conventional ones. Nothing
138
+ # is inferred from a file's CONTENTS here -- that is I10's line.
139
+ _ENTRY_POINTS = {
140
+ "Python": ("main.py", "app.py", "cli.py", "__main__.py"),
141
+ "Node/JavaScript": ("index.js", "index.ts", "src/index.js",
142
+ "src/index.ts"),
143
+ }
144
+
145
+ _TEST_DIRS = ("tests", "test", "__tests__", "spec")
146
+
147
+ # Never listed, by name or in the layout. The read tool refuses these
148
+ # anyway, so this is not the control -- it stops the agent spending one of
149
+ # its bounded steps discovering that, and stops a credential FILENAME (which
150
+ # can name a service, an account or an environment) reaching the model at
151
+ # all. Dotfiles are already excluded from the walk; this covers the rest:
152
+ # five store names, `.env*`, and since #227 the conventional shapes of key
153
+ # material -- an SSH private key's default name, and the suffixes private
154
+ # keys and keystores are saved under. Until then this comment claimed all
155
+ # credential filenames and the set held the first five: `id_rsa`,
156
+ # `deploy-key.pem` and `api.key` were listed. A public key (`id_rsa.pub`)
157
+ # still is: it is not a secret, and its name says only that a key exists.
158
+ _SECRET_FILENAMES = frozenset({
159
+ "providers.json", "credentials.json", "secrets.json", "secrets.yaml",
160
+ "secrets.yml",
161
+ })
162
+ _KEY_FILENAMES = frozenset({
163
+ "id_rsa", "id_dsa", "id_ecdsa", "id_ed25519", "id_ecdsa_sk",
164
+ "id_ed25519_sk",
165
+ })
166
+ _KEY_SUFFIXES = (".pem", ".key", ".p12", ".pfx", ".ppk", ".jks", ".keystore")
167
+
168
+
169
+ def _is_secret(name: str) -> bool:
170
+ lower = name.lower()
171
+ return (lower in _SECRET_FILENAMES or lower.startswith(".env")
172
+ or lower in _KEY_FILENAMES or lower.endswith(_KEY_SUFFIXES))
173
+
174
+
175
+ _TREE_MAX_DEPTH = 2
176
+ _TREE_MAX_ENTRIES = 120
177
+
178
+
179
+ def is_blank(project_path: str) -> bool:
180
+ """True when there is nothing to infer a project type from (I13).
181
+
182
+ A folder holding only `.git` and `.venastine` is a project someone is
183
+ about to start, not one /init can characterise. Proposing a type from
184
+ an empty directory would be a guess presented as a finding, so the
185
+ caller asks outright instead.
186
+ """
187
+ try:
188
+ entries = os.listdir(project_path)
189
+ except OSError:
190
+ return True
191
+ return not [e for e in entries
192
+ if e not in _PRUNED and not e.startswith(".")]
193
+
194
+
195
+ def _first_heading(path: str) -> str:
196
+ """The document's first markdown heading, or its first non-empty line.
197
+
198
+ Read a few kilobytes, not the file: this runs over every root document
199
+ before the agent has chosen anything, and DEVLOG.md is 226KB in this
200
+ repo alone.
201
+ """
202
+ try:
203
+ with open(path, encoding="utf-8", errors="replace") as f:
204
+ head = f.read(4096)
205
+ except OSError:
206
+ return ""
207
+ for line in head.splitlines():
208
+ stripped = line.strip()
209
+ if stripped.startswith("#"):
210
+ return stripped.lstrip("#").strip()
211
+ if stripped:
212
+ return stripped[:100]
213
+ return ""
214
+
215
+
216
+ def _root_documents(project_path: str) -> list:
217
+ """(relative path, size, first heading) for every root-level document
218
+ and manifest, sorted so the listing is stable across runs."""
219
+ out = []
220
+ try:
221
+ names = sorted(os.listdir(project_path))
222
+ except OSError:
223
+ return out
224
+ for name in names:
225
+ full = os.path.join(project_path, name)
226
+ if not os.path.isfile(full):
227
+ continue
228
+ # ONE PRODUCER for "what is readable" (#94). This asked its own
229
+ # question -- .md/.markdown/.rst, anything starting with `readme`,
230
+ # or a manifest filename -- and the line printed above this listing
231
+ # says "Paths below are exactly what read_project_doc expects".
232
+ # They disagreed in both directions: `readme.bin` and five build
233
+ # manifests were advertised and refused, and every `.txt` document
234
+ # in a project was readable and invisible. The tool's test is the
235
+ # one that decides, so the sentence is true by construction.
236
+ #
237
+ # Imported here rather than at module scope: tools/registry.py
238
+ # imports project_docs at import time and project_docs reaches
239
+ # config, so a top-level import would tie this module's import
240
+ # order to the tool layer's for no gain. generator.py does the
241
+ # same for doc_path.
242
+ from tools.builtin.project_docs import is_readable_doc
243
+ # `_is_secret` is belt-and-braces HERE and load-bearing in _tree,
244
+ # which is where the mutation actually turns a test red. It is
245
+ # unreachable today for a NEW reason: the tool denies
246
+ # providers.json by name, and credentials.json / secrets.* are not
247
+ # readable under any rule it has. It stays because both lists grow
248
+ # -- one more entry on either and the reachability flips, silently.
249
+ if not is_readable_doc(project_path, name) or _is_secret(name):
250
+ continue
251
+ try:
252
+ size = os.path.getsize(full)
253
+ except OSError:
254
+ continue
255
+ out.append((name, size, _first_heading(full)))
256
+ return out
257
+
258
+
259
+ def _tree(project_path: str) -> list:
260
+ """A depth-limited directory listing, so the agent can see the shape of
261
+ the codebase without a file-by-file enumeration of it."""
262
+ lines = []
263
+ for dirpath, dirs, files in os.walk(project_path):
264
+ dirs[:] = sorted(d for d in dirs
265
+ if d not in _PRUNED and not d.startswith("."))
266
+ rel = os.path.relpath(dirpath, project_path)
267
+ depth = 0 if rel == "." else rel.count(os.sep) + 1
268
+ if depth > _TREE_MAX_DEPTH:
269
+ dirs[:] = []
270
+ continue
271
+ visible = sorted(f for f in files
272
+ if not f.startswith(".") and not _is_secret(f))
273
+ prefix = "" if rel == "." else f"{rel.replace(os.sep, '/')}/"
274
+ if prefix:
275
+ lines.append(f"{prefix} ({len(visible)} files)")
276
+ for fname in visible[:12]:
277
+ lines.append(f" {prefix}{fname}")
278
+ if len(visible) > 12:
279
+ lines.append(f" {prefix}... and {len(visible) - 12} more")
280
+ if len(lines) > _TREE_MAX_ENTRIES:
281
+ lines.append(" ... listing truncated")
282
+ break
283
+ return lines
284
+
285
+
286
+ def detect_facts(project_path: str) -> dict:
287
+ """The deterministic facts the stubs interpolate (I10).
288
+
289
+ Everything here is read off a manifest file or the presence of a
290
+ directory. Nothing is inferred from source contents, because a fact in
291
+ a committed document is read as established -- and a wrong one is worse
292
+ than an absent one.
293
+ """
294
+ facts: dict = {"project_name": os.path.basename(os.path.realpath(project_path))}
295
+ # A LIST in table order, not a set: it is reported to the user as the
296
+ # working shown behind a proposal (#96), and a set would print in
297
+ # whatever order the interpreter felt like.
298
+ present = [name for name in _MANIFEST_TABLE
299
+ if os.path.isfile(os.path.join(project_path, name))]
300
+
301
+ # Deduplicated in table order: four filenames say Python, and the stack
302
+ # is named once.
303
+ stacks = list(dict.fromkeys(
304
+ _MANIFEST_TABLE[name].stack for name in present
305
+ if _MANIFEST_TABLE[name].stack))
306
+
307
+ entry_points = []
308
+ for stack in stacks:
309
+ for candidate in _ENTRY_POINTS.get(stack, ()):
310
+ if os.path.isfile(os.path.join(project_path, candidate)):
311
+ entry_points.append(candidate)
312
+
313
+ # PRECEDENCE, unchanged by the table: a command read out of the project
314
+ # beats a default, and package.json's `scripts.test` beats pytest
315
+ # because it is the more specific statement of the two -- a Python
316
+ # project with a package.json has decided something. Everything else
317
+ # fills an empty slot only, in table order.
318
+ test_command = None
319
+ if "Python" in stacks and _mentions(
320
+ project_path, ("requirements.txt", "pyproject.toml"), "pytest"):
321
+ test_command = "pytest"
322
+ if "Node/JavaScript" in stacks:
323
+ test_command = _package_json_test(project_path) or test_command
324
+ for name in present:
325
+ if test_command is not None:
326
+ break
327
+ declared = _MANIFEST_TABLE[name].test_command
328
+ test_command = (declared(project_path) if callable(declared)
329
+ else declared)
330
+
331
+ test_dirs = [d for d in _TEST_DIRS
332
+ if os.path.isdir(os.path.join(project_path, d))]
333
+
334
+ # The presence set is a FACT, and the only producer of it. propose_kind
335
+ # reads this rather than keeping its own idea of what a manifest is.
336
+ if present:
337
+ facts["code_manifests"] = present
338
+ if stacks:
339
+ facts["stack"] = ", ".join(stacks)
340
+ if entry_points:
341
+ facts["entry_points"] = entry_points
342
+ if test_command:
343
+ facts["test_command"] = test_command
344
+ if test_dirs:
345
+ facts["test_dirs"] = test_dirs
346
+ return facts
347
+
348
+
349
+ def _mentions(project_path: str, filenames, needle: str) -> bool:
350
+ for name in filenames:
351
+ path = os.path.join(project_path, name)
352
+ if not os.path.isfile(path):
353
+ continue
354
+ try:
355
+ with open(path, encoding="utf-8", errors="replace") as f:
356
+ if needle in f.read(64_000):
357
+ return True
358
+ except OSError:
359
+ continue
360
+ return False
361
+
362
+
363
+ def _package_json_test(project_path: str) -> Optional[str]:
364
+ path = os.path.join(project_path, "package.json")
365
+ try:
366
+ with open(path, encoding="utf-8-sig") as f:
367
+ data = json.load(f)
368
+ except (OSError, ValueError):
369
+ # A malformed package.json is the project's problem, not /init's.
370
+ # Degrade like config_loader does with an unreadable hub document.
371
+ logger.debug("Could not parse package.json at %s", path, exc_info=True)
372
+ return None
373
+ if isinstance(data, dict) and isinstance(data.get("scripts"), dict):
374
+ if data["scripts"].get("test"):
375
+ return "npm test"
376
+ return None
377
+
378
+
379
+ def build_manifest(project_path: str) -> str:
380
+ """The listing the agent is shown, as one block of text."""
381
+ documents = _root_documents(project_path)
382
+ facts = detect_facts(project_path)
383
+
384
+ lines = [f"# Project: {facts['project_name']}",
385
+ f"Path: {project_path}", ""]
386
+
387
+ if facts.get("code_manifests"):
388
+ # The signal propose_kind acted on, shown to the agent as well as
389
+ # to the user. A Makefile or a Dockerfile appears here and in no
390
+ # stack line, which is the whole distinction #96 turned on.
391
+ lines.append("Code manifests present: "
392
+ + ", ".join(facts["code_manifests"]))
393
+ if facts.get("stack"):
394
+ lines.append(f"Detected stack: {facts['stack']}")
395
+ if facts.get("entry_points"):
396
+ lines.append(f"Candidate entry points: {', '.join(facts['entry_points'])}")
397
+ if facts.get("test_command"):
398
+ lines.append(f"Detected test command: {facts['test_command']}")
399
+ if facts.get("test_dirs"):
400
+ lines.append(f"Test directories: {', '.join(facts['test_dirs'])}")
401
+ lines.append("")
402
+
403
+ lines.append("## Readable documents")
404
+ if documents:
405
+ lines.append("Paths below are exactly what read_project_doc expects.")
406
+ lines.append("")
407
+ for name, size, heading in documents:
408
+ suffix = f" — {heading}" if heading else ""
409
+ lines.append(f"- {name} ({size:,} bytes){suffix}")
410
+ else:
411
+ lines.append("None. This project has no documentation yet.")
412
+ lines.append("")
413
+
414
+ tree = _tree(project_path)
415
+ lines.append("## Layout")
416
+ lines += tree if tree else ["(empty)"]
417
+ lines.append("")
418
+ return "\n".join(lines)