@yahyahammad/venastine 0.3.8 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/.env.example +19 -19
  2. package/CONFIG_ARCHITECTURE.md +2030 -2002
  3. package/LICENSE +201 -201
  4. package/README.md +1034 -1034
  5. package/agents/builtin/compactor.md +75 -75
  6. package/agents/builtin/explore.md +77 -77
  7. package/agents/builtin/pipeline-reviewer.md +100 -100
  8. package/agents/builtin/plan.md +106 -106
  9. package/agents/builtin/review.md +74 -74
  10. package/agents/manager.py +261 -261
  11. package/agents/subagent_tool.py +472 -472
  12. package/agents/tui_commands.py +238 -146
  13. package/bin/venastine.mjs +528 -528
  14. package/bin/venastine.py +71 -71
  15. package/cli_commands.py +838 -835
  16. package/config.yaml +13 -0
  17. package/config_edit.py +1301 -1301
  18. package/config_schema.py +971 -966
  19. package/config_update.py +749 -749
  20. package/core/agent_activity.py +256 -256
  21. package/core/ansi.py +452 -452
  22. package/core/approval.py +137 -137
  23. package/core/call_view.py +364 -364
  24. package/core/cancel.py +121 -121
  25. package/core/chatgpt_auth.py +747 -747
  26. package/core/claims_view.py +107 -107
  27. package/core/client.py +2715 -2715
  28. package/core/commands.py +460 -454
  29. package/core/compaction.py +1735 -1735
  30. package/core/config_loader.py +1982 -1982
  31. package/core/connect.py +686 -686
  32. package/core/events.py +123 -123
  33. package/core/file_mentions.py +280 -280
  34. package/core/interaction.py +411 -411
  35. package/core/loop.py +2566 -2566
  36. package/core/memory.py +482 -482
  37. package/core/pipeline_models.py +190 -190
  38. package/core/provider_errors.py +234 -234
  39. package/core/reasoning/authorization.py +130 -130
  40. package/core/reasoning/base.py +219 -219
  41. package/core/reasoning/confidence_scoring.py +484 -484
  42. package/core/reasoning/events.py +149 -149
  43. package/core/reasoning/orchestrator.py +1561 -1561
  44. package/core/reasoning/output_writer.py +345 -345
  45. package/core/reasoning/payload_validation.py +524 -524
  46. package/core/reasoning/pipeline_storage.py +427 -427
  47. package/core/reasoning/scholar.py +458 -458
  48. package/core/reasoning/source_corpus.py +433 -433
  49. package/core/reasoning/source_scoring.py +855 -855
  50. package/core/replay.py +315 -315
  51. package/core/shell_sessions.py +1553 -1553
  52. package/credentials.py +307 -307
  53. package/database.py +333 -333
  54. package/docs/PRIVACY.md +78 -78
  55. package/docs/SECURITY.md +108 -108
  56. package/docs/THIRD_PARTY_NOTICES.md +148 -148
  57. package/env_secrets.py +118 -118
  58. package/json_store.py +343 -343
  59. package/logging_setup.py +295 -295
  60. package/main.py +3433 -3433
  61. package/mcp_client/client.py +733 -733
  62. package/mcp_client/config.py +484 -484
  63. package/mcp_client/registration.py +343 -343
  64. package/memories/manager.py +147 -147
  65. package/memories/tui_commands.py +319 -239
  66. package/package.json +303 -303
  67. package/project_init/config_files.py +282 -282
  68. package/project_init/generator.py +728 -728
  69. package/project_init/manifest.py +418 -418
  70. package/project_init/tui_commands.py +147 -147
  71. package/prompts/critic_pass.md +24 -24
  72. package/prompts/revalidate.md +25 -25
  73. package/prompts/revise.md +17 -17
  74. package/prompts/source_grounding.md +66 -66
  75. package/prompts/system_prompts.py +316 -316
  76. package/providers.json.example +94 -94
  77. package/pyproject.toml +149 -149
  78. package/requirements.txt +295 -295
  79. package/safety/policy_enforcement.py +1082 -1082
  80. package/security/askpass.py +419 -419
  81. package/security/capability.py +207 -207
  82. package/security/confine.py +484 -484
  83. package/security/hosted.py +182 -182
  84. package/security/permissions.py +237 -237
  85. package/security/protected_paths.py +501 -501
  86. package/security/sandbox.py +4108 -4108
  87. package/skills/builtin/math/formal-specification.md +67 -67
  88. package/skills/builtin/math/numerical-methods.md +66 -66
  89. package/skills/builtin/research/experiment-design.md +62 -62
  90. package/skills/builtin/research/reproducibility.md +55 -55
  91. package/skills/builtin/research/source-evaluation.md +70 -70
  92. package/skills/builtin/research/statistical-inference.md +63 -63
  93. package/skills/builtin/research/technical-writing.md +59 -59
  94. package/skills/builtin/software/code-review.md +61 -61
  95. package/skills/builtin/software/debugging.md +61 -61
  96. package/skills/builtin/software/test-design.md +67 -67
  97. package/skills/tui_commands.py +110 -110
  98. package/storage.py +1338 -1338
  99. package/tools/base.py +465 -465
  100. package/tools/builtin/_math_common.py +667 -667
  101. package/tools/builtin/_net_common.py +267 -267
  102. package/tools/builtin/arxiv.py +681 -681
  103. package/tools/builtin/fetch_url.py +371 -371
  104. package/tools/builtin/file_ops.py +720 -720
  105. package/tools/builtin/geometry.py +184 -184
  106. package/tools/builtin/load_skill.py +76 -76
  107. package/tools/builtin/pin.py +227 -227
  108. package/tools/builtin/probability_stats.py +224 -224
  109. package/tools/builtin/project_docs.py +464 -464
  110. package/tools/builtin/remember.py +175 -175
  111. package/tools/builtin/shell.py +980 -980
  112. package/tools/builtin/todo.py +149 -149
  113. package/tools/builtin/web_search.py +328 -328
  114. package/tools/context.py +159 -159
  115. package/tools/isolation.py +387 -387
  116. package/tools/registry.py +1242 -1242
  117. package/tui/app.py +8318 -8307
  118. package/tui/app.tcss +747 -739
  119. package/tui/commands.py +26 -26
  120. package/tui/diffs.py +286 -286
  121. package/tui/history.py +128 -128
  122. package/tui/hosted_driver.py +102 -102
  123. package/tui/markdown.py +1329 -1329
  124. package/tui/meters.py +364 -364
  125. package/tui/ravens.py +146 -146
  126. package/tui/screens.py +1255 -1197
  127. package/tui/themes.py +599 -599
  128. package/tui/widgets.py +3813 -3796
@@ -1,418 +1,418 @@
1
- """
2
- project_init/manifest.py
3
-
4
- ROADMAP_v2 §24 (I3): what `/init` shows the initializer agent before it
5
- reads anything, and the deterministic facts the stubs are built from.
6
-
7
- NO MODEL CALL AND NO TOOL CALL. Discovery is a directory walk, so two runs
8
- against an unchanged project produce byte-identical input -- which is what
9
- makes /init's output diffable in the first place, and what keeps the agent
10
- spending its bounded read budget on documents rather than on navigation.
11
-
12
- WHY A MANIFEST RATHER THAN A LISTING TOOL. The agent needs to choose what to
13
- read, and choosing needs sizes: this repo's own root markdown is 721KB, with
14
- DEVLOG.md at 226KB against a 20KB per-read cap. A path list without sizes
15
- invites it to start reading the largest file first and exhaust INIT_MAX_STEPS
16
- inside one document.
17
-
18
- THE MANIFEST GOES IN THE USER MESSAGE, NOT THE SYSTEM PROMPT. §19's K6 has
19
- been hit four times -- K6 itself, §21b's M13, §21c's M19 and §23's with_todos
20
- -- by appending a tier inside with_catalogs(), which feeds pass_prompt().
21
- (This said three; §23 slice 2 shipped before §24, so the fourth already
22
- existed when the sentence was written -- audit #97. Counting instances is
23
- the kind of claim that only stays right if someone increments it, and audit
24
- #146 has since turned K6 into a parametrised rule so a new tier is covered
25
- without anyone counting.) It does not apply here only
26
- because this is the TASK -- the same way _summarize passes segment_text as
27
- user_goal. Putting a project file listing in the system prompt would deliver
28
- it to all ten research passes.
29
- """
30
-
31
- from __future__ import annotations
32
-
33
- import json
34
- import logging
35
- import os
36
- import re
37
- from typing import NamedTuple, Optional
38
-
39
- logger = logging.getLogger(__name__)
40
-
41
- # Pruned from the walk entirely. Not merely uninteresting -- `.venastine`
42
- # holds mcp.json and settings.json, and node_modules can be tens of
43
- # thousands of files, which turns a listing into a denial of service on the
44
- # context window.
45
- _PRUNED = frozenset({
46
- ".git", ".venastine", "node_modules", ".venv", "venv", "env",
47
- "__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache",
48
- "dist", "build", ".tox", ".idea", ".vscode", "target",
49
- "site-packages", ".next", ".cache",
50
- })
51
-
52
- class _Manifest(NamedTuple):
53
- """What one manifest filename tells us.
54
-
55
- `stack` is None for a manifest that says a project BUILDS without
56
- saying what it is written in.
57
-
58
- `test_command` is a fixed default, or a callable that reads one off
59
- the project, or None.
60
- """
61
- stack: Optional[str]
62
- test_command: object
63
-
64
-
65
- def _rspec_if_spec(project_path: str) -> Optional[str]:
66
- """Ruby's test runner is a choice rather than a given -- minitest and
67
- rspec are both ordinary. `spec/` is the checkable signal that this
68
- project made the second one, and without it nothing is claimed."""
69
- return ("bundle exec rspec"
70
- if os.path.isdir(os.path.join(project_path, "spec")) else None)
71
-
72
-
73
- def _gradle_test(project_path: str) -> str:
74
- """The wrapper when the project ships one, which is the entire point of
75
- shipping it: `gradle test` against whatever Gradle happens to be on the
76
- machine is how a build that passes in CI fails on a laptop."""
77
- return ("./gradlew test"
78
- if os.path.isfile(os.path.join(project_path, "gradlew"))
79
- else "gradle test")
80
-
81
-
82
- # `test:` at the start of a line, and not `tests:` -- a target name is
83
- # exact. `.PHONY: test` does not match, which is right: it declares the
84
- # target's kind and is not itself the declaration.
85
- _MAKE_TEST_TARGET = re.compile(r"^test\s*:", re.M)
86
-
87
-
88
- def _make_test(project_path: str) -> Optional[str]:
89
- """`make test` only when the Makefile declares that target."""
90
- try:
91
- with open(os.path.join(project_path, "Makefile"),
92
- encoding="utf-8", errors="replace") as f:
93
- head = f.read(64_000)
94
- except OSError:
95
- return None
96
- return "make test" if _MAKE_TEST_TARGET.search(head) else None
97
-
98
-
99
- # THE ONE TABLE (#96, #94). Every question this module answers about a code
100
- # manifest is answered from here: which filenames count as one, what stack
101
- # each implies, and how this project runs its tests.
102
- #
103
- # It is one table because it used to be four, and no two agreed. This tuple
104
- # listed twelve names; `detect_facts` branched on seven of them;
105
- # `_root_documents` kept a third test of what is interesting; and
106
- # `tools/builtin/project_docs` a fourth of what is readable. The five
107
- # filenames no branch read -- Gemfile, pom.xml, build.gradle, Makefile,
108
- # Dockerfile -- were exactly the five projects proposed as `research` with
109
- # "no code manifests found" printed as the reason, while the manifest shown
110
- # to the agent listed the manifest (#96). Two lists that happen to overlap
111
- # is how that happens. The fix is that there is nothing to keep in step.
112
- #
113
- # I10 governs the third column. A fact in a committed document is read as
114
- # established and a wrong one is worse than an absent one, so `make test`
115
- # is claimed only when the Makefile declares that target, and `bundle exec
116
- # rspec` only when there is a spec/ directory to run.
117
- _MANIFEST_TABLE = {
118
- "pyproject.toml": _Manifest("Python", None),
119
- "setup.py": _Manifest("Python", None),
120
- "setup.cfg": _Manifest("Python", None),
121
- "requirements.txt": _Manifest("Python", None),
122
- "package.json": _Manifest("Node/JavaScript", None),
123
- "Cargo.toml": _Manifest("Rust", "cargo test"),
124
- "go.mod": _Manifest("Go", "go test ./..."),
125
- "Gemfile": _Manifest("Ruby", _rspec_if_spec),
126
- "pom.xml": _Manifest("Java/Maven", "mvn test"),
127
- "build.gradle": _Manifest("Java/Gradle", _gradle_test),
128
- # A build, and no claim about the language. C, Go, LaTeX and this
129
- # repository all have a Makefile; a Dockerfile says even less about
130
- # what is inside the image it builds. Both are strong evidence that
131
- # this is a codebase, which is the question propose_kind asks, and
132
- # terrible evidence about the stack, which is the question it does not.
133
- "Makefile": _Manifest(None, _make_test),
134
- "Dockerfile": _Manifest(None, None),
135
- }
136
-
137
- # Read off the filesystem for a stack that has conventional ones. Nothing
138
- # is inferred from a file's CONTENTS here -- that is I10's line.
139
- _ENTRY_POINTS = {
140
- "Python": ("main.py", "app.py", "cli.py", "__main__.py"),
141
- "Node/JavaScript": ("index.js", "index.ts", "src/index.js",
142
- "src/index.ts"),
143
- }
144
-
145
- _TEST_DIRS = ("tests", "test", "__tests__", "spec")
146
-
147
- # Never listed, by name or in the layout. The read tool refuses these
148
- # anyway, so this is not the control -- it stops the agent spending one of
149
- # its bounded steps discovering that, and stops a credential FILENAME (which
150
- # can name a service, an account or an environment) reaching the model at
151
- # all. Dotfiles are already excluded from the walk; this covers the rest:
152
- # five store names, `.env*`, and since #227 the conventional shapes of key
153
- # material -- an SSH private key's default name, and the suffixes private
154
- # keys and keystores are saved under. Until then this comment claimed all
155
- # credential filenames and the set held the first five: `id_rsa`,
156
- # `deploy-key.pem` and `api.key` were listed. A public key (`id_rsa.pub`)
157
- # still is: it is not a secret, and its name says only that a key exists.
158
- _SECRET_FILENAMES = frozenset({
159
- "providers.json", "credentials.json", "secrets.json", "secrets.yaml",
160
- "secrets.yml",
161
- })
162
- _KEY_FILENAMES = frozenset({
163
- "id_rsa", "id_dsa", "id_ecdsa", "id_ed25519", "id_ecdsa_sk",
164
- "id_ed25519_sk",
165
- })
166
- _KEY_SUFFIXES = (".pem", ".key", ".p12", ".pfx", ".ppk", ".jks", ".keystore")
167
-
168
-
169
- def _is_secret(name: str) -> bool:
170
- lower = name.lower()
171
- return (lower in _SECRET_FILENAMES or lower.startswith(".env")
172
- or lower in _KEY_FILENAMES or lower.endswith(_KEY_SUFFIXES))
173
-
174
-
175
- _TREE_MAX_DEPTH = 2
176
- _TREE_MAX_ENTRIES = 120
177
-
178
-
179
- def is_blank(project_path: str) -> bool:
180
- """True when there is nothing to infer a project type from (I13).
181
-
182
- A folder holding only `.git` and `.venastine` is a project someone is
183
- about to start, not one /init can characterise. Proposing a type from
184
- an empty directory would be a guess presented as a finding, so the
185
- caller asks outright instead.
186
- """
187
- try:
188
- entries = os.listdir(project_path)
189
- except OSError:
190
- return True
191
- return not [e for e in entries
192
- if e not in _PRUNED and not e.startswith(".")]
193
-
194
-
195
- def _first_heading(path: str) -> str:
196
- """The document's first markdown heading, or its first non-empty line.
197
-
198
- Read a few kilobytes, not the file: this runs over every root document
199
- before the agent has chosen anything, and DEVLOG.md is 226KB in this
200
- repo alone.
201
- """
202
- try:
203
- with open(path, encoding="utf-8", errors="replace") as f:
204
- head = f.read(4096)
205
- except OSError:
206
- return ""
207
- for line in head.splitlines():
208
- stripped = line.strip()
209
- if stripped.startswith("#"):
210
- return stripped.lstrip("#").strip()
211
- if stripped:
212
- return stripped[:100]
213
- return ""
214
-
215
-
216
- def _root_documents(project_path: str) -> list:
217
- """(relative path, size, first heading) for every root-level document
218
- and manifest, sorted so the listing is stable across runs."""
219
- out = []
220
- try:
221
- names = sorted(os.listdir(project_path))
222
- except OSError:
223
- return out
224
- for name in names:
225
- full = os.path.join(project_path, name)
226
- if not os.path.isfile(full):
227
- continue
228
- # ONE PRODUCER for "what is readable" (#94). This asked its own
229
- # question -- .md/.markdown/.rst, anything starting with `readme`,
230
- # or a manifest filename -- and the line printed above this listing
231
- # says "Paths below are exactly what read_project_doc expects".
232
- # They disagreed in both directions: `readme.bin` and five build
233
- # manifests were advertised and refused, and every `.txt` document
234
- # in a project was readable and invisible. The tool's test is the
235
- # one that decides, so the sentence is true by construction.
236
- #
237
- # Imported here rather than at module scope: tools/registry.py
238
- # imports project_docs at import time and project_docs reaches
239
- # config, so a top-level import would tie this module's import
240
- # order to the tool layer's for no gain. generator.py does the
241
- # same for doc_path.
242
- from tools.builtin.project_docs import is_readable_doc
243
- # `_is_secret` is belt-and-braces HERE and load-bearing in _tree,
244
- # which is where the mutation actually turns a test red. It is
245
- # unreachable today for a NEW reason: the tool denies
246
- # providers.json by name, and credentials.json / secrets.* are not
247
- # readable under any rule it has. It stays because both lists grow
248
- # -- one more entry on either and the reachability flips, silently.
249
- if not is_readable_doc(project_path, name) or _is_secret(name):
250
- continue
251
- try:
252
- size = os.path.getsize(full)
253
- except OSError:
254
- continue
255
- out.append((name, size, _first_heading(full)))
256
- return out
257
-
258
-
259
- def _tree(project_path: str) -> list:
260
- """A depth-limited directory listing, so the agent can see the shape of
261
- the codebase without a file-by-file enumeration of it."""
262
- lines = []
263
- for dirpath, dirs, files in os.walk(project_path):
264
- dirs[:] = sorted(d for d in dirs
265
- if d not in _PRUNED and not d.startswith("."))
266
- rel = os.path.relpath(dirpath, project_path)
267
- depth = 0 if rel == "." else rel.count(os.sep) + 1
268
- if depth > _TREE_MAX_DEPTH:
269
- dirs[:] = []
270
- continue
271
- visible = sorted(f for f in files
272
- if not f.startswith(".") and not _is_secret(f))
273
- prefix = "" if rel == "." else f"{rel.replace(os.sep, '/')}/"
274
- if prefix:
275
- lines.append(f"{prefix} ({len(visible)} files)")
276
- for fname in visible[:12]:
277
- lines.append(f" {prefix}{fname}")
278
- if len(visible) > 12:
279
- lines.append(f" {prefix}... and {len(visible) - 12} more")
280
- if len(lines) > _TREE_MAX_ENTRIES:
281
- lines.append(" ... listing truncated")
282
- break
283
- return lines
284
-
285
-
286
- def detect_facts(project_path: str) -> dict:
287
- """The deterministic facts the stubs interpolate (I10).
288
-
289
- Everything here is read off a manifest file or the presence of a
290
- directory. Nothing is inferred from source contents, because a fact in
291
- a committed document is read as established -- and a wrong one is worse
292
- than an absent one.
293
- """
294
- facts: dict = {"project_name": os.path.basename(os.path.realpath(project_path))}
295
- # A LIST in table order, not a set: it is reported to the user as the
296
- # working shown behind a proposal (#96), and a set would print in
297
- # whatever order the interpreter felt like.
298
- present = [name for name in _MANIFEST_TABLE
299
- if os.path.isfile(os.path.join(project_path, name))]
300
-
301
- # Deduplicated in table order: four filenames say Python, and the stack
302
- # is named once.
303
- stacks = list(dict.fromkeys(
304
- _MANIFEST_TABLE[name].stack for name in present
305
- if _MANIFEST_TABLE[name].stack))
306
-
307
- entry_points = []
308
- for stack in stacks:
309
- for candidate in _ENTRY_POINTS.get(stack, ()):
310
- if os.path.isfile(os.path.join(project_path, candidate)):
311
- entry_points.append(candidate)
312
-
313
- # PRECEDENCE, unchanged by the table: a command read out of the project
314
- # beats a default, and package.json's `scripts.test` beats pytest
315
- # because it is the more specific statement of the two -- a Python
316
- # project with a package.json has decided something. Everything else
317
- # fills an empty slot only, in table order.
318
- test_command = None
319
- if "Python" in stacks and _mentions(
320
- project_path, ("requirements.txt", "pyproject.toml"), "pytest"):
321
- test_command = "pytest"
322
- if "Node/JavaScript" in stacks:
323
- test_command = _package_json_test(project_path) or test_command
324
- for name in present:
325
- if test_command is not None:
326
- break
327
- declared = _MANIFEST_TABLE[name].test_command
328
- test_command = (declared(project_path) if callable(declared)
329
- else declared)
330
-
331
- test_dirs = [d for d in _TEST_DIRS
332
- if os.path.isdir(os.path.join(project_path, d))]
333
-
334
- # The presence set is a FACT, and the only producer of it. propose_kind
335
- # reads this rather than keeping its own idea of what a manifest is.
336
- if present:
337
- facts["code_manifests"] = present
338
- if stacks:
339
- facts["stack"] = ", ".join(stacks)
340
- if entry_points:
341
- facts["entry_points"] = entry_points
342
- if test_command:
343
- facts["test_command"] = test_command
344
- if test_dirs:
345
- facts["test_dirs"] = test_dirs
346
- return facts
347
-
348
-
349
- def _mentions(project_path: str, filenames, needle: str) -> bool:
350
- for name in filenames:
351
- path = os.path.join(project_path, name)
352
- if not os.path.isfile(path):
353
- continue
354
- try:
355
- with open(path, encoding="utf-8", errors="replace") as f:
356
- if needle in f.read(64_000):
357
- return True
358
- except OSError:
359
- continue
360
- return False
361
-
362
-
363
- def _package_json_test(project_path: str) -> Optional[str]:
364
- path = os.path.join(project_path, "package.json")
365
- try:
366
- with open(path, encoding="utf-8-sig") as f:
367
- data = json.load(f)
368
- except (OSError, ValueError):
369
- # A malformed package.json is the project's problem, not /init's.
370
- # Degrade like config_loader does with an unreadable hub document.
371
- logger.debug("Could not parse package.json at %s", path, exc_info=True)
372
- return None
373
- if isinstance(data, dict) and isinstance(data.get("scripts"), dict):
374
- if data["scripts"].get("test"):
375
- return "npm test"
376
- return None
377
-
378
-
379
- def build_manifest(project_path: str) -> str:
380
- """The listing the agent is shown, as one block of text."""
381
- documents = _root_documents(project_path)
382
- facts = detect_facts(project_path)
383
-
384
- lines = [f"# Project: {facts['project_name']}",
385
- f"Path: {project_path}", ""]
386
-
387
- if facts.get("code_manifests"):
388
- # The signal propose_kind acted on, shown to the agent as well as
389
- # to the user. A Makefile or a Dockerfile appears here and in no
390
- # stack line, which is the whole distinction #96 turned on.
391
- lines.append("Code manifests present: "
392
- + ", ".join(facts["code_manifests"]))
393
- if facts.get("stack"):
394
- lines.append(f"Detected stack: {facts['stack']}")
395
- if facts.get("entry_points"):
396
- lines.append(f"Candidate entry points: {', '.join(facts['entry_points'])}")
397
- if facts.get("test_command"):
398
- lines.append(f"Detected test command: {facts['test_command']}")
399
- if facts.get("test_dirs"):
400
- lines.append(f"Test directories: {', '.join(facts['test_dirs'])}")
401
- lines.append("")
402
-
403
- lines.append("## Readable documents")
404
- if documents:
405
- lines.append("Paths below are exactly what read_project_doc expects.")
406
- lines.append("")
407
- for name, size, heading in documents:
408
- suffix = f" — {heading}" if heading else ""
409
- lines.append(f"- {name} ({size:,} bytes){suffix}")
410
- else:
411
- lines.append("None. This project has no documentation yet.")
412
- lines.append("")
413
-
414
- tree = _tree(project_path)
415
- lines.append("## Layout")
416
- lines += tree if tree else ["(empty)"]
417
- lines.append("")
418
- return "\n".join(lines)
1
+ """
2
+ project_init/manifest.py
3
+
4
+ ROADMAP_v2 §24 (I3): what `/init` shows the initializer agent before it
5
+ reads anything, and the deterministic facts the stubs are built from.
6
+
7
+ NO MODEL CALL AND NO TOOL CALL. Discovery is a directory walk, so two runs
8
+ against an unchanged project produce byte-identical input -- which is what
9
+ makes /init's output diffable in the first place, and what keeps the agent
10
+ spending its bounded read budget on documents rather than on navigation.
11
+
12
+ WHY A MANIFEST RATHER THAN A LISTING TOOL. The agent needs to choose what to
13
+ read, and choosing needs sizes: this repo's own root markdown is 721KB, with
14
+ DEVLOG.md at 226KB against a 20KB per-read cap. A path list without sizes
15
+ invites it to start reading the largest file first and exhaust INIT_MAX_STEPS
16
+ inside one document.
17
+
18
+ THE MANIFEST GOES IN THE USER MESSAGE, NOT THE SYSTEM PROMPT. §19's K6 has
19
+ been hit four times -- K6 itself, §21b's M13, §21c's M19 and §23's with_todos
20
+ -- by appending a tier inside with_catalogs(), which feeds pass_prompt().
21
+ (This said three; §23 slice 2 shipped before §24, so the fourth already
22
+ existed when the sentence was written -- audit #97. Counting instances is
23
+ the kind of claim that only stays right if someone increments it, and audit
24
+ #146 has since turned K6 into a parametrised rule so a new tier is covered
25
+ without anyone counting.) It does not apply here only
26
+ because this is the TASK -- the same way _summarize passes segment_text as
27
+ user_goal. Putting a project file listing in the system prompt would deliver
28
+ it to all ten research passes.
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import json
34
+ import logging
35
+ import os
36
+ import re
37
+ from typing import NamedTuple, Optional
38
+
39
+ logger = logging.getLogger(__name__)
40
+
41
+ # Pruned from the walk entirely. Not merely uninteresting -- `.venastine`
42
+ # holds mcp.json and settings.json, and node_modules can be tens of
43
+ # thousands of files, which turns a listing into a denial of service on the
44
+ # context window.
45
+ _PRUNED = frozenset({
46
+ ".git", ".venastine", "node_modules", ".venv", "venv", "env",
47
+ "__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache",
48
+ "dist", "build", ".tox", ".idea", ".vscode", "target",
49
+ "site-packages", ".next", ".cache",
50
+ })
51
+
52
+ class _Manifest(NamedTuple):
53
+ """What one manifest filename tells us.
54
+
55
+ `stack` is None for a manifest that says a project BUILDS without
56
+ saying what it is written in.
57
+
58
+ `test_command` is a fixed default, or a callable that reads one off
59
+ the project, or None.
60
+ """
61
+ stack: Optional[str]
62
+ test_command: object
63
+
64
+
65
+ def _rspec_if_spec(project_path: str) -> Optional[str]:
66
+ """Ruby's test runner is a choice rather than a given -- minitest and
67
+ rspec are both ordinary. `spec/` is the checkable signal that this
68
+ project made the second one, and without it nothing is claimed."""
69
+ return ("bundle exec rspec"
70
+ if os.path.isdir(os.path.join(project_path, "spec")) else None)
71
+
72
+
73
+ def _gradle_test(project_path: str) -> str:
74
+ """The wrapper when the project ships one, which is the entire point of
75
+ shipping it: `gradle test` against whatever Gradle happens to be on the
76
+ machine is how a build that passes in CI fails on a laptop."""
77
+ return ("./gradlew test"
78
+ if os.path.isfile(os.path.join(project_path, "gradlew"))
79
+ else "gradle test")
80
+
81
+
82
+ # `test:` at the start of a line, and not `tests:` -- a target name is
83
+ # exact. `.PHONY: test` does not match, which is right: it declares the
84
+ # target's kind and is not itself the declaration.
85
+ _MAKE_TEST_TARGET = re.compile(r"^test\s*:", re.M)
86
+
87
+
88
+ def _make_test(project_path: str) -> Optional[str]:
89
+ """`make test` only when the Makefile declares that target."""
90
+ try:
91
+ with open(os.path.join(project_path, "Makefile"),
92
+ encoding="utf-8", errors="replace") as f:
93
+ head = f.read(64_000)
94
+ except OSError:
95
+ return None
96
+ return "make test" if _MAKE_TEST_TARGET.search(head) else None
97
+
98
+
99
+ # THE ONE TABLE (#96, #94). Every question this module answers about a code
100
+ # manifest is answered from here: which filenames count as one, what stack
101
+ # each implies, and how this project runs its tests.
102
+ #
103
+ # It is one table because it used to be four, and no two agreed. This tuple
104
+ # listed twelve names; `detect_facts` branched on seven of them;
105
+ # `_root_documents` kept a third test of what is interesting; and
106
+ # `tools/builtin/project_docs` a fourth of what is readable. The five
107
+ # filenames no branch read -- Gemfile, pom.xml, build.gradle, Makefile,
108
+ # Dockerfile -- were exactly the five projects proposed as `research` with
109
+ # "no code manifests found" printed as the reason, while the manifest shown
110
+ # to the agent listed the manifest (#96). Two lists that happen to overlap
111
+ # is how that happens. The fix is that there is nothing to keep in step.
112
+ #
113
+ # I10 governs the third column. A fact in a committed document is read as
114
+ # established and a wrong one is worse than an absent one, so `make test`
115
+ # is claimed only when the Makefile declares that target, and `bundle exec
116
+ # rspec` only when there is a spec/ directory to run.
117
+ _MANIFEST_TABLE = {
118
+ "pyproject.toml": _Manifest("Python", None),
119
+ "setup.py": _Manifest("Python", None),
120
+ "setup.cfg": _Manifest("Python", None),
121
+ "requirements.txt": _Manifest("Python", None),
122
+ "package.json": _Manifest("Node/JavaScript", None),
123
+ "Cargo.toml": _Manifest("Rust", "cargo test"),
124
+ "go.mod": _Manifest("Go", "go test ./..."),
125
+ "Gemfile": _Manifest("Ruby", _rspec_if_spec),
126
+ "pom.xml": _Manifest("Java/Maven", "mvn test"),
127
+ "build.gradle": _Manifest("Java/Gradle", _gradle_test),
128
+ # A build, and no claim about the language. C, Go, LaTeX and this
129
+ # repository all have a Makefile; a Dockerfile says even less about
130
+ # what is inside the image it builds. Both are strong evidence that
131
+ # this is a codebase, which is the question propose_kind asks, and
132
+ # terrible evidence about the stack, which is the question it does not.
133
+ "Makefile": _Manifest(None, _make_test),
134
+ "Dockerfile": _Manifest(None, None),
135
+ }
136
+
137
+ # Read off the filesystem for a stack that has conventional ones. Nothing
138
+ # is inferred from a file's CONTENTS here -- that is I10's line.
139
+ _ENTRY_POINTS = {
140
+ "Python": ("main.py", "app.py", "cli.py", "__main__.py"),
141
+ "Node/JavaScript": ("index.js", "index.ts", "src/index.js",
142
+ "src/index.ts"),
143
+ }
144
+
145
+ _TEST_DIRS = ("tests", "test", "__tests__", "spec")
146
+
147
+ # Never listed, by name or in the layout. The read tool refuses these
148
+ # anyway, so this is not the control -- it stops the agent spending one of
149
+ # its bounded steps discovering that, and stops a credential FILENAME (which
150
+ # can name a service, an account or an environment) reaching the model at
151
+ # all. Dotfiles are already excluded from the walk; this covers the rest:
152
+ # five store names, `.env*`, and since #227 the conventional shapes of key
153
+ # material -- an SSH private key's default name, and the suffixes private
154
+ # keys and keystores are saved under. Until then this comment claimed all
155
+ # credential filenames and the set held the first five: `id_rsa`,
156
+ # `deploy-key.pem` and `api.key` were listed. A public key (`id_rsa.pub`)
157
+ # still is: it is not a secret, and its name says only that a key exists.
158
+ _SECRET_FILENAMES = frozenset({
159
+ "providers.json", "credentials.json", "secrets.json", "secrets.yaml",
160
+ "secrets.yml",
161
+ })
162
+ _KEY_FILENAMES = frozenset({
163
+ "id_rsa", "id_dsa", "id_ecdsa", "id_ed25519", "id_ecdsa_sk",
164
+ "id_ed25519_sk",
165
+ })
166
+ _KEY_SUFFIXES = (".pem", ".key", ".p12", ".pfx", ".ppk", ".jks", ".keystore")
167
+
168
+
169
+ def _is_secret(name: str) -> bool:
170
+ lower = name.lower()
171
+ return (lower in _SECRET_FILENAMES or lower.startswith(".env")
172
+ or lower in _KEY_FILENAMES or lower.endswith(_KEY_SUFFIXES))
173
+
174
+
175
+ _TREE_MAX_DEPTH = 2
176
+ _TREE_MAX_ENTRIES = 120
177
+
178
+
179
+ def is_blank(project_path: str) -> bool:
180
+ """True when there is nothing to infer a project type from (I13).
181
+
182
+ A folder holding only `.git` and `.venastine` is a project someone is
183
+ about to start, not one /init can characterise. Proposing a type from
184
+ an empty directory would be a guess presented as a finding, so the
185
+ caller asks outright instead.
186
+ """
187
+ try:
188
+ entries = os.listdir(project_path)
189
+ except OSError:
190
+ return True
191
+ return not [e for e in entries
192
+ if e not in _PRUNED and not e.startswith(".")]
193
+
194
+
195
+ def _first_heading(path: str) -> str:
196
+ """The document's first markdown heading, or its first non-empty line.
197
+
198
+ Read a few kilobytes, not the file: this runs over every root document
199
+ before the agent has chosen anything, and DEVLOG.md is 226KB in this
200
+ repo alone.
201
+ """
202
+ try:
203
+ with open(path, encoding="utf-8", errors="replace") as f:
204
+ head = f.read(4096)
205
+ except OSError:
206
+ return ""
207
+ for line in head.splitlines():
208
+ stripped = line.strip()
209
+ if stripped.startswith("#"):
210
+ return stripped.lstrip("#").strip()
211
+ if stripped:
212
+ return stripped[:100]
213
+ return ""
214
+
215
+
216
+ def _root_documents(project_path: str) -> list:
217
+ """(relative path, size, first heading) for every root-level document
218
+ and manifest, sorted so the listing is stable across runs."""
219
+ out = []
220
+ try:
221
+ names = sorted(os.listdir(project_path))
222
+ except OSError:
223
+ return out
224
+ for name in names:
225
+ full = os.path.join(project_path, name)
226
+ if not os.path.isfile(full):
227
+ continue
228
+ # ONE PRODUCER for "what is readable" (#94). This asked its own
229
+ # question -- .md/.markdown/.rst, anything starting with `readme`,
230
+ # or a manifest filename -- and the line printed above this listing
231
+ # says "Paths below are exactly what read_project_doc expects".
232
+ # They disagreed in both directions: `readme.bin` and five build
233
+ # manifests were advertised and refused, and every `.txt` document
234
+ # in a project was readable and invisible. The tool's test is the
235
+ # one that decides, so the sentence is true by construction.
236
+ #
237
+ # Imported here rather than at module scope: tools/registry.py
238
+ # imports project_docs at import time and project_docs reaches
239
+ # config, so a top-level import would tie this module's import
240
+ # order to the tool layer's for no gain. generator.py does the
241
+ # same for doc_path.
242
+ from tools.builtin.project_docs import is_readable_doc
243
+ # `_is_secret` is belt-and-braces HERE and load-bearing in _tree,
244
+ # which is where the mutation actually turns a test red. It is
245
+ # unreachable today for a NEW reason: the tool denies
246
+ # providers.json by name, and credentials.json / secrets.* are not
247
+ # readable under any rule it has. It stays because both lists grow
248
+ # -- one more entry on either and the reachability flips, silently.
249
+ if not is_readable_doc(project_path, name) or _is_secret(name):
250
+ continue
251
+ try:
252
+ size = os.path.getsize(full)
253
+ except OSError:
254
+ continue
255
+ out.append((name, size, _first_heading(full)))
256
+ return out
257
+
258
+
259
+ def _tree(project_path: str) -> list:
260
+ """A depth-limited directory listing, so the agent can see the shape of
261
+ the codebase without a file-by-file enumeration of it."""
262
+ lines = []
263
+ for dirpath, dirs, files in os.walk(project_path):
264
+ dirs[:] = sorted(d for d in dirs
265
+ if d not in _PRUNED and not d.startswith("."))
266
+ rel = os.path.relpath(dirpath, project_path)
267
+ depth = 0 if rel == "." else rel.count(os.sep) + 1
268
+ if depth > _TREE_MAX_DEPTH:
269
+ dirs[:] = []
270
+ continue
271
+ visible = sorted(f for f in files
272
+ if not f.startswith(".") and not _is_secret(f))
273
+ prefix = "" if rel == "." else f"{rel.replace(os.sep, '/')}/"
274
+ if prefix:
275
+ lines.append(f"{prefix} ({len(visible)} files)")
276
+ for fname in visible[:12]:
277
+ lines.append(f" {prefix}{fname}")
278
+ if len(visible) > 12:
279
+ lines.append(f" {prefix}... and {len(visible) - 12} more")
280
+ if len(lines) > _TREE_MAX_ENTRIES:
281
+ lines.append(" ... listing truncated")
282
+ break
283
+ return lines
284
+
285
+
286
+ def detect_facts(project_path: str) -> dict:
287
+ """The deterministic facts the stubs interpolate (I10).
288
+
289
+ Everything here is read off a manifest file or the presence of a
290
+ directory. Nothing is inferred from source contents, because a fact in
291
+ a committed document is read as established -- and a wrong one is worse
292
+ than an absent one.
293
+ """
294
+ facts: dict = {"project_name": os.path.basename(os.path.realpath(project_path))}
295
+ # A LIST in table order, not a set: it is reported to the user as the
296
+ # working shown behind a proposal (#96), and a set would print in
297
+ # whatever order the interpreter felt like.
298
+ present = [name for name in _MANIFEST_TABLE
299
+ if os.path.isfile(os.path.join(project_path, name))]
300
+
301
+ # Deduplicated in table order: four filenames say Python, and the stack
302
+ # is named once.
303
+ stacks = list(dict.fromkeys(
304
+ _MANIFEST_TABLE[name].stack for name in present
305
+ if _MANIFEST_TABLE[name].stack))
306
+
307
+ entry_points = []
308
+ for stack in stacks:
309
+ for candidate in _ENTRY_POINTS.get(stack, ()):
310
+ if os.path.isfile(os.path.join(project_path, candidate)):
311
+ entry_points.append(candidate)
312
+
313
+ # PRECEDENCE, unchanged by the table: a command read out of the project
314
+ # beats a default, and package.json's `scripts.test` beats pytest
315
+ # because it is the more specific statement of the two -- a Python
316
+ # project with a package.json has decided something. Everything else
317
+ # fills an empty slot only, in table order.
318
+ test_command = None
319
+ if "Python" in stacks and _mentions(
320
+ project_path, ("requirements.txt", "pyproject.toml"), "pytest"):
321
+ test_command = "pytest"
322
+ if "Node/JavaScript" in stacks:
323
+ test_command = _package_json_test(project_path) or test_command
324
+ for name in present:
325
+ if test_command is not None:
326
+ break
327
+ declared = _MANIFEST_TABLE[name].test_command
328
+ test_command = (declared(project_path) if callable(declared)
329
+ else declared)
330
+
331
+ test_dirs = [d for d in _TEST_DIRS
332
+ if os.path.isdir(os.path.join(project_path, d))]
333
+
334
+ # The presence set is a FACT, and the only producer of it. propose_kind
335
+ # reads this rather than keeping its own idea of what a manifest is.
336
+ if present:
337
+ facts["code_manifests"] = present
338
+ if stacks:
339
+ facts["stack"] = ", ".join(stacks)
340
+ if entry_points:
341
+ facts["entry_points"] = entry_points
342
+ if test_command:
343
+ facts["test_command"] = test_command
344
+ if test_dirs:
345
+ facts["test_dirs"] = test_dirs
346
+ return facts
347
+
348
+
349
+ def _mentions(project_path: str, filenames, needle: str) -> bool:
350
+ for name in filenames:
351
+ path = os.path.join(project_path, name)
352
+ if not os.path.isfile(path):
353
+ continue
354
+ try:
355
+ with open(path, encoding="utf-8", errors="replace") as f:
356
+ if needle in f.read(64_000):
357
+ return True
358
+ except OSError:
359
+ continue
360
+ return False
361
+
362
+
363
+ def _package_json_test(project_path: str) -> Optional[str]:
364
+ path = os.path.join(project_path, "package.json")
365
+ try:
366
+ with open(path, encoding="utf-8-sig") as f:
367
+ data = json.load(f)
368
+ except (OSError, ValueError):
369
+ # A malformed package.json is the project's problem, not /init's.
370
+ # Degrade like config_loader does with an unreadable hub document.
371
+ logger.debug("Could not parse package.json at %s", path, exc_info=True)
372
+ return None
373
+ if isinstance(data, dict) and isinstance(data.get("scripts"), dict):
374
+ if data["scripts"].get("test"):
375
+ return "npm test"
376
+ return None
377
+
378
+
379
+ def build_manifest(project_path: str) -> str:
380
+ """The listing the agent is shown, as one block of text."""
381
+ documents = _root_documents(project_path)
382
+ facts = detect_facts(project_path)
383
+
384
+ lines = [f"# Project: {facts['project_name']}",
385
+ f"Path: {project_path}", ""]
386
+
387
+ if facts.get("code_manifests"):
388
+ # The signal propose_kind acted on, shown to the agent as well as
389
+ # to the user. A Makefile or a Dockerfile appears here and in no
390
+ # stack line, which is the whole distinction #96 turned on.
391
+ lines.append("Code manifests present: "
392
+ + ", ".join(facts["code_manifests"]))
393
+ if facts.get("stack"):
394
+ lines.append(f"Detected stack: {facts['stack']}")
395
+ if facts.get("entry_points"):
396
+ lines.append(f"Candidate entry points: {', '.join(facts['entry_points'])}")
397
+ if facts.get("test_command"):
398
+ lines.append(f"Detected test command: {facts['test_command']}")
399
+ if facts.get("test_dirs"):
400
+ lines.append(f"Test directories: {', '.join(facts['test_dirs'])}")
401
+ lines.append("")
402
+
403
+ lines.append("## Readable documents")
404
+ if documents:
405
+ lines.append("Paths below are exactly what read_project_doc expects.")
406
+ lines.append("")
407
+ for name, size, heading in documents:
408
+ suffix = f" — {heading}" if heading else ""
409
+ lines.append(f"- {name} ({size:,} bytes){suffix}")
410
+ else:
411
+ lines.append("None. This project has no documentation yet.")
412
+ lines.append("")
413
+
414
+ tree = _tree(project_path)
415
+ lines.append("## Layout")
416
+ lines += tree if tree else ["(empty)"]
417
+ lines.append("")
418
+ return "\n".join(lines)