@yahyahammad/venastine 0.3.8 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -19
- package/CONFIG_ARCHITECTURE.md +2030 -2002
- package/LICENSE +201 -201
- package/README.md +1034 -1034
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -106
- package/agents/builtin/review.md +74 -74
- package/agents/manager.py +261 -261
- package/agents/subagent_tool.py +472 -472
- package/agents/tui_commands.py +238 -146
- package/bin/venastine.mjs +528 -528
- package/bin/venastine.py +71 -71
- package/cli_commands.py +838 -835
- package/config.yaml +13 -0
- package/config_edit.py +1301 -1301
- package/config_schema.py +971 -966
- package/config_update.py +749 -749
- package/core/agent_activity.py +256 -256
- package/core/ansi.py +452 -452
- package/core/approval.py +137 -137
- package/core/call_view.py +364 -364
- package/core/cancel.py +121 -121
- package/core/chatgpt_auth.py +747 -747
- package/core/claims_view.py +107 -107
- package/core/client.py +2715 -2715
- package/core/commands.py +460 -454
- package/core/compaction.py +1735 -1735
- package/core/config_loader.py +1982 -1982
- package/core/connect.py +686 -686
- package/core/events.py +123 -123
- package/core/file_mentions.py +280 -280
- package/core/interaction.py +411 -411
- package/core/loop.py +2566 -2566
- package/core/memory.py +482 -482
- package/core/pipeline_models.py +190 -190
- package/core/provider_errors.py +234 -234
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -484
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/orchestrator.py +1561 -1561
- package/core/reasoning/output_writer.py +345 -345
- package/core/reasoning/payload_validation.py +524 -524
- package/core/reasoning/pipeline_storage.py +427 -427
- package/core/reasoning/scholar.py +458 -458
- package/core/reasoning/source_corpus.py +433 -433
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -315
- package/core/shell_sessions.py +1553 -1553
- package/credentials.py +307 -307
- package/database.py +333 -333
- package/docs/PRIVACY.md +78 -78
- package/docs/SECURITY.md +108 -108
- package/docs/THIRD_PARTY_NOTICES.md +148 -148
- package/env_secrets.py +118 -118
- package/json_store.py +343 -343
- package/logging_setup.py +295 -295
- package/main.py +3433 -3433
- package/mcp_client/client.py +733 -733
- package/mcp_client/config.py +484 -484
- package/mcp_client/registration.py +343 -343
- package/memories/manager.py +147 -147
- package/memories/tui_commands.py +319 -239
- package/package.json +303 -303
- package/project_init/config_files.py +282 -282
- package/project_init/generator.py +728 -728
- package/project_init/manifest.py +418 -418
- package/project_init/tui_commands.py +147 -147
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +17 -17
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -316
- package/providers.json.example +94 -94
- package/pyproject.toml +149 -149
- package/requirements.txt +295 -295
- package/safety/policy_enforcement.py +1082 -1082
- package/security/askpass.py +419 -419
- package/security/capability.py +207 -207
- package/security/confine.py +484 -484
- package/security/hosted.py +182 -182
- package/security/permissions.py +237 -237
- package/security/protected_paths.py +501 -501
- package/security/sandbox.py +4108 -4108
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -110
- package/storage.py +1338 -1338
- package/tools/base.py +465 -465
- package/tools/builtin/_math_common.py +667 -667
- package/tools/builtin/_net_common.py +267 -267
- package/tools/builtin/arxiv.py +681 -681
- package/tools/builtin/fetch_url.py +371 -371
- package/tools/builtin/file_ops.py +720 -720
- package/tools/builtin/geometry.py +184 -184
- package/tools/builtin/load_skill.py +76 -76
- package/tools/builtin/pin.py +227 -227
- package/tools/builtin/probability_stats.py +224 -224
- package/tools/builtin/project_docs.py +464 -464
- package/tools/builtin/remember.py +175 -175
- package/tools/builtin/shell.py +980 -980
- package/tools/builtin/todo.py +149 -149
- package/tools/builtin/web_search.py +328 -328
- package/tools/context.py +159 -159
- package/tools/isolation.py +387 -387
- package/tools/registry.py +1242 -1242
- package/tui/app.py +8318 -8307
- package/tui/app.tcss +747 -739
- package/tui/commands.py +26 -26
- package/tui/diffs.py +286 -286
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -102
- package/tui/markdown.py +1329 -1329
- package/tui/meters.py +364 -364
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -1197
- package/tui/themes.py +599 -599
- package/tui/widgets.py +3813 -3796
package/core/config_loader.py
CHANGED
|
@@ -1,1982 +1,1982 @@
|
|
|
1
|
-
"""
|
|
2
|
-
core/config_loader.py
|
|
3
|
-
|
|
4
|
-
ROADMAP_v2 §14. Discovers and parses agent/skill `.md` files across the
|
|
5
|
-
three tiers (D8) -- harness (`agents/builtin/`, `skills/builtin/` shipped
|
|
6
|
-
with the repo), project (`.venastine/`, only if trusted per D17), user
|
|
7
|
-
(`~/.config/venastine/`) -- merges `settings.json` across tiers, and
|
|
8
|
-
exposes the skill catalog (frontmatter only, never bodies) for system
|
|
9
|
-
prompt injection.
|
|
10
|
-
|
|
11
|
-
Progressive disclosure: the model sees name+description of every
|
|
12
|
-
discovered skill in its system prompt and must call the `load_skill`
|
|
13
|
-
tool to view a body. Bodies never enter context unrequested.
|
|
14
|
-
|
|
15
|
-
Tier precedence: a harness-tier name can never be shadowed -- a
|
|
16
|
-
project/user file colliding with a builtin is rejected with a warning
|
|
17
|
-
(D18). Project vs user collisions resolve USER-wins (D29): the user tier is
|
|
18
|
-
what you authored; the project tier is what arrived with a directory
|
|
19
|
-
you cloned.
|
|
20
|
-
|
|
21
|
-
`initialize(project_path)` is called once at startup (main.py) and caches
|
|
22
|
-
everything; getters return the cached state. Tests call initialize() with
|
|
23
|
-
their own tmp dirs and reset() afterwards.
|
|
24
|
-
"""
|
|
25
|
-
|
|
26
|
-
import json
|
|
27
|
-
import logging
|
|
28
|
-
import os
|
|
29
|
-
import re
|
|
30
|
-
from dataclasses import dataclass
|
|
31
|
-
from typing import Optional
|
|
32
|
-
|
|
33
|
-
import yaml
|
|
34
|
-
|
|
35
|
-
import config_schema
|
|
36
|
-
from core import workspace_trust
|
|
37
|
-
from json_store import read_regular_text
|
|
38
|
-
|
|
39
|
-
logger = logging.getLogger(__name__)
|
|
40
|
-
|
|
41
|
-
# The repo root (this file lives in <root>/core/). The harness tier is
|
|
42
|
-
# shipped with the harness itself, so it resolves from here, not from cwd.
|
|
43
|
-
HARNESS_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
44
|
-
|
|
45
|
-
NUMBER_TYPES = (int, float)
|
|
46
|
-
|
|
47
|
-
# §45 (SQ6): the closed vocabulary `confidence.grounding_weights`
|
|
48
|
-
# keys against. Re-exported from base.py rather than re-typed, the
|
|
49
|
-
# way payload_validation does -- a second hand-written copy of a
|
|
50
|
-
# vocabulary is a second thing to keep in step.
|
|
51
|
-
from core.reasoning.base import GROUNDING_STATUSES # noqa: E402
|
|
52
|
-
|
|
53
|
-
_FRONTMATTER_DELIM = re.compile(r"^---\s*$", re.MULTILINE)
|
|
54
|
-
|
|
55
|
-
_KNOWN_SETTINGS = {
|
|
56
|
-
"default_provider": str,
|
|
57
|
-
"default_model": str,
|
|
58
|
-
# Batch 25 (#139). The effort level every path runs at unless something
|
|
59
|
-
# more specific speaks: CLI flag > this key > config.DEFAULT_EFFORT,
|
|
60
|
-
# with tui.effort still winning INSIDE the TUI (it predates this key
|
|
61
|
-
# and changing its meaning would surprise existing configs).
|
|
62
|
-
#
|
|
63
|
-
# Deliberately NOT an R12 by-name rejection, and the reason is worth
|
|
64
|
-
# stating so nobody "completes" the pattern: R12 rejects keys that are
|
|
65
|
-
# AUTHORITY. This one is cost -- the worst a cloned repo's settings.json
|
|
66
|
-
# can do here is make runs think harder and spend more tokens, which is
|
|
67
|
-
# compaction-settings territory, not granted-tools territory.
|
|
68
|
-
"effort": str,
|
|
69
|
-
# Batch 27 (#4). The optional per-run SPEND ceiling, in billed
|
|
70
|
-
# input+output tokens. None/absent -- the default -- means UNCAPPED:
|
|
71
|
-
# hard spend limits are the provider's dashboard's job, and a harness
|
|
72
|
-
# cap that misreads as a context limit is the exact defect #4 records.
|
|
73
|
-
# Cost knob, not authority, so effort's merge rule applies (normal
|
|
74
|
-
# tiers; deliberately NOT an R12 by-name rejection).
|
|
75
|
-
"max_token_budget": (int, type(None)),
|
|
76
|
-
"ensemble_mode": bool,
|
|
77
|
-
"ensemble_n": int,
|
|
78
|
-
"compaction": dict,
|
|
79
|
-
"tui": dict,
|
|
80
|
-
"research": dict,
|
|
81
|
-
# ROADMAP_v2 §45 (SQ6). TOP-LEVEL, not nested under `research`:
|
|
82
|
-
# _NESTED_SETTINGS supports exactly one level and both the validator
|
|
83
|
-
# and the cross-tier merge iterate it, so `research.confidence` would
|
|
84
|
-
# need the merge deepened -- the code path review finding F2 broke
|
|
85
|
-
# once. These are cost/quality knobs, not authority, so effort's merge
|
|
86
|
-
# rule applies: normal tiers, deliberately NOT an R12 by-name
|
|
87
|
-
# rejection. The two keys that ARE authority -- critic_model and
|
|
88
|
-
# embedder_model -- are rejected above.
|
|
89
|
-
"confidence": dict,
|
|
90
|
-
"source_scoring": dict,
|
|
91
|
-
}
|
|
92
|
-
# ROADMAP_v2 §25 (R12). The authorization MODE is persistable and the
|
|
93
|
-
# grant LIST deliberately is not.
|
|
94
|
-
#
|
|
95
|
-
# The asymmetry is the design. A persisted mode can only ever ADD prompts,
|
|
96
|
-
# so the worst a hostile settings.json can do is make your runs more
|
|
97
|
-
# annoying. A persisted grant list could only ever REMOVE them -- standing
|
|
98
|
-
# authorization for named tools, carried by any repo you clone, and
|
|
99
|
-
# settings.json is the one config file where project tier beats user tier.
|
|
100
|
-
#
|
|
101
|
-
# `granted_tools` is therefore rejected BY NAME rather than left to the
|
|
102
|
-
# unknown-key branch, because "unknown key" reads as an oversight to be
|
|
103
|
-
# fixed by adding support for it. See _validate_settings.
|
|
104
|
-
_KNOWN_RESEARCH = {
|
|
105
|
-
"approval_mode": str, # "none" (default) or "attended"
|
|
106
|
-
# §20/D9. A MODE, so R12's rule permits persisting it: the worst a
|
|
107
|
-
# hostile settings.json can do here is make a run stop and ask you
|
|
108
|
-
# about corrections it wants to make, which is more friction and never
|
|
109
|
-
# more authority. --no-review escapes it for one run.
|
|
110
|
-
"subagent_review": bool,
|
|
111
|
-
}
|
|
112
|
-
RESEARCH_APPROVAL_MODES = ("none", "attended")
|
|
113
|
-
_KNOWN_COMPACTION = {
|
|
114
|
-
"strength": int,
|
|
115
|
-
"keep_recent_tokens": int,
|
|
116
|
-
# ROADMAP_v2 §21 (M1). This was `buffer_tokens` while §21 was specified
|
|
117
|
-
# as "headroom before the model's context window". It is not headroom:
|
|
118
|
-
# the trigger is a WORKING-SET TARGET -- a window-derived threshold
|
|
119
|
-
# sits at a size a working thread never reaches, while ~40k keeps turns
|
|
120
|
-
# cheap and their tool steps intact. Renamed rather than redefined -- a
|
|
121
|
-
# key that keeps its name and changes its meaning is worse than one
|
|
122
|
-
# that moves, and these keys have been inert since §14 so nothing
|
|
123
|
-
# depends on the old spelling.
|
|
124
|
-
"trigger_tokens": int,
|
|
125
|
-
"warning_margin_tokens": int,
|
|
126
|
-
# §21 M5 / M2 / retry bound.
|
|
127
|
-
"keep_recent_turns": int,
|
|
128
|
-
"strategy": str,
|
|
129
|
-
"max_retries": int,
|
|
130
|
-
}
|
|
131
|
-
# ROADMAP_v2 §16. TUI preferences live here rather than in a dotfile of
|
|
132
|
-
# their own because unknown settings keys RAISE (§14 amendment 1) -- a
|
|
133
|
-
# preference the loader doesn't know about is a startup error, not a
|
|
134
|
-
# silently ignored line. That is the tradeoff the loud-rejection rule
|
|
135
|
-
# buys: adding a preference is a schema change, on purpose.
|
|
136
|
-
_KNOWN_TUI = {
|
|
137
|
-
"theme": str, # one of tui/themes.py's registered names
|
|
138
|
-
"animations": bool, # master switch for the raven + transitions
|
|
139
|
-
"effort": str, # persisted reasoning-effort level (§16)
|
|
140
|
-
"todo_position": str, # §23 slice 2: one of TODO_POSITIONS below
|
|
141
|
-
# §38 (O6): render a turn's reasoning inline, or collapse it to an
|
|
142
|
-
# animated one-line indicator. Defaults to True in tui/app.py rather
|
|
143
|
-
# than to a config.yaml key, following `animations` above -- config.yaml
|
|
144
|
-
# holds no TUI values.
|
|
145
|
-
"show_thinking": bool,
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
# §23 slice 2. Validated AT LOAD and raising, following
|
|
149
|
-
# research.approval_mode rather than tui.theme.
|
|
150
|
-
#
|
|
151
|
-
# tui.theme validates at use and falls back, for a stated reason: the loader
|
|
152
|
-
# cannot know the valid names without importing tui/themes.py, and a stale
|
|
153
|
-
# theme name should not stop the app from starting. Neither applies here --
|
|
154
|
-
# the vocabulary is three words that live in this file, and a position the
|
|
155
|
-
# renderer does not understand would silently put the panel somewhere the
|
|
156
|
-
# user did not ask for. tui.effort had neither check and that was a shipped
|
|
157
|
-
# bug (§16), which is the third precedent and the one that decided it.
|
|
158
|
-
TODO_POSITIONS = ("top", "bottom", "side")
|
|
159
|
-
|
|
160
|
-
# Settings whose value is a nested object. Both the validator and the
|
|
161
|
-
# cross-tier merge below iterate this rather than naming keys twice --
|
|
162
|
-
# `compaction` was the only member until §16 added `tui`, and hardcoding
|
|
163
|
-
# it in two places is what made the shallow-merge defect
|
|
164
|
-
# (review finding F2) possible in the first place.
|
|
165
|
-
_KNOWN_CONFIDENCE = {
|
|
166
|
-
"grounding_weight_factor": float,
|
|
167
|
-
"critic_weight_factor": float,
|
|
168
|
-
"assumption_flag_penalty": float,
|
|
169
|
-
# Retired by ROADMAP_v3 §90 and still KNOWN, as `ensemble_n` is (E4):
|
|
170
|
-
# unknown keys raise, so dropping it would stop every settings.json
|
|
171
|
-
# that sets it from loading. effective_confidence warns instead.
|
|
172
|
-
"disagreement_penalty_factor": float,
|
|
173
|
-
"non_factual_score_cap": float,
|
|
174
|
-
"source_quality_floor": float,
|
|
175
|
-
"authority_full_credit": float,
|
|
176
|
-
"grounding_weights": dict,
|
|
177
|
-
"tier_thresholds": dict,
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
_KNOWN_SOURCE_SCORING = {
|
|
181
|
-
"similarity_floor": float,
|
|
182
|
-
"similarity_ceiling": float,
|
|
183
|
-
"authority_adjustment_cap": float,
|
|
184
|
-
"window_chars": int,
|
|
185
|
-
"max_windows": int,
|
|
186
|
-
"claim_min_chars": int,
|
|
187
|
-
"top_k": int,
|
|
188
|
-
"domain_overrides": dict,
|
|
189
|
-
"venue_weight": float,
|
|
190
|
-
"citation_weight": float,
|
|
191
|
-
"author_weight": float,
|
|
192
|
-
"h_saturation": int,
|
|
193
|
-
"min_cohort_size": int,
|
|
194
|
-
"min_citation_age_days": int,
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
_NESTED_SETTINGS = {
|
|
198
|
-
"compaction": _KNOWN_COMPACTION,
|
|
199
|
-
"tui": _KNOWN_TUI,
|
|
200
|
-
"research": _KNOWN_RESEARCH,
|
|
201
|
-
"confidence": _KNOWN_CONFIDENCE,
|
|
202
|
-
"source_scoring": _KNOWN_SOURCE_SCORING,
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
@dataclass
|
|
207
|
-
class SkillDef:
|
|
208
|
-
name: str
|
|
209
|
-
description: str
|
|
210
|
-
# ROADMAP_v2 §19 (K2): the tools this skill's methodology DEPENDS on,
|
|
211
|
-
# not tools it grants. It cannot grant any -- ToolContext.allowed_tools
|
|
212
|
-
# is a whitelist that narrows, and D14 forbids widening -- so this is
|
|
213
|
-
# checked at activation and reported, never applied. See SkillManager.
|
|
214
|
-
additional_tools: list
|
|
215
|
-
body: str
|
|
216
|
-
tier: str
|
|
217
|
-
path: str
|
|
218
|
-
# §19 (K4): folder path from the tier root, posix separators, "" for a
|
|
219
|
-
# skill sitting directly in skills/. Metadata, not identity -- the name
|
|
220
|
-
# stays global, so load_skill, /skill and D18's collision rule are
|
|
221
|
-
# untouched by where a file lives. Used to GROUP the catalog, which is
|
|
222
|
-
# the point: progressive disclosure puts every skill's name and
|
|
223
|
-
# description in every system prompt, and a flat list of forty is
|
|
224
|
-
# exactly what that design exists to avoid.
|
|
225
|
-
category: str = ""
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
@dataclass
|
|
229
|
-
class AgentDef:
|
|
230
|
-
name: str
|
|
231
|
-
description: str
|
|
232
|
-
model: Optional[str]
|
|
233
|
-
provider: Optional[str]
|
|
234
|
-
allowed_tools: Optional[list]
|
|
235
|
-
approval_overrides: dict
|
|
236
|
-
use_project_context: bool
|
|
237
|
-
use_memory: bool
|
|
238
|
-
max_steps: Optional[int]
|
|
239
|
-
body: str
|
|
240
|
-
tier: str
|
|
241
|
-
path: str
|
|
242
|
-
# ROADMAP_v2 §32 (A3), #69. Whether spawn_subagent can actually
|
|
243
|
-
# FEED this agent -- i.e. whether a task string, as the first
|
|
244
|
-
# message of a fresh thread, is the input its body expects.
|
|
245
|
-
#
|
|
246
|
-
# None means UNDECLARED, and what that means depends on who wrote
|
|
247
|
-
# the file. For the harness tier it is a build error, caught by
|
|
248
|
-
# assert_spawnable_declared below: our own omission is a bug, and
|
|
249
|
-
# R13's rule applies -- omission has to be a detectable mistake
|
|
250
|
-
# rather than an inherited answer. For the user and project tiers
|
|
251
|
-
# it reads as False, because a third party's silence must get the
|
|
252
|
-
# SAFE answer: an agent written for /agent and advertised as
|
|
253
|
-
# spawnable does not fail, it under-performs, and the parent has
|
|
254
|
-
# no way to tell (which is #69's actual complaint).
|
|
255
|
-
#
|
|
256
|
-
# IT IS A PERMISSION SINCE A16 (ROADMAP_v3 section 62), and this
|
|
257
|
-
# comment said the opposite for as long as it was true. Consulted by
|
|
258
|
-
# prompts.system_prompts.agent_catalog_text AND by
|
|
259
|
-
# agents.subagent_tool.refusal_reason, which refuses a spawn of
|
|
260
|
-
# anything whose value is not True.
|
|
261
|
-
#
|
|
262
|
-
# WHY IT CHANGED. "Advertisement only" meant `spawn_subagent` would
|
|
263
|
-
# spawn any agent the loader had discovered, by name -- including one
|
|
264
|
-
# the asymmetry above had deliberately left off the catalog. So the
|
|
265
|
-
# measured consequence of a stranger's silence was INVISIBLE rather
|
|
266
|
-
# than unreachable: a project-tier agent omitting the field was absent
|
|
267
|
-
# from the prompt a user might read and callable by any text that named
|
|
268
|
-
# it. For the careless author the default was safe; for the adversarial
|
|
269
|
-
# one it was the quieter option, which inverts the ranking the SAFE
|
|
270
|
-
# answer above is reaching for. Its own sibling `spawn_targets` had
|
|
271
|
-
# already chosen refusal for the same class of problem.
|
|
272
|
-
#
|
|
273
|
-
# C6 still caps a child's tools and the depth limit still applies;
|
|
274
|
-
# those were never what this field was about. D24's rule
|
|
275
|
-
# ("advertising an uncallable tool is not harmless: the model keeps
|
|
276
|
-
# choosing it") still applies to the catalog's entries -- and now its
|
|
277
|
-
# mirror applies too: a catalog that hides a REACHABLE agent is the
|
|
278
|
-
# same defect read the other way.
|
|
279
|
-
spawnable: Optional[bool] = None
|
|
280
|
-
# WHICH agents this one may spawn. `spawnable` above is the other
|
|
281
|
-
# half of the same question -- that one is about being spawned, this
|
|
282
|
-
# one about spawning -- and they are separate fields because an agent
|
|
283
|
-
# can be either, both or neither.
|
|
284
|
-
#
|
|
285
|
-
# None means NO OPINION: every spawnable agent, which is what every
|
|
286
|
-
# definition written before this field got and therefore what it has
|
|
287
|
-
# to keep meaning. A list restricts, and only ever restricts: it
|
|
288
|
-
# composes by intersection on ToolContext exactly as allowed_tools
|
|
289
|
-
# does, so it cannot be widened by nesting.
|
|
290
|
-
#
|
|
291
|
-
# It is a REFUSAL, unlike `spawnable`'s advertisement. C6 already
|
|
292
|
-
# narrows a child's tools to the parent's, which is the wrong answer
|
|
293
|
-
# when the parent should not be delegating that work at all: `plan`
|
|
294
|
-
# spawning `build` produced a build stripped of `write` and `edit`,
|
|
295
|
-
# degraded exactly the way A4 describes and invisible from both ends.
|
|
296
|
-
# Read by agents.subagent_tool.refusal_reason (the refusal) and by
|
|
297
|
-
# prompts.system_prompts.agent_catalog_text (D24: an agent that will
|
|
298
|
-
# be refused is not advertised in the first place).
|
|
299
|
-
spawn_targets: Optional[list] = None
|
|
300
|
-
|
|
301
|
-
@property
|
|
302
|
-
def pins_model(self) -> bool:
|
|
303
|
-
"""Whether this file names EITHER half of its model pair.
|
|
304
|
-
|
|
305
|
-
One predicate, because three readers ask it and have to agree:
|
|
306
|
-
the subagent spawn, which takes the `/submodel` pair only for an
|
|
307
|
-
agent that pins neither half (ROADMAP_v3 §57, SM6); the note a
|
|
308
|
-
`/model` switch owes an active agent that pins; and `/submodel`'s
|
|
309
|
-
list of the agents it does not reach. A half pin opts out WHOLE:
|
|
310
|
-
a model name means nothing against the wrong provider, so filling
|
|
311
|
-
the missing half from `/submodel` would build a pair nobody chose.
|
|
312
|
-
"""
|
|
313
|
-
return bool(self.model or self.provider)
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
def _user_config_dir() -> str:
|
|
317
|
-
# Call-time resolution so tests can redirect it without touching the
|
|
318
|
-
# real user config dir.
|
|
319
|
-
return os.path.expanduser("~/.config/venastine")
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
def _catalog_text(value) -> str:
|
|
323
|
-
"""One line, bounded, for a string that enters every system prompt.
|
|
324
|
-
|
|
325
|
-
Applied to a skill's and an agent's `name` and `description` at
|
|
326
|
-
PARSE TIME, for every tier -- ROADMAP_v2 §32 A5, #131.
|
|
327
|
-
|
|
328
|
-
TWO RULES, ONE FUNCTION, because they are one rule about one kind
|
|
329
|
-
of string:
|
|
330
|
-
|
|
331
|
-
COLLAPSE WHITESPACE, and this is the security half. The catalogs
|
|
332
|
-
render `- {name}: {description}`, so a value containing a
|
|
333
|
-
newline does not merely look untidy -- it LEAVES ITS BULLET.
|
|
334
|
-
A project skill described with a YAML block scalar holding
|
|
335
|
-
`## Available tools` renders as a top-level section of the
|
|
336
|
-
system prompt, indistinguishable from one this harness wrote.
|
|
337
|
-
Driven, and it worked.
|
|
338
|
-
|
|
339
|
-
CAP THE LENGTH, and this is the budget half. Progressive
|
|
340
|
-
disclosure puts every name and description in every prompt of
|
|
341
|
-
every run; without a bound, one project file sets that cost.
|
|
342
|
-
|
|
343
|
-
TRUNCATE RATHER THAN REFUSE THE FILE. `_parse_md_file` skips a
|
|
344
|
-
malformed definition, which is right for `additional_tools` --
|
|
345
|
-
a non-string element crashes its first consumer. A description is
|
|
346
|
-
advisory prose read by a model, so it degrades gracefully, and
|
|
347
|
-
losing a whole working skill over the length of its summary would
|
|
348
|
-
be the worse trade. The ellipsis is visible on purpose: a model
|
|
349
|
-
reading a cut-off summary should be able to tell.
|
|
350
|
-
|
|
351
|
-
EVERY TIER, not just project. A rule that applies only to
|
|
352
|
-
untrusted input is a second source of truth about what a catalog
|
|
353
|
-
entry is, and the harness tier is where a regression would be
|
|
354
|
-
least visible -- so the shipped files are held to it too, and
|
|
355
|
-
tests/test_catalog_text.py asserts they already comply.
|
|
356
|
-
"""
|
|
357
|
-
cap = config_schema.current().max_catalog_text_chars
|
|
358
|
-
text = " ".join(str(value).split())
|
|
359
|
-
if len(text) <= cap:
|
|
360
|
-
return text
|
|
361
|
-
return text[:cap - 1].rstrip() + "\u2026"
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
def _parse_frontmatter(text: str) -> tuple[dict, str]:
|
|
365
|
-
"""Returns (frontmatter_dict, body). Uses a delimiter that must occupy
|
|
366
|
-
its own line -- NOT text.find("---"), which matches the first bare
|
|
367
|
-
'---' anywhere in the file, including a markdown horizontal rule
|
|
368
|
-
inside the body itself (a very common convention in longer
|
|
369
|
-
instructional content, which is exactly what agent/skill bodies are).
|
|
370
|
-
"""
|
|
371
|
-
matches = list(_FRONTMATTER_DELIM.finditer(text))
|
|
372
|
-
if len(matches) < 2:
|
|
373
|
-
raise ValueError("No valid YAML frontmatter block found (need two '---' lines)")
|
|
374
|
-
# The opening delimiter must be the FIRST thing in the file. Without
|
|
375
|
-
# this, a file with no frontmatter at all but two horizontal rules in
|
|
376
|
-
# its body has the prose between them handed to yaml.safe_load --
|
|
377
|
-
# which usually fails the "is it a mapping / does it have a name"
|
|
378
|
-
# checks downstream, but reports the wrong reason when it does.
|
|
379
|
-
if matches[0].start() != 0:
|
|
380
|
-
raise ValueError(
|
|
381
|
-
"File does not begin with a YAML frontmatter block "
|
|
382
|
-
"(the opening '---' must be the first line)")
|
|
383
|
-
frontmatter_text = text[matches[0].end():matches[1].start()]
|
|
384
|
-
body = text[matches[1].end():].strip()
|
|
385
|
-
return yaml.safe_load(frontmatter_text), body
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
def _parse_md_file(path: str, kind: str, tier: str, category: str = ""):
|
|
389
|
-
"""Parse one agent/skill .md file. Returns a SkillDef/AgentDef, or
|
|
390
|
-
None (with a warning) for malformed files -- a broken definition
|
|
391
|
-
warns and skips rather than taking down startup."""
|
|
392
|
-
try:
|
|
393
|
-
# utf-8-sig, not utf-8: a BOM (Notepad's default on this project's
|
|
394
|
-
# platform) is decoded as a leading , so the frontmatter's
|
|
395
|
-
# ^--- no longer sits at position 0 and the file is skipped with
|
|
396
|
-
# the false diagnosis "does not begin with a YAML frontmatter
|
|
397
|
-
# block". utf-8-sig strips a BOM and is a no-op without one.
|
|
398
|
-
# Regular files only (batch 219, #214): this runs for the trust
|
|
399
|
-
# prompt's listing, before any grant, and a cloned skill linked to
|
|
400
|
-
# /dev/zero was read until memory ran out. The refusal is an
|
|
401
|
-
# OSError, so it warns and skips below like any broken definition.
|
|
402
|
-
text = read_regular_text(path)
|
|
403
|
-
fm, body = _parse_frontmatter(text)
|
|
404
|
-
except (OSError, ValueError, yaml.YAMLError) as e:
|
|
405
|
-
logger.warning("Skipping %s file %s: %s", kind, path, e)
|
|
406
|
-
return None
|
|
407
|
-
if not isinstance(fm, dict):
|
|
408
|
-
logger.warning("Skipping %s file %s: frontmatter is not a YAML mapping", kind, path)
|
|
409
|
-
return None
|
|
410
|
-
name = fm.get("name")
|
|
411
|
-
if not isinstance(name, str) or not name:
|
|
412
|
-
logger.warning("Skipping %s file %s: missing 'name' in frontmatter", kind, path)
|
|
413
|
-
return None
|
|
414
|
-
# #131: the NAME is interpolated into the same catalog line as the
|
|
415
|
-
# description (`- {name}: {description}`), so it is the same
|
|
416
|
-
# injection surface and gets the same treatment. Bounded AFTER the
|
|
417
|
-
# emptiness check, because an all-whitespace name collapses to ""
|
|
418
|
-
# and must be reported as missing rather than registered as a
|
|
419
|
-
# nameless definition that D18's collision rule cannot reason about.
|
|
420
|
-
name = _catalog_text(name)
|
|
421
|
-
if not name:
|
|
422
|
-
logger.warning("Skipping %s file %s: 'name' is only whitespace",
|
|
423
|
-
kind, path)
|
|
424
|
-
return None
|
|
425
|
-
if kind == "skills":
|
|
426
|
-
tools = fm.get("additional_tools") or []
|
|
427
|
-
if not isinstance(tools, list):
|
|
428
|
-
logger.warning("Skipping skill file %s: additional_tools is not a list", path)
|
|
429
|
-
return None
|
|
430
|
-
# Element types, not just the container (review §19-20 f6): the
|
|
431
|
-
# first consumer feeds each element to a startswith() check, so a
|
|
432
|
-
# non-string element parsed cleanly here and crashed /skill at
|
|
433
|
-
# first consumption -- after "Activated" was already shown.
|
|
434
|
-
if not all(isinstance(t, str) for t in tools):
|
|
435
|
-
logger.warning("Skipping skill file %s: additional_tools is not a list of strings", path)
|
|
436
|
-
return None
|
|
437
|
-
return SkillDef(
|
|
438
|
-
name=name,
|
|
439
|
-
description=_catalog_text(fm.get("description", "")),
|
|
440
|
-
additional_tools=tools,
|
|
441
|
-
body=body,
|
|
442
|
-
tier=tier,
|
|
443
|
-
path=path,
|
|
444
|
-
category=category,
|
|
445
|
-
)
|
|
446
|
-
allowed = fm.get("allowed_tools")
|
|
447
|
-
if allowed is not None and not isinstance(allowed, list):
|
|
448
|
-
logger.warning("Skipping agent file %s: allowed_tools is not a list", path)
|
|
449
|
-
return None
|
|
450
|
-
# ELEMENT TYPES TOO, not just the container. This is the lesson the
|
|
451
|
-
# skills branch above already paid for with `additional_tools`: a
|
|
452
|
-
# non-string element parses cleanly here and fails at first
|
|
453
|
-
# consumption, layers away from the file that caused it. THIS FIELD IS
|
|
454
|
-
# THE ONE THE RULE WAS NEVER APPLIED TO (batch 155) -- it sat under
|
|
455
|
-
# the comment stating it, with the container checked and the elements
|
|
456
|
-
# not.
|
|
457
|
-
#
|
|
458
|
-
# Its two consumers fail in OPPOSITE directions, both measured. A
|
|
459
|
-
# hashable non-string (`[read, 123]`) survives `set()` and then
|
|
460
|
-
# matches nothing, so the whitelist silently holds one fewer entry
|
|
461
|
-
# than it reads as -- the whitelist-typo class
|
|
462
|
-
# tests/test_shipped_roster.py documents for misspelled strings. An
|
|
463
|
-
# unhashable one (`[read, [write]]`, `[read, {write: true}]`) raises
|
|
464
|
-
# `TypeError: unhashable type` out of `AgentManager.active_context`
|
|
465
|
-
# and `child_context`, which are the single place this `Optional[list]`
|
|
466
|
-
# becomes `ToolContext.allowed_tools`'s declared `Optional[set[str]]`.
|
|
467
|
-
#
|
|
468
|
-
# SKIP, NOT DROP THE ELEMENT, for the reason `approval_overrides`
|
|
469
|
-
# states below: half a whitelist is worse than no file at all for a
|
|
470
|
-
# field whose job is deciding what an agent may call.
|
|
471
|
-
if allowed is not None and not all(isinstance(t, str) for t in allowed):
|
|
472
|
-
logger.warning(
|
|
473
|
-
"Skipping agent file %s: allowed_tools is not a list of strings",
|
|
474
|
-
path)
|
|
475
|
-
return None
|
|
476
|
-
# The consumer here is a set membership test against an agent NAME, so
|
|
477
|
-
# a nested list or a mapping raises where that set is built and a
|
|
478
|
-
# hashable non-string sits inert and never matches -- the same pair
|
|
479
|
-
# measured one field up.
|
|
480
|
-
targets = fm.get("spawn_targets")
|
|
481
|
-
if targets is not None and not isinstance(targets, list):
|
|
482
|
-
logger.warning(
|
|
483
|
-
"Skipping agent file %s: spawn_targets is not a list", path)
|
|
484
|
-
return None
|
|
485
|
-
if targets is not None and not all(isinstance(t, str) for t in targets):
|
|
486
|
-
logger.warning(
|
|
487
|
-
"Skipping agent file %s: spawn_targets is not a list of strings",
|
|
488
|
-
path)
|
|
489
|
-
return None
|
|
490
|
-
overrides = fm.get("approval_overrides") or {}
|
|
491
|
-
if not isinstance(overrides, dict):
|
|
492
|
-
logger.warning("Skipping agent file %s: approval_overrides is not a mapping", path)
|
|
493
|
-
return None
|
|
494
|
-
# THE VALUES TOO, not just the mapping (batch 150). The two scalar
|
|
495
|
-
# booleans below are validated for the reason stated there, and this
|
|
496
|
-
# field is the third one a restrictive agent definition uses to opt
|
|
497
|
-
# IN to prompting -- so stopping at the container left
|
|
498
|
-
# `web_search: "false"` parsing to the STRING, and
|
|
499
|
-
# requires_approval() returning 'false' from a function documented to
|
|
500
|
-
# return a bool. Measured: no value widens a gate, because approval
|
|
501
|
-
# ORs and every falsy value is already indistinguishable from the key
|
|
502
|
-
# being absent -- so this is the type boundary, not an escalation.
|
|
503
|
-
# Same warn-and-skip contract as every other typed field here; a
|
|
504
|
-
# per-key repair would leave half a policy, which is worse than no
|
|
505
|
-
# file at all for a field whose job is adding prompts.
|
|
506
|
-
if not all(isinstance(v, bool) for v in overrides.values()):
|
|
507
|
-
bad = sorted(k for k, v in overrides.items()
|
|
508
|
-
if not isinstance(v, bool))
|
|
509
|
-
logger.warning(
|
|
510
|
-
"Skipping agent file %s: approval_overrides values must be "
|
|
511
|
-
"booleans; %s %s not.",
|
|
512
|
-
path, ", ".join(repr(k) for k in bad),
|
|
513
|
-
"is" if len(bad) == 1 else "are")
|
|
514
|
-
return None
|
|
515
|
-
max_steps = fm.get("max_steps")
|
|
516
|
-
# REPAIR, NOT REJECT (#45, batch 16 -- explicit owner decision). This
|
|
517
|
-
# guard used to SKIP the whole file for a bad max_steps, which made it
|
|
518
|
-
# the one frontmatter field whose typo cost the agent entirely while
|
|
519
|
-
# every sibling produced a warning-and-carry-on. Worse, a negative int
|
|
520
|
-
# passed the type check and crashed `_run` with an AttributeError on
|
|
521
|
-
# `response.stop_reason` three layers away from the line that caused it.
|
|
522
|
-
#
|
|
523
|
-
# Now: any unusable value -- non-int, YAML's bool-reading of yes/on,
|
|
524
|
-
# or < 1 (`isinstance(True, int)` is True in Python) -- is repaired to
|
|
525
|
-
# None with a warning NAMING the original. None is what "not declared"
|
|
526
|
-
# means downstream, so every call site's existing
|
|
527
|
-
# `max_steps or config.MAX_ITERATIONS` fallback applies unchanged, and
|
|
528
|
-
# the warning's default figure and the one that takes effect are the
|
|
529
|
-
# same number by construction rather than by two copies staying in
|
|
530
|
-
# step. The belt for values that bypass this loader lives in
|
|
531
|
-
# core/loop._run, which raises instead of repairing: a programmatic
|
|
532
|
-
# caller passing a bad value directly has no file to lose, only a bug
|
|
533
|
-
# worth naming.
|
|
534
|
-
if max_steps is not None:
|
|
535
|
-
valid = (isinstance(max_steps, int)
|
|
536
|
-
and not isinstance(max_steps, bool)
|
|
537
|
-
and max_steps >= 1)
|
|
538
|
-
if not valid:
|
|
539
|
-
logger.warning(
|
|
540
|
-
"Agent file %s: max_steps must be a positive integer, got "
|
|
541
|
-
"%r; defaulting to %s (config.MAX_ITERATIONS).",
|
|
542
|
-
path, max_steps, config_schema.current().max_iterations)
|
|
543
|
-
max_steps = None
|
|
544
|
-
# Booleans are VALIDATED, not coerced. bool("false") is True, so
|
|
545
|
-
# `use_memory: "false"` would invert a deliberate opt-out with no
|
|
546
|
-
# warning -- and these two fields are exactly the ones a restrictive
|
|
547
|
-
# agent definition uses to opt OUT. Same warn-and-skip contract every
|
|
548
|
-
# other typed field in this function has.
|
|
549
|
-
flags = {}
|
|
550
|
-
for key, default in (("use_project_context", False), ("use_memory", True)):
|
|
551
|
-
value = fm.get(key, default)
|
|
552
|
-
if not isinstance(value, bool):
|
|
553
|
-
logger.warning(
|
|
554
|
-
"Skipping agent file %s: %s is not a boolean", path, key)
|
|
555
|
-
return None
|
|
556
|
-
flags[key] = value
|
|
557
|
-
spawnable = fm.get("spawnable")
|
|
558
|
-
if spawnable is not None and not isinstance(spawnable, bool):
|
|
559
|
-
logger.warning(
|
|
560
|
-
"Skipping agent file %s: spawnable is not a boolean", path)
|
|
561
|
-
return None
|
|
562
|
-
# STRINGS OR ABSENT, and these are the other two fields the rule above
|
|
563
|
-
# was never applied to (batch 155). Both were read straight out of
|
|
564
|
-
# frontmatter and handed to code that has always assumed a string, so
|
|
565
|
-
# a well-typed-but-wrong-SHAPED value loaded clean here and failed
|
|
566
|
-
# layers away with nothing naming the file. Measured against the real
|
|
567
|
-
# resolvers, nothing monkeypatched: `provider: 123` reaches
|
|
568
|
-
# `client.api_initialization` and raises `ValueError: Unknown
|
|
569
|
-
# provider: 123`, which names the provider and not the file that
|
|
570
|
-
# supplied it; `model: [a, b]` reaches the effort cache's
|
|
571
|
-
# `(provider_name, model)` key and raises `TypeError: unhashable
|
|
572
|
-
# type: 'list'`, which names neither. Both arrive there through
|
|
573
|
-
# `agents/subagent_tool.py` and `tui/app.py`, which pass
|
|
574
|
-
# `agent.model or ...` through verbatim.
|
|
575
|
-
#
|
|
576
|
-
# SKIP RATHER THAN REPAIR TO None, which is where this parts company
|
|
577
|
-
# with `max_steps` above (#45). None is a REAL value for both fields
|
|
578
|
-
# and means "inherit the session's", so repairing would leave the
|
|
579
|
-
# agent available and running on a model nobody chose for it. A file
|
|
580
|
-
# names a model when being a DIFFERENT model is the point, and §11's
|
|
581
|
-
# critic rationale is the sharpest form of that -- "a model checking
|
|
582
|
-
# its own output for errors shares that model's blind spots". A bad
|
|
583
|
-
# `max_steps` has no honest reading, so repairing it invents nothing;
|
|
584
|
-
# an unreadable `model` has one, and it is the wrong one.
|
|
585
|
-
for key in ("model", "provider"):
|
|
586
|
-
value = fm.get(key)
|
|
587
|
-
if value is not None and not isinstance(value, str):
|
|
588
|
-
logger.warning(
|
|
589
|
-
"Skipping agent file %s: %s is not a string", path, key)
|
|
590
|
-
return None
|
|
591
|
-
return AgentDef(
|
|
592
|
-
name=name,
|
|
593
|
-
description=_catalog_text(fm.get("description", "")),
|
|
594
|
-
model=fm.get("model"),
|
|
595
|
-
provider=fm.get("provider"),
|
|
596
|
-
allowed_tools=allowed,
|
|
597
|
-
approval_overrides=overrides,
|
|
598
|
-
use_project_context=flags["use_project_context"],
|
|
599
|
-
use_memory=flags["use_memory"],
|
|
600
|
-
spawnable=spawnable,
|
|
601
|
-
spawn_targets=targets,
|
|
602
|
-
max_steps=max_steps,
|
|
603
|
-
body=body,
|
|
604
|
-
tier=tier,
|
|
605
|
-
path=path,
|
|
606
|
-
)
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
def _tier_dirs(kind: str, project_path: str, trusted: bool) -> list:
|
|
610
|
-
"""Search order, and _discover() is first-wins, so this list IS the
|
|
611
|
-
precedence rule: harness > user > project.
|
|
612
|
-
|
|
613
|
-
ORDER CHANGED IN §17 (D29). It was harness > project > user, i.e. the
|
|
614
|
-
conventional "more specific config wins". That is the wrong default
|
|
615
|
-
here, because "project" does not mean "more specific" -- it means "it
|
|
616
|
-
arrived with a directory you cloned", which is the entire premise of
|
|
617
|
-
D17. The user tier is the one you actually authored, so it should
|
|
618
|
-
never be silently displaced by the tier you merely trusted once.
|
|
619
|
-
|
|
620
|
-
Harness stays first and un-overridable (D18, unchanged): a project can
|
|
621
|
-
still never impersonate a builtin.
|
|
622
|
-
|
|
623
|
-
Note this only ever breaks SAME-NAME ties. Non-colliding definitions
|
|
624
|
-
from every tier all load -- _discover() accumulates a union and
|
|
625
|
-
consults precedence only when a name repeats.
|
|
626
|
-
"""
|
|
627
|
-
dirs = [("harness", os.path.join(HARNESS_ROOT, kind, "builtin"))]
|
|
628
|
-
dirs.append(("user", os.path.join(_user_config_dir(), kind)))
|
|
629
|
-
if trusted:
|
|
630
|
-
project_dir = os.path.join(
|
|
631
|
-
workspace_trust.venastine_dir(project_path), kind)
|
|
632
|
-
# #18. A SYMLINK at the project tier root escapes the D17 hash.
|
|
633
|
-
# `os.walk` follows the path handed to it as `top` but not
|
|
634
|
-
# symlinked subdirectories, and the two sides of the trust
|
|
635
|
-
# boundary start from different places: workspace_trust walks from
|
|
636
|
-
# `.venastine`, so a symlinked `skills/` is a subdirectory it will
|
|
637
|
-
# not descend, while _md_files walks from `.venastine/skills`, so
|
|
638
|
-
# it IS the top and gets followed. Directory names never enter the
|
|
639
|
-
# hash either, so the link contributes nothing -- not even its own
|
|
640
|
-
# name. The pointed-to definitions therefore load as project-tier
|
|
641
|
-
# agents/skills while being absent from the trust prompt's listing
|
|
642
|
-
# AND from the hash, so their bodies can be rewritten freely after
|
|
643
|
-
# one grant and `is_trusted()` keeps returning True.
|
|
644
|
-
#
|
|
645
|
-
# Treated as ABSENT rather than loaded-and-warned, matching the
|
|
646
|
-
# module's posture for untrusted content generally. The payload
|
|
647
|
-
# here is not data but instructions: a project-tier body becomes
|
|
648
|
-
# system-prompt content for the user's sessions.
|
|
649
|
-
#
|
|
650
|
-
# Scoped to the PROJECT tier deliberately. Harness and user
|
|
651
|
-
# directories are not hashed by anything, so a symlink in them is
|
|
652
|
-
# not a trust question -- and symlinking `~/.config/venastine/`
|
|
653
|
-
# into a dotfiles repo is a legitimate thing an operator does to
|
|
654
|
-
# their own config.
|
|
655
|
-
if os.path.islink(project_dir):
|
|
656
|
-
logger.warning(
|
|
657
|
-
"Ignoring project %s directory %s: it is a symlink, and "
|
|
658
|
-
"symlinked tier roots are invisible to the workspace-trust "
|
|
659
|
-
"content hash (see issue #18). Move the definitions inside "
|
|
660
|
-
".venastine/ to load them.", kind, project_dir)
|
|
661
|
-
else:
|
|
662
|
-
dirs.append(("project", project_dir))
|
|
663
|
-
return dirs
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
def _md_files(directory: str, recursive: bool):
|
|
667
|
-
"""(absolute path, category) for every .md file in a tier directory.
|
|
668
|
-
|
|
669
|
-
ROADMAP_v2 §19 (K4): SKILLS nest under category folders --
|
|
670
|
-
`skills/builtin/<category>/<skill>.md` -- and agents stay flat, since
|
|
671
|
-
`agents/builtin/` holds one file and speculative structure for it
|
|
672
|
-
would be structure nobody asked for. One branch here rather than a
|
|
673
|
-
second discovery function, so tier handling and collision handling
|
|
674
|
-
cannot diverge between the two kinds.
|
|
675
|
-
|
|
676
|
-
The category is DERIVED from the folder, never authored: a `category:`
|
|
677
|
-
frontmatter key would immediately be able to disagree with where the
|
|
678
|
-
file actually sits, and then two places would claim to know.
|
|
679
|
-
|
|
680
|
-
Walk order matches workspace_trust.content_files() -- dirs.sort() plus
|
|
681
|
-
sorted(files) -- because same-name collisions resolve first-wins, so a
|
|
682
|
-
nondeterministic order would make WHICH definition loads depend on the
|
|
683
|
-
filesystem.
|
|
684
|
-
"""
|
|
685
|
-
if not recursive:
|
|
686
|
-
return [
|
|
687
|
-
(os.path.join(directory, f), "")
|
|
688
|
-
for f in sorted(os.listdir(directory)) if f.endswith(".md")
|
|
689
|
-
]
|
|
690
|
-
out = []
|
|
691
|
-
for dirpath, dirs, files in os.walk(directory):
|
|
692
|
-
dirs.sort()
|
|
693
|
-
category = os.path.relpath(dirpath, directory).replace(os.sep, "/")
|
|
694
|
-
if category == ".":
|
|
695
|
-
category = ""
|
|
696
|
-
for fname in sorted(files):
|
|
697
|
-
if fname.endswith(".md"):
|
|
698
|
-
out.append((os.path.join(dirpath, fname), category))
|
|
699
|
-
return out
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
def _discover(kind: str, project_path: str, trusted: bool) -> dict:
|
|
703
|
-
found = {}
|
|
704
|
-
for tier, directory in _tier_dirs(kind, project_path, trusted):
|
|
705
|
-
if not os.path.isdir(directory):
|
|
706
|
-
continue
|
|
707
|
-
for path, category in _md_files(directory, recursive=(kind == "skills")):
|
|
708
|
-
defn = _parse_md_file(path, kind, tier, category)
|
|
709
|
-
if defn is None:
|
|
710
|
-
continue
|
|
711
|
-
if defn.name in found:
|
|
712
|
-
winner = found[defn.name]
|
|
713
|
-
if winner.tier == "harness":
|
|
714
|
-
logger.warning(
|
|
715
|
-
"%s-tier %s %r collides with a harness builtin; "
|
|
716
|
-
"the builtin wins and the %s file is ignored (D18).",
|
|
717
|
-
tier, kind, defn.name, tier,
|
|
718
|
-
)
|
|
719
|
-
else:
|
|
720
|
-
# Cross-tier, USER beats project for agents and skills:
|
|
721
|
-
# _tier_dirs returns [harness, user, project] and this
|
|
722
|
-
# loop is first-wins. That is D29, not D8 -- D8 is where
|
|
723
|
-
# definitions are discovered, D29 is the §17 inversion
|
|
724
|
-
# that put user ahead of project, because "project" does
|
|
725
|
-
# not mean "more specific", it means "it arrived with a
|
|
726
|
-
# directory you cloned". (This comment said the opposite
|
|
727
|
-
# and cited D8; it was pre-D29 text that survived the
|
|
728
|
-
# change. See audit #20 -- the warning below was always
|
|
729
|
-
# right, because it names the winner dynamically.)
|
|
730
|
-
#
|
|
731
|
-
# settings.json deliberately goes the OTHER way,
|
|
732
|
-
# project-over-user, at _load_merged_settings -- because
|
|
733
|
-
# the trust prompt shows its values verbatim. Two
|
|
734
|
-
# opposite rules in one module is exactly what a comment
|
|
735
|
-
# here should disambiguate.
|
|
736
|
-
#
|
|
737
|
-
# Same-tier is alphabetical-first-wins. None of this is
|
|
738
|
-
# guessable from the outside, so say so:
|
|
739
|
-
# Silently shadowing a definition is how someone spends
|
|
740
|
-
# an afternoon editing a file that is never loaded.
|
|
741
|
-
logger.warning(
|
|
742
|
-
"%s-tier %s %r at %s is shadowed by the %s-tier "
|
|
743
|
-
"definition at %s, which wins.",
|
|
744
|
-
tier, kind, defn.name, defn.path, winner.tier,
|
|
745
|
-
winner.path,
|
|
746
|
-
)
|
|
747
|
-
continue
|
|
748
|
-
found[defn.name] = defn
|
|
749
|
-
return found
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
def _type_ok(value, expected) -> bool:
|
|
753
|
-
# bool is a subclass of int in Python, so int-typed keys must reject
|
|
754
|
-
# booleans explicitly, and bool-typed keys must reject ints -- including
|
|
755
|
-
# through a tuple expected-type: isinstance(True, (int, type(None))) is
|
|
756
|
-
# True, and max_token_budget=True would otherwise sail through as 1.
|
|
757
|
-
expecteds = expected if isinstance(expected, tuple) else (expected,)
|
|
758
|
-
if isinstance(value, bool):
|
|
759
|
-
return bool in expecteds
|
|
760
|
-
ok = False
|
|
761
|
-
for e in expecteds:
|
|
762
|
-
if e is bool:
|
|
763
|
-
continue # a non-bool value can never match bool
|
|
764
|
-
if e is int:
|
|
765
|
-
ok = ok or isinstance(value, int)
|
|
766
|
-
else:
|
|
767
|
-
ok = ok or isinstance(value, e)
|
|
768
|
-
return ok
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
def _validate_settings(data, source: str) -> None:
|
|
772
|
-
"""Loud rejection of unknown keys (a typo must not masquerade as a
|
|
773
|
-
setting) and type checks for known ones. Compaction keys are known
|
|
774
|
-
but not consumed until §21 -- they validate here and warn at init."""
|
|
775
|
-
if not isinstance(data, dict):
|
|
776
|
-
raise ValueError(f"settings.json at {source} is not a JSON object")
|
|
777
|
-
for key, value in data.items():
|
|
778
|
-
if key == "ensemble_models":
|
|
779
|
-
# ROADMAP §10 revisit (E2), and R12's rule applied to a second
|
|
780
|
-
# kind of authority. Rejected BY NAME rather than as an unknown
|
|
781
|
-
# key, because the generic message reads as an oversight to be
|
|
782
|
-
# fixed by adding support for it.
|
|
783
|
-
#
|
|
784
|
-
# Turning ensemble mode on can only spend more of the provider
|
|
785
|
-
# the user already chose. A ROSTER chooses providers -- so a
|
|
786
|
-
# project's settings.json, which beats the user's, could point N
|
|
787
|
-
# research passes at endpoints the user never configured for this
|
|
788
|
-
# work and multiply the run's cost by the length of a list it
|
|
789
|
-
# supplied. §14 already flagged project-tier provider selection as
|
|
790
|
-
# a distinct grant; this is that grant times N.
|
|
791
|
-
raise ValueError(
|
|
792
|
-
f"settings.json at {source}: ensemble_models is deliberately "
|
|
793
|
-
f"not supported -- a roster chooses which providers N research "
|
|
794
|
-
f"passes call, and a project's settings.json beats the "
|
|
795
|
-
f"user's. Set ensemble_models in config.yaml instead "
|
|
796
|
-
f"(same posture as critic_model).")
|
|
797
|
-
if key == "shell_approval_mode":
|
|
798
|
-
# ROADMAP_v2 §28 (G7), and the third application of R12's rule.
|
|
799
|
-
# Rejected BY NAME for the reason the other two are: the
|
|
800
|
-
# generic unknown-key message reads as an oversight someone
|
|
801
|
-
# should fix by adding support, and this omission IS the
|
|
802
|
-
# design.
|
|
803
|
-
#
|
|
804
|
-
# A project's settings.json beats the user's (D29), and it
|
|
805
|
-
# arrives with a directory you cloned. This key decides
|
|
806
|
-
# whether shell commands are asked about at all -- so
|
|
807
|
-
# supporting it would let a cloned repo set "never" and turn
|
|
808
|
-
# `cat ~/.aws/credentials` into an unprompted host read. D17's
|
|
809
|
-
# trust gate is not a substitute: the same prompt covers a
|
|
810
|
-
# README-shaped AGENTS.md, and nobody reading it is deciding
|
|
811
|
-
# about their shell gate.
|
|
812
|
-
raise ValueError(
|
|
813
|
-
f"settings.json at {source}: shell_approval_mode is "
|
|
814
|
-
f"deliberately not supported -- it decides whether shell "
|
|
815
|
-
f"commands are approved at all, and a project's "
|
|
816
|
-
f"settings.json beats the user's. Set "
|
|
817
|
-
f"shell_approval_mode in config.yaml instead (same "
|
|
818
|
-
f"posture as ensemble_models and research.granted_tools).")
|
|
819
|
-
if key in ("critic_model", "embedder_model"):
|
|
820
|
-
# ROADMAP_v2 §45 (SQ7), and the fourth application of R12's
|
|
821
|
-
# rule. Both name a PROVIDER the pipeline will call: the critic
|
|
822
|
-
# sees every claim, and the embedder is sent claim text and the
|
|
823
|
-
# text of every page the run fetched. A project's settings.json
|
|
824
|
-
# beats the user's and arrives with a directory you cloned, so
|
|
825
|
-
# supporting either would let that directory choose where a
|
|
826
|
-
# research run's content is sent.
|
|
827
|
-
#
|
|
828
|
-
# By name rather than as an unknown key for the reason the
|
|
829
|
-
# other three are: the generic message reads as an oversight
|
|
830
|
-
# someone should fix by adding support, and this omission is
|
|
831
|
-
# the design. `/critic` and `/embedder` write to the user-tier
|
|
832
|
-
# store in core/pipeline_models.py, which is the route.
|
|
833
|
-
raise ValueError(
|
|
834
|
-
f"settings.json at {source}: {key} is deliberately not "
|
|
835
|
-
f"supported -- it names a provider this harness sends "
|
|
836
|
-
f"research content to, and a project's settings.json beats "
|
|
837
|
-
f"the user's. Use /critic or /embedder, or set "
|
|
838
|
-
f"critic_model / embedder_model in config.yaml "
|
|
839
|
-
f"(same posture as ensemble_models).")
|
|
840
|
-
if key not in _KNOWN_SETTINGS:
|
|
841
|
-
raise ValueError(f"settings.json at {source}: unknown key {key!r}")
|
|
842
|
-
expected = _KNOWN_SETTINGS[key]
|
|
843
|
-
if not _type_ok(value, expected):
|
|
844
|
-
expected_name = getattr(expected, "__name__", None) \
|
|
845
|
-
or "an int or null"
|
|
846
|
-
raise ValueError(
|
|
847
|
-
f"settings.json at {source}: key {key!r} must be {expected_name}, "
|
|
848
|
-
f"got {type(value).__name__}")
|
|
849
|
-
if key == "max_token_budget" and value is not None and value <= 0:
|
|
850
|
-
# #213. The type gate passes 0 and -5 -- they are ints -- and
|
|
851
|
-
# the loop's check is `used >= cap`, so a cap of zero or less
|
|
852
|
-
# let every turn spend ONE model call and then die reporting a
|
|
853
|
-
# budget problem for what is a typo in this file.
|
|
854
|
-
raise ValueError(
|
|
855
|
-
f"settings.json at {source}: max_token_budget must be a "
|
|
856
|
-
f"positive number of tokens, or null for no cap -- got "
|
|
857
|
-
f"{value}, which would stop every turn after its first "
|
|
858
|
-
f"model call.")
|
|
859
|
-
for section, known_keys in _NESTED_SETTINGS.items():
|
|
860
|
-
block = data.get(section)
|
|
861
|
-
if block is None:
|
|
862
|
-
continue
|
|
863
|
-
if not isinstance(block, dict):
|
|
864
|
-
raise ValueError(
|
|
865
|
-
f"settings.json at {source}: {section!r} must be an object")
|
|
866
|
-
for key, value in block.items():
|
|
867
|
-
if section == "research" and key == "granted_tools":
|
|
868
|
-
# §25 R12: rejected BY NAME, not as an unknown key. The
|
|
869
|
-
# generic message reads as an oversight someone should fix
|
|
870
|
-
# by adding support; this one says the omission is the
|
|
871
|
-
# design, so nobody "fixes" it into standing authorization
|
|
872
|
-
# that a cloned repo would carry.
|
|
873
|
-
raise ValueError(
|
|
874
|
-
f"settings.json at {source}: research.granted_tools is "
|
|
875
|
-
f"deliberately not supported -- a persisted grant list "
|
|
876
|
-
f"would authorise named tools for every future run, and "
|
|
877
|
-
f"a project's settings.json beats the user's. Grant per "
|
|
878
|
-
f"run with --grant / --grant-tools instead.")
|
|
879
|
-
if key not in known_keys:
|
|
880
|
-
raise ValueError(
|
|
881
|
-
f"settings.json at {source}: unknown {section} key {key!r}")
|
|
882
|
-
expected = known_keys[key]
|
|
883
|
-
if not _type_ok(value, expected):
|
|
884
|
-
raise ValueError(
|
|
885
|
-
f"settings.json at {source}: {section} key {key!r} must be "
|
|
886
|
-
f"{expected.__name__}, got {type(value).__name__}")
|
|
887
|
-
if section == "research" and key == "approval_mode" and \
|
|
888
|
-
value not in RESEARCH_APPROVAL_MODES:
|
|
889
|
-
raise ValueError(
|
|
890
|
-
f"settings.json at {source}: research.approval_mode must "
|
|
891
|
-
f"be one of {', '.join(RESEARCH_APPROVAL_MODES)}, "
|
|
892
|
-
f"got {value!r}")
|
|
893
|
-
if section == "tui" and key == "todo_position" and \
|
|
894
|
-
value not in TODO_POSITIONS:
|
|
895
|
-
raise ValueError(
|
|
896
|
-
f"settings.json at {source}: tui.todo_position must be "
|
|
897
|
-
f"one of {', '.join(TODO_POSITIONS)}, got {value!r}")
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
# ---------------------------------------------------------------------------
|
|
901
|
-
# ---- Compaction settings (ROADMAP_v2 §21, D27) ----------------------------
|
|
902
|
-
# ---------------------------------------------------------------------------
|
|
903
|
-
#
|
|
904
|
-
# Resolution lives HERE rather than in core/compaction.py because
|
|
905
|
-
# config_loader already owns the settings merge, and core/compaction.py
|
|
906
|
-
# reaches agents.manager, which imports this module -- resolving there and
|
|
907
|
-
# validating here would be a cycle. One function, called at startup so a
|
|
908
|
-
# bad value is a startup error, and called again per compaction so a
|
|
909
|
-
# per-invocation `/compact --strength 4` composes with the same rules.
|
|
910
|
-
|
|
911
|
-
# settings.json key -> the `config.yaml` field it defaults from.
|
|
912
|
-
#
|
|
913
|
-
# THESE ARE SCHEMA FIELD NAMES, NOT STRINGS LOOKED UP ON A MODULE. This map
|
|
914
|
-
# used to hold `"COMPACTION_TRIGGER_TOKENS"` and friends and reach them with
|
|
915
|
-
# `getattr(config, attr)` -- no default, so renaming or dropping any one of
|
|
916
|
-
# the seven turned `shipped_defaults()` into an AttributeError that surfaced
|
|
917
|
-
# through `initialize()` at startup, and nothing but this dictionary tied the
|
|
918
|
-
# two names together. `config_schema.HarnessConfig` is the source of truth
|
|
919
|
-
# now, so a missing field is a validation error naming the key and a renamed
|
|
920
|
-
# one is an AttributeError here at the same moment either way -- but the names
|
|
921
|
-
# below are checked against the model by
|
|
922
|
-
# `tests/test_config_loader.py::test_every_compaction_default_names_a_schema_field`.
|
|
923
|
-
_COMPACTION_DEFAULTS = {
|
|
924
|
-
"trigger_tokens": "compaction_trigger_tokens",
|
|
925
|
-
"warning_margin_tokens": "compaction_warning_margin_tokens",
|
|
926
|
-
"keep_recent_tokens": "compaction_keep_recent_tokens",
|
|
927
|
-
"keep_recent_turns": "compaction_keep_recent_turns",
|
|
928
|
-
"strength": "compaction_strength",
|
|
929
|
-
"max_retries": "compaction_max_retries",
|
|
930
|
-
"strategy": "compaction_strategy",
|
|
931
|
-
}
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
def shipped_defaults() -> dict:
|
|
935
|
-
"""Every nested knob at the value it takes when no settings.json
|
|
936
|
-
speaks. `{section: {key: default}}`, sections only -- the top-level
|
|
937
|
-
scalars are plain `config.yaml` keys and need no accessor.
|
|
938
|
-
|
|
939
|
-
EXISTS SO A TEMPLATE CANNOT QUOTE A STALE DEFAULT (§24 I15).
|
|
940
|
-
`/init --config` writes a settings.json of defaults, and a settings
|
|
941
|
-
file is unforgiving: `_validate_settings` raises on an unknown key, so
|
|
942
|
-
the scaffold has to be right the first time or the harness will not
|
|
943
|
-
start. Retyping these values there would be a second copy of them,
|
|
944
|
-
which is how the first one goes stale.
|
|
945
|
-
|
|
946
|
-
The two research-tuning builders live further down this file and are
|
|
947
|
-
resolved at call time, which is why this can sit beside the compaction
|
|
948
|
-
map it also reads.
|
|
949
|
-
"""
|
|
950
|
-
cfg = config_schema.current()
|
|
951
|
-
return {
|
|
952
|
-
"compaction": {key: getattr(cfg, field)
|
|
953
|
-
for key, field in _COMPACTION_DEFAULTS.items()},
|
|
954
|
-
"confidence": _confidence_defaults(),
|
|
955
|
-
"source_scoring": _source_scoring_defaults(),
|
|
956
|
-
}
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
def effective_compaction(overrides: Optional[dict] = None,
|
|
960
|
-
warn: bool = False) -> dict:
|
|
961
|
-
"""The compaction values actually in force.
|
|
962
|
-
|
|
963
|
-
A flat {key: value}, and DELIBERATELY STILL FLAT. Where each value
|
|
964
|
-
came from is `compaction_sources()` below, beside this rather than
|
|
965
|
-
inside it: twelve call sites subscript what this returns, so
|
|
966
|
-
`{key: (value, tier)}` breaks every one of them and a parallel dict
|
|
967
|
-
breaks none -- TECHNICAL_DEBT 12's first blocker, which is a decision
|
|
968
|
-
about shape and is unchanged by having built the second half.
|
|
969
|
-
|
|
970
|
-
This docstring used to open "and where they came from" and return a
|
|
971
|
-
flat dict, which is audit #91's finding; then it promised nothing and
|
|
972
|
-
pointed at the register. Now it points at the function that answers.
|
|
973
|
-
|
|
974
|
-
config.yaml default -> user settings.json -> trusted project
|
|
975
|
-
settings.json -> `overrides` (a per-invocation `/compact --strength 4`,
|
|
976
|
-
which applies to that one run and persists nothing). Nearest wins,
|
|
977
|
-
which is the precedence the rest of the config system already uses.
|
|
978
|
-
|
|
979
|
-
RELATIONSHIPS ARE VALIDATED, not just documented -- §21's own
|
|
980
|
-
instruction about warning_margin, which is the one value whose wrong
|
|
981
|
-
setting is not self-correcting: a margin at or above the trigger fires
|
|
982
|
-
the warning after the thing it warns about, which is no warning at all.
|
|
983
|
-
|
|
984
|
-
Raises ValueError on a value that cannot produce coherent trigger
|
|
985
|
-
math. That is the same posture settings.json already takes toward an
|
|
986
|
-
unknown key: a project's settings are content the user may not have
|
|
987
|
-
authored (D17's premise), and while nothing here can destroy anything
|
|
988
|
-
-- the archive is never edited by compaction, which is exactly the
|
|
989
|
-
property that makes this safe -- an absurd trigger would force
|
|
990
|
-
compaction constantly, and every compaction is a real model call the
|
|
991
|
-
user pays for.
|
|
992
|
-
"""
|
|
993
|
-
cfg = config_schema.current()
|
|
994
|
-
|
|
995
|
-
values, _ = _compaction_merge(overrides)
|
|
996
|
-
|
|
997
|
-
strength = values["strength"]
|
|
998
|
-
if strength not in cfg.compaction_target_ratios:
|
|
999
|
-
raise ValueError(
|
|
1000
|
-
f"compaction.strength must be one of "
|
|
1001
|
-
f"{sorted(cfg.compaction_target_ratios)}, got {strength!r}")
|
|
1002
|
-
if values["strategy"] not in cfg.compaction_strategies:
|
|
1003
|
-
raise ValueError(
|
|
1004
|
-
f"compaction.strategy must be one of "
|
|
1005
|
-
f"{', '.join(cfg.compaction_strategies)}, "
|
|
1006
|
-
f"got {values['strategy']!r}")
|
|
1007
|
-
if values["trigger_tokens"] < 1:
|
|
1008
|
-
raise ValueError(
|
|
1009
|
-
f"compaction.trigger_tokens must be positive, got "
|
|
1010
|
-
f"{values['trigger_tokens']}")
|
|
1011
|
-
if values["warning_margin_tokens"] >= values["trigger_tokens"]:
|
|
1012
|
-
raise ValueError(
|
|
1013
|
-
f"compaction.warning_margin_tokens ({values['warning_margin_tokens']}) "
|
|
1014
|
-
f"must be strictly less than trigger_tokens "
|
|
1015
|
-
f"({values['trigger_tokens']}), or the early warning fires after "
|
|
1016
|
-
f"the compaction it is warning about.")
|
|
1017
|
-
if values["keep_recent_tokens"] >= values["trigger_tokens"]:
|
|
1018
|
-
raise ValueError(
|
|
1019
|
-
f"compaction.keep_recent_tokens ({values['keep_recent_tokens']}) "
|
|
1020
|
-
f"must be strictly less than trigger_tokens "
|
|
1021
|
-
f"({values['trigger_tokens']}), or every message is protected at "
|
|
1022
|
-
f"the moment compaction fires and there is nothing to summarize.")
|
|
1023
|
-
for key in ("keep_recent_turns", "max_retries"):
|
|
1024
|
-
if values[key] < 0:
|
|
1025
|
-
raise ValueError(f"compaction.{key} must not be negative, "
|
|
1026
|
-
f"got {values[key]}")
|
|
1027
|
-
|
|
1028
|
-
# M1's arithmetic, enforced rather than left in a comment. The prompt
|
|
1029
|
-
# is re-billed on every step of a tool-using turn, so a turn starting
|
|
1030
|
-
# at T tokens and running k steps spends roughly k*T against the
|
|
1031
|
-
# CONFIGURED spend cap (settings.json max_token_budget). A trigger too
|
|
1032
|
-
# close to that cap lets a thread reach a size where the cap ends the
|
|
1033
|
-
# turn after one response -- compaction would then only ever fire on
|
|
1034
|
-
# the turn AFTER the one it should have saved.
|
|
1035
|
-
#
|
|
1036
|
-
# ONLY MEANINGFUL WHEN A CAP EXISTS (#4). Uncapped -- the default --
|
|
1037
|
-
# nothing competes with compaction: the trigger is a context-size
|
|
1038
|
-
# target and always was; the billing meter never gated it and now
|
|
1039
|
-
# gates nothing at all by default.
|
|
1040
|
-
# WARN ONLY WHEN ASKED, which is once at startup. This function is
|
|
1041
|
-
# on should_compact()'s path, so it runs once per step of every
|
|
1042
|
-
# turn -- an unconditional warning here would repeat a
|
|
1043
|
-
# configuration complaint dozens of times per conversation, which
|
|
1044
|
-
# is how a real warning becomes one nobody reads. The VALIDATION
|
|
1045
|
-
# above still raises on every call; only the advisory is gated.
|
|
1046
|
-
cap = spend_cap()
|
|
1047
|
-
if warn and cap is not None:
|
|
1048
|
-
headroom = cap / max(values["trigger_tokens"], 1)
|
|
1049
|
-
if headroom < 3:
|
|
1050
|
-
logger.warning(
|
|
1051
|
-
"compaction.trigger_tokens (%s) leaves room for only ~%.1f model "
|
|
1052
|
-
"calls within your configured spend cap max_token_budget (%s), "
|
|
1053
|
-
"because a tool-using turn re-sends its whole prompt each "
|
|
1054
|
-
"step. Turns on a nearly-full thread will stop early with "
|
|
1055
|
-
"token_budget_exceeded before compaction gets a chance. "
|
|
1056
|
-
"Lower the trigger or raise the cap.",
|
|
1057
|
-
values["trigger_tokens"], headroom, cap)
|
|
1058
|
-
return values
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
def _compaction_merge(overrides: Optional[dict] = None) -> tuple[dict, dict]:
|
|
1062
|
-
"""`(values, sources)` for the seven compaction keys -- ONE ladder.
|
|
1063
|
-
|
|
1064
|
-
TECHNICAL_DEBT 12, batch 111. Both halves come out of the same three
|
|
1065
|
-
`update()` calls because the alternative is two ladders that have to
|
|
1066
|
-
agree about precedence, and a provenance report that disagrees with
|
|
1067
|
-
the value it describes is worse than no provenance: it is a confident
|
|
1068
|
-
wrong answer to "why is this number what it is".
|
|
1069
|
-
|
|
1070
|
-
`sources` names a tier for every key, always -- `"config.yaml"` when
|
|
1071
|
-
nothing outranks the file -- where `settings_sources()` records only
|
|
1072
|
-
what a settings file set. The difference is deliberate: that one
|
|
1073
|
-
answers "did anything override this", which absence can express, and
|
|
1074
|
-
this one answers "where did this number come from", which it cannot.
|
|
1075
|
-
"""
|
|
1076
|
-
values = shipped_defaults()["compaction"]
|
|
1077
|
-
sources = dict.fromkeys(values, "config.yaml")
|
|
1078
|
-
|
|
1079
|
-
tiers = settings_sources()
|
|
1080
|
-
for key, value in (get_settings().get("compaction") or {}).items():
|
|
1081
|
-
values[key] = value
|
|
1082
|
-
# The tier that WROTE it, not "settings.json": the merge upstream
|
|
1083
|
-
# kept them apart precisely so this line can tell them apart.
|
|
1084
|
-
sources[key] = tiers.get(f"compaction.{key}", "user")
|
|
1085
|
-
|
|
1086
|
-
for key, value in (overrides or {}).items():
|
|
1087
|
-
if value is not None:
|
|
1088
|
-
values[key] = value
|
|
1089
|
-
sources[key] = "override"
|
|
1090
|
-
|
|
1091
|
-
return values, sources
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
def compaction_sources(overrides: Optional[dict] = None) -> dict:
|
|
1095
|
-
"""`{key: tier}` for the seven compaction keys -- where each value in
|
|
1096
|
-
force came from. TECHNICAL_DEBT 12, and D27's third implementation
|
|
1097
|
-
note, whose whole argument was that "compaction feels too aggressive"
|
|
1098
|
-
should be a one-line answer rather than a value three files away.
|
|
1099
|
-
|
|
1100
|
-
Tiers: `"config.yaml"`, `"user"`, `"project"`, `"override"` -- nearest
|
|
1101
|
-
wins, the same ladder `effective_compaction` walks, walked once by the
|
|
1102
|
-
function they share.
|
|
1103
|
-
|
|
1104
|
-
NO VALIDATION HERE, and that is not an oversight. This is a display
|
|
1105
|
-
call on a process whose configuration was validated at startup by
|
|
1106
|
-
`initialize()`; raising from a `/config` explanation would replace a
|
|
1107
|
-
sentence about a number with a traceback about a number.
|
|
1108
|
-
"""
|
|
1109
|
-
return _compaction_merge(overrides)[1]
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
def _read_settings_file(path: str) -> dict:
|
|
1113
|
-
if not os.path.exists(path):
|
|
1114
|
-
return {}
|
|
1115
|
-
# utf-8-sig for the same reason as _parse_md_file: a BOM'd
|
|
1116
|
-
# settings.json makes json.load raise at the first character. And
|
|
1117
|
-
# regular files only, for the same reason too (batch 219).
|
|
1118
|
-
data = json.loads(read_regular_text(path)) # a JSONDecodeError is a ValueError
|
|
1119
|
-
_validate_settings(data, path)
|
|
1120
|
-
return data
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
def user_settings() -> dict:
|
|
1124
|
-
"""The user tier's settings.json, alone and unmerged.
|
|
1125
|
-
|
|
1126
|
-
`get_settings()` answers what is IN FORCE, which is the wrong question
|
|
1127
|
-
for two callers. The merge below needs the user tier before the
|
|
1128
|
-
project tier lands on it, and `/init --config` needs it to say which
|
|
1129
|
-
of the user's own keys the file it is about to write will start
|
|
1130
|
-
deciding instead (§24 I16) -- a project key wins by PRESENCE, so a
|
|
1131
|
-
scaffold of pure defaults still displaces a customised user tier.
|
|
1132
|
-
|
|
1133
|
-
Resolved at call time, like every other path in this module.
|
|
1134
|
-
"""
|
|
1135
|
-
return _read_settings_file(
|
|
1136
|
-
os.path.join(_user_config_dir(), "settings.json"))
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
def _load_merged_settings(project_path: str,
|
|
1140
|
-
trusted: bool) -> tuple[dict, dict]:
|
|
1141
|
-
"""`(values, sources)`. Resolution order: project (trusted) > user.
|
|
1142
|
-
Anything absent falls through to config.yaml defaults at the consumer.
|
|
1143
|
-
|
|
1144
|
-
Nested sections (`_NESTED_SETTINGS`) merge one level deeper than the
|
|
1145
|
-
rest. Every other setting is a scalar, so whole-value replacement IS
|
|
1146
|
-
per-key override for them; for a nested key, a plain dict.update()
|
|
1147
|
-
would let a project setting `buffer_tokens` silently discard the
|
|
1148
|
-
user's `strength` -- a per-key override everywhere else that becomes a
|
|
1149
|
-
wholesale reset here, purely because of the value's type.
|
|
1150
|
-
|
|
1151
|
-
Driven off _NESTED_SETTINGS rather than naming sections inline: §16
|
|
1152
|
-
added `tui` alongside `compaction`, and a second hardcoded section name
|
|
1153
|
-
here is exactly how the first one came to be missed.
|
|
1154
|
-
|
|
1155
|
-
`sources` MAPS EACH SETTING TO THE TIER THAT LAST WROTE IT, dotted for
|
|
1156
|
-
a nested key: `{"compaction.strength": "project"}`. TECHNICAL_DEBT 12,
|
|
1157
|
-
batch 111. It is built HERE, in the same pass as the merge, because
|
|
1158
|
-
this is the only moment the two tiers are still distinguishable -- the
|
|
1159
|
-
entry's own words for why provenance was missing were that it is "not
|
|
1160
|
-
omitted, it is destroyed by the merge", and a second function
|
|
1161
|
-
re-reading both files to work it out afterwards would be two
|
|
1162
|
-
traversals that must agree, which is the shape this module's own
|
|
1163
|
-
docstrings keep naming as how the first one drifts.
|
|
1164
|
-
|
|
1165
|
-
A value the user set and the project did not keeps `user`, which is
|
|
1166
|
-
the whole point of recording it per key rather than per file: one
|
|
1167
|
-
`settings.json` arriving with a cloned directory does not make every
|
|
1168
|
-
compaction value the project's.
|
|
1169
|
-
"""
|
|
1170
|
-
merged = user_settings()
|
|
1171
|
-
sources = _tier_of(merged, "user")
|
|
1172
|
-
if trusted:
|
|
1173
|
-
project = _read_settings_file(os.path.join(
|
|
1174
|
-
workspace_trust.venastine_dir(project_path), "settings.json"))
|
|
1175
|
-
deep = {
|
|
1176
|
-
section: {**merged.get(section, {}), **project.get(section, {})}
|
|
1177
|
-
for section in _NESTED_SETTINGS
|
|
1178
|
-
}
|
|
1179
|
-
merged.update(project)
|
|
1180
|
-
sources.update(_tier_of(project, "project"))
|
|
1181
|
-
for section, value in deep.items():
|
|
1182
|
-
if value:
|
|
1183
|
-
merged[section] = value
|
|
1184
|
-
return merged, sources
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
def _tier_of(settings: dict, tier: str) -> dict:
|
|
1188
|
-
"""`{dotted key: tier}` for everything *settings* actually sets.
|
|
1189
|
-
|
|
1190
|
-
One level deep, for `_NESTED_SETTINGS`' reason and driven off the same
|
|
1191
|
-
table: a section is the only kind of value whose keys are settings in
|
|
1192
|
-
their own right, so a section is the only thing flattened. Its own
|
|
1193
|
-
name is recorded too -- a consumer that reads a whole section, as
|
|
1194
|
-
`tui` does, wants to know who supplied it.
|
|
1195
|
-
"""
|
|
1196
|
-
flat = {}
|
|
1197
|
-
for key, value in settings.items():
|
|
1198
|
-
flat[key] = tier
|
|
1199
|
-
if key in _NESTED_SETTINGS and isinstance(value, dict):
|
|
1200
|
-
for leaf in value:
|
|
1201
|
-
flat[f"{key}.{leaf}"] = tier
|
|
1202
|
-
return flat
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
_state: Optional[dict] = None
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
def initialize(project_path: str) -> None:
|
|
1209
|
-
"""Load everything once at startup. Untrusted project content is
|
|
1210
|
-
absent entirely -- not loaded-but-disabled (D17)."""
|
|
1211
|
-
global _state
|
|
1212
|
-
trusted = workspace_trust.is_trusted(project_path)
|
|
1213
|
-
if not trusted:
|
|
1214
|
-
logger.warning(
|
|
1215
|
-
"Project .venastine/ at %s is not trusted; project-level "
|
|
1216
|
-
"content (agents, skills, settings, AGENTS.md) will not load.",
|
|
1217
|
-
project_path,
|
|
1218
|
-
)
|
|
1219
|
-
settings, settings_tiers = _load_merged_settings(project_path, trusted)
|
|
1220
|
-
context = None
|
|
1221
|
-
if trusted:
|
|
1222
|
-
# §44: the project ROOT, not .venastine/. The hub is an ordinary
|
|
1223
|
-
# committed file now, so a project shared WITHOUT its configuration
|
|
1224
|
-
# is not a project with a dangling pointer -- which is the whole
|
|
1225
|
-
# reason it moved. workspace_trust still covers it: the file is in
|
|
1226
|
-
# content_files() and in the hash, because what reaches a system
|
|
1227
|
-
# prompt is what a grant has to be about.
|
|
1228
|
-
context_path = os.path.join(
|
|
1229
|
-
project_path, workspace_trust.PROJECT_CONTEXT_FILENAME)
|
|
1230
|
-
if os.path.exists(context_path):
|
|
1231
|
-
# Degrade, don't abort. UnicodeDecodeError is a ValueError, so
|
|
1232
|
-
# an unreadable AGENTS.md would reach main.load_project_config's
|
|
1233
|
-
# handler and SystemExit(1) EVERY invocation in this directory
|
|
1234
|
-
# -- including plain chat, which never reads the file. Every
|
|
1235
|
-
# other malformed content file in this module warns and skips.
|
|
1236
|
-
try:
|
|
1237
|
-
context = read_regular_text(context_path)
|
|
1238
|
-
except (OSError, UnicodeDecodeError) as e:
|
|
1239
|
-
logger.warning(
|
|
1240
|
-
"Could not read %s; continuing without project "
|
|
1241
|
-
"context: %s", context_path, e)
|
|
1242
|
-
_state = {
|
|
1243
|
-
"project_path": os.path.realpath(project_path),
|
|
1244
|
-
"trusted": trusted,
|
|
1245
|
-
"agents": _discover("agents", project_path, trusted),
|
|
1246
|
-
"skills": _discover("skills", project_path, trusted),
|
|
1247
|
-
"settings": settings,
|
|
1248
|
-
# Beside the values rather than folded into them, because twelve
|
|
1249
|
-
# call sites subscript that dict and `{key: (value, tier)}` breaks
|
|
1250
|
-
# every one (TECHNICAL_DEBT 12's first blocker, unchanged).
|
|
1251
|
-
"settings_sources": settings_tiers,
|
|
1252
|
-
"context": context,
|
|
1253
|
-
}
|
|
1254
|
-
# §32 A3. Before effective_compaction, because this one is about
|
|
1255
|
-
# THIS repository's own files rather than about the user's -- if
|
|
1256
|
-
# it fires, every other startup check is reporting on a build that
|
|
1257
|
-
# should not have shipped.
|
|
1258
|
-
assert_spawnable_declared(_state["agents"])
|
|
1259
|
-
# AFTER _state is assigned, because effective_compaction() reads
|
|
1260
|
-
# get_settings(). Called here so an incoherent compaction block is a
|
|
1261
|
-
# STARTUP error naming the file, rather than a ValueError from inside
|
|
1262
|
-
# a compaction three hours into a long thread -- which is the one
|
|
1263
|
-
# moment the user least wants to find out. §21's "reject at load time,
|
|
1264
|
-
# not incoherent trigger math later".
|
|
1265
|
-
effective_compaction(warn=True)
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
def assert_spawnable_declared(agents: dict) -> None:
|
|
1269
|
-
"""Every HARNESS-tier agent must say whether it is spawnable.
|
|
1270
|
-
|
|
1271
|
-
ROADMAP_v2 §32 (A3). The same shape as
|
|
1272
|
-
tools.base.assert_grant_policy_declared and
|
|
1273
|
-
assert_budget_declared, and for the same stated reason: a field
|
|
1274
|
-
whose absence silently means something is a field nobody will
|
|
1275
|
-
remember to set. #69 exists because there was no field at all, so
|
|
1276
|
-
every agent inherited "spawnable" and four shipped ones were
|
|
1277
|
-
advertised as a delegation route that cannot carry them.
|
|
1278
|
-
|
|
1279
|
-
HARNESS TIER ONLY, and that asymmetry is the decision rather than
|
|
1280
|
-
an omission. These are files this project ships, so a missing
|
|
1281
|
-
declaration is a build error and CI is where it should surface.
|
|
1282
|
-
A user's or a project's agent gets the safe default instead --
|
|
1283
|
-
raising there would take down startup over somebody else's file,
|
|
1284
|
-
which is the trade _parse_md_file already refuses everywhere else
|
|
1285
|
-
in this module.
|
|
1286
|
-
"""
|
|
1287
|
-
undeclared = sorted(
|
|
1288
|
-
f"{a.name} ({a.path})" for a in agents.values()
|
|
1289
|
-
if a.tier == "harness" and a.spawnable is None)
|
|
1290
|
-
if undeclared:
|
|
1291
|
-
raise RuntimeError(
|
|
1292
|
-
"These harness agents do not declare `spawnable` in their "
|
|
1293
|
-
"frontmatter, so nothing can tell whether spawn_subagent "
|
|
1294
|
-
"is able to feed them (ROADMAP_v2 §32 A3): "
|
|
1295
|
-
+ ", ".join(undeclared))
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
def reset() -> None:
|
|
1299
|
-
"""Clear the startup cache (tests)."""
|
|
1300
|
-
global _state
|
|
1301
|
-
_state = None
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
def get_agents() -> dict:
|
|
1305
|
-
"""Empty before initialize(), matching every other getter here.
|
|
1306
|
-
|
|
1307
|
-
Uninitialized means "no discovery has run", which is materially the
|
|
1308
|
-
same state as "discovery ran and found nothing" for every consumer --
|
|
1309
|
-
both mean there are no agents to offer. get_agents/get_skills used to
|
|
1310
|
-
raise while get_settings/get_skill/skill_catalog_text/context_for_agent
|
|
1311
|
-
returned empty, so the answer to "is calling this before startup an
|
|
1312
|
-
error?" depended on which getter you happened to pick. Only main.py's
|
|
1313
|
-
__main__ calls initialize(); §16's TUI, §18 and §19 add more entry
|
|
1314
|
-
points, so one convention now beats six discoveries later.
|
|
1315
|
-
"""
|
|
1316
|
-
if _state is None:
|
|
1317
|
-
return {}
|
|
1318
|
-
return dict(_state["agents"])
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
def get_skills() -> dict:
|
|
1322
|
-
"""Empty before initialize() -- see get_agents()."""
|
|
1323
|
-
if _state is None:
|
|
1324
|
-
return {}
|
|
1325
|
-
return dict(_state["skills"])
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
def get_settings() -> dict:
|
|
1329
|
-
if _state is None:
|
|
1330
|
-
return {} # pre-init consumers fall through to config.yaml defaults
|
|
1331
|
-
return dict(_state["settings"])
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
def settings_sources() -> dict:
|
|
1335
|
-
"""`{dotted setting: "user" | "project"}` for everything a settings
|
|
1336
|
-
file actually set. TECHNICAL_DEBT 12.
|
|
1337
|
-
|
|
1338
|
-
A setting absent here was not set by either file, which means the
|
|
1339
|
-
value in force came from `config.yaml` -- so the caller reads the
|
|
1340
|
-
absence rather than needing a third tier name in the mapping.
|
|
1341
|
-
|
|
1342
|
-
Empty pre-init for `get_settings()`'s reason, and the consequence is
|
|
1343
|
-
the same one: a caller that runs before `initialize()` is told nothing
|
|
1344
|
-
outranks the file, which is true of that moment.
|
|
1345
|
-
"""
|
|
1346
|
-
if _state is None:
|
|
1347
|
-
return {}
|
|
1348
|
-
return dict(_state["settings_sources"])
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
def spend_cap() -> Optional[int]:
|
|
1352
|
-
"""The configured per-run spend ceiling (#4), or None for uncapped.
|
|
1353
|
-
|
|
1354
|
-
The single resolution point every path reaches through the loop
|
|
1355
|
-
wrappers' `_SPEND_UNSET` sentinel: chat turns, each research pass,
|
|
1356
|
-
/init, the reviewer and its retries. One number applies per _run()
|
|
1357
|
-
invocation -- a user turn, a pass, an init run -- because that is what
|
|
1358
|
-
a spend ceiling means; nothing here reasons about context size, which
|
|
1359
|
-
is thresholds()/context_limit()'s instrument (compaction has NEVER
|
|
1360
|
-
keyed off this figure).
|
|
1361
|
-
|
|
1362
|
-
None -- the default, with or without the key present -- runs uncapped:
|
|
1363
|
-
#4's whole point is that the harness's cumulative counter is a billing
|
|
1364
|
-
meter, and hard spend limits belong to the provider's dashboard.
|
|
1365
|
-
"""
|
|
1366
|
-
return get_settings().get("max_token_budget")
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
def get_project_path() -> Optional[str]:
|
|
1370
|
-
"""The RESOLVED project path this session was initialized against.
|
|
1371
|
-
|
|
1372
|
-
ROADMAP_v2 §21b (M12/D25): a project-scoped memory is keyed to "the
|
|
1373
|
-
same resolved project path the workspace-trust store uses", so there
|
|
1374
|
-
has to be exactly one answer to "which project" across trust, config
|
|
1375
|
-
and memory. This is it -- the realpath already computed at
|
|
1376
|
-
initialize() time, not a fresh resolution that could differ.
|
|
1377
|
-
|
|
1378
|
-
None before initialize(), matching every other getter here. A caller
|
|
1379
|
-
that gets None has no project to scope to, which memories/manager.py
|
|
1380
|
-
treats as "global memories only" rather than as "show everything":
|
|
1381
|
-
guessing a project would surface another one's facts.
|
|
1382
|
-
"""
|
|
1383
|
-
if _state is None:
|
|
1384
|
-
return None
|
|
1385
|
-
return _state["project_path"]
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
def get_skill(name: str) -> Optional[SkillDef]:
|
|
1389
|
-
if _state is None:
|
|
1390
|
-
return None
|
|
1391
|
-
return _state["skills"].get(name)
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
def get_agent(name: str) -> Optional[AgentDef]:
|
|
1395
|
-
"""None before initialize() or for an unknown name -- see
|
|
1396
|
-
get_agents(). AgentManager (§18) is the consumer."""
|
|
1397
|
-
if _state is None:
|
|
1398
|
-
return None
|
|
1399
|
-
return _state["agents"].get(name)
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
def context_for_agent(agent: Optional[AgentDef]) -> Optional[str]:
|
|
1403
|
-
"""AGENTS.md is opt-in per agent (use_project_context), so no agent
|
|
1404
|
-
(or an agent without the flag) never pays its token cost. Untrusted
|
|
1405
|
-
projects have context=None regardless."""
|
|
1406
|
-
if _state is None or agent is None or not agent.use_project_context:
|
|
1407
|
-
return None
|
|
1408
|
-
return _state["context"]
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
def skill_catalog_text(active: Optional[list] = None) -> str:
|
|
1412
|
-
"""Frontmatter-only catalog for system prompt injection. Bodies are
|
|
1413
|
-
deliberately absent -- the model requests them via load_skill.
|
|
1414
|
-
Empty string when uninitialized or when no skills exist, so prompt
|
|
1415
|
-
assembly is a no-op append.
|
|
1416
|
-
|
|
1417
|
-
GROUPED BY CATEGORY (§19 K4). Uncategorised skills come first, then
|
|
1418
|
-
categories alphabetically, so a flat catalog reads exactly as it did
|
|
1419
|
-
before any category folder existed.
|
|
1420
|
-
|
|
1421
|
-
`active` names skills whose full body is already pinned into this
|
|
1422
|
-
prompt (§19 K1). They are MARKED rather than omitted: dropping them
|
|
1423
|
-
would make the catalog disagree with `/skill`'s listing, and leaving
|
|
1424
|
-
them unmarked invites the model to spend a turn calling load_skill for
|
|
1425
|
-
text it can already read.
|
|
1426
|
-
"""
|
|
1427
|
-
if _state is None or not _state["skills"]:
|
|
1428
|
-
return ""
|
|
1429
|
-
active_set = set(active or ())
|
|
1430
|
-
lines = [
|
|
1431
|
-
"## Available skills",
|
|
1432
|
-
"The skills below are available in this session. Only their "
|
|
1433
|
-
"summaries are listed here; call the load_skill tool with a "
|
|
1434
|
-
"skill name to view a skill's full instructions before "
|
|
1435
|
-
"following one.",
|
|
1436
|
-
]
|
|
1437
|
-
by_category: dict = {}
|
|
1438
|
-
for name in sorted(_state["skills"]):
|
|
1439
|
-
skill = _state["skills"][name]
|
|
1440
|
-
by_category.setdefault(skill.category, []).append(skill)
|
|
1441
|
-
# "" sorts before any real category name, which is the order wanted:
|
|
1442
|
-
# uncategorised first, then categories alphabetically.
|
|
1443
|
-
for category in sorted(by_category):
|
|
1444
|
-
if category:
|
|
1445
|
-
lines.append(f"### {category}")
|
|
1446
|
-
for skill in by_category[category]:
|
|
1447
|
-
suffix = (" [ACTIVE — full text below]"
|
|
1448
|
-
if skill.name in active_set else "")
|
|
1449
|
-
lines.append(f"- {skill.name}: {skill.description}{suffix}")
|
|
1450
|
-
return "\n".join(lines)
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
def _catalog_entries(root: str, files: list, shadows=None) -> list:
|
|
1454
|
-
"""(kind, name, description, reach) for every agent/skill in the
|
|
1455
|
-
listing; `reach` is the lines saying where that description goes.
|
|
1456
|
-
|
|
1457
|
-
ROADMAP_v2 §32 A6, #131. The trust prompt's stated criterion is
|
|
1458
|
-
written in describe_project_content below -- the files shown
|
|
1459
|
-
verbatim are "the ones whose contents change what runs". An
|
|
1460
|
-
agent's and a skill's `description` meets it exactly: one `y` lets
|
|
1461
|
-
it into system prompts in this project, with no tool call and no
|
|
1462
|
-
further consent, and the reason they were left out was that nobody
|
|
1463
|
-
had noticed they qualify.
|
|
1464
|
-
|
|
1465
|
-
WHERE EACH ONE GOES, SINCE BATCH 215 (#259, #260, register items
|
|
1466
|
-
112 and 113, the owner's wording). The section used
|
|
1467
|
-
to say every description entered EVERY system prompt. A skill's is
|
|
1468
|
-
listed wherever load_skill is offered; an agent's only if it is
|
|
1469
|
-
spawnable, and the project tier's are not unless they say so (§32
|
|
1470
|
-
A3/A4); and a definition whose name a harness or user one already
|
|
1471
|
-
has never loads at all, D29's precedence being first-wins. Each is
|
|
1472
|
-
still LISTED -- the grant covers its bytes, and renaming the other
|
|
1473
|
-
makes it live with no new prompt -- but no longer described as
|
|
1474
|
-
reaching further than it does. `shadows` is `{kind: {name: tier}}`
|
|
1475
|
-
for the tiers above the project.
|
|
1476
|
-
|
|
1477
|
-
PARSED THROUGH _parse_md_file, so what is shown is the NORMALISED
|
|
1478
|
-
text -- the same value A5 will put in the prompt, not the raw
|
|
1479
|
-
frontmatter. That is the whole point of showing it: a summary
|
|
1480
|
-
displaying something other than what gets injected is a worse
|
|
1481
|
-
answer than showing nothing, because it invites trust in the
|
|
1482
|
-
wrong string. It also means the listing is bounded by
|
|
1483
|
-
construction, which is what makes printing it safe at all.
|
|
1484
|
-
|
|
1485
|
-
Runs on UNTRUSTED content by definition, so a file that will not
|
|
1486
|
-
parse is reported as unreadable rather than skipped in silence:
|
|
1487
|
-
'this file is here and I cannot tell you what it says' is a fact
|
|
1488
|
-
the person answering the prompt needs.
|
|
1489
|
-
"""
|
|
1490
|
-
shadows = shadows or {}
|
|
1491
|
-
out = []
|
|
1492
|
-
for rel in files:
|
|
1493
|
-
# §44: the listing is PROJECT-relative now, so the .venastine/
|
|
1494
|
-
# prefix is stripped before matching. Matching the raw string
|
|
1495
|
-
# would silently describe nothing -- the trust prompt would still
|
|
1496
|
-
# render, still list the files, and simply stop saying what any of
|
|
1497
|
-
# them would tell the model, which is the half #131 added.
|
|
1498
|
-
if not rel.startswith(".venastine/"):
|
|
1499
|
-
continue
|
|
1500
|
-
parts = rel[len(".venastine/"):].split("/")
|
|
1501
|
-
if len(parts) < 2 or parts[0] not in ("agents", "skills"):
|
|
1502
|
-
continue
|
|
1503
|
-
if not rel.lower().endswith(".md"):
|
|
1504
|
-
continue
|
|
1505
|
-
kind = parts[0]
|
|
1506
|
-
defn = _parse_md_file(os.path.join(root, *parts), kind,
|
|
1507
|
-
"project")
|
|
1508
|
-
if defn is None:
|
|
1509
|
-
out.append((kind, "/".join(parts),
|
|
1510
|
-
"(could not be read -- see the log)", []))
|
|
1511
|
-
continue
|
|
1512
|
-
out.append((kind, defn.name, defn.description,
|
|
1513
|
-
_reach(kind, defn, shadows.get(kind, {}))))
|
|
1514
|
-
return out
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
def _reach(kind: str, defn, shadowed_by: dict) -> list:
|
|
1518
|
-
"""Where a project definition's description goes, as the lines the
|
|
1519
|
-
trust prompt prints under it (#259, #260)."""
|
|
1520
|
-
tier = shadowed_by.get(defn.name)
|
|
1521
|
-
if tier is not None:
|
|
1522
|
-
owner = ("the harness's own" if tier == "harness"
|
|
1523
|
-
else "your own (user configuration)")
|
|
1524
|
-
return [f'shadowed by {owner} "{defn.name}", which is what loads;',
|
|
1525
|
-
"this file is used only if that one goes away"]
|
|
1526
|
-
if kind == "skills":
|
|
1527
|
-
return ["listed wherever load_skill is offered"]
|
|
1528
|
-
if defn.spawnable:
|
|
1529
|
-
return ["spawnable: listed wherever spawn_subagent is offered"]
|
|
1530
|
-
return ["not spawnable: in no system prompt; /agent can still pick it"]
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
def _shadowing_names(project_path: str) -> dict:
|
|
1534
|
-
"""`{kind: {name: tier}}` for the tiers that win over the project's,
|
|
1535
|
-
harness first, in `_discover`'s own order and with its own parser, so
|
|
1536
|
-
the trust prompt and the loader cannot disagree about which loads."""
|
|
1537
|
-
out: dict = {}
|
|
1538
|
-
for kind in ("agents", "skills"):
|
|
1539
|
-
names: dict = {}
|
|
1540
|
-
for tier, directory in _tier_dirs(kind, project_path, False):
|
|
1541
|
-
if not os.path.isdir(directory):
|
|
1542
|
-
continue
|
|
1543
|
-
for path, category in _md_files(directory,
|
|
1544
|
-
recursive=(kind == "skills")):
|
|
1545
|
-
defn = _parse_md_file(path, kind, tier, category)
|
|
1546
|
-
if defn is not None:
|
|
1547
|
-
names.setdefault(defn.name, tier)
|
|
1548
|
-
out[kind] = names
|
|
1549
|
-
return out
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
#: How much of any ONE section of the trust prompt is printed. Enough to
|
|
1553
|
-
#: recognise what a file is for, short enough that the prompt stays
|
|
1554
|
-
#: readable -- the whole thing is what a grant covers, not what it prints.
|
|
1555
|
-
_SECTION_LINES = 60
|
|
1556
|
-
_SECTION_CHARS = 6000
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
def _bounded(source: list, prefix: str, what: str) -> list:
|
|
1560
|
-
"""`source`'s lines, prefixed and cut to BOTH bounds, with a marker
|
|
1561
|
-
naming what was withheld.
|
|
1562
|
-
|
|
1563
|
-
THE CONSENT PROMPT IS THE ONE PLACE A PROJECT'S CONTENT IS SHOWN TO
|
|
1564
|
-
SOMEONE WHO HAS NOT TRUSTED IT YET (D17), so how long it is must not
|
|
1565
|
-
be the project's choice. Every section goes through here (A54) --
|
|
1566
|
-
the file listing, the hub, the two JSON files and the catalog --
|
|
1567
|
-
because capping some of them leaves the prompt exactly as floodable
|
|
1568
|
-
and the cap stops being worth having.
|
|
1569
|
-
|
|
1570
|
-
TWO BOUNDS, NOT ONE, AND THE PAIR IS THE POINT. §44 gave the hub a
|
|
1571
|
-
LINE bound alone, and measured against it: a 2 MB AGENTS.md written
|
|
1572
|
-
as one line printed all 2 MB in four lines, and twenty 100 KB lines
|
|
1573
|
-
printed 2 MB from inside the cap. A char bound alone fails the other
|
|
1574
|
-
way -- 30,000 short lines are 2.4 MB of prompt. The two JSON files
|
|
1575
|
-
had neither bound: 2 x 2.2 MB rendered as 4.5 MB in 30 lines, with
|
|
1576
|
-
the `mcp.json` command the section exists to show sitting at
|
|
1577
|
-
99.9994% of the way in.
|
|
1578
|
-
|
|
1579
|
-
THE NUMBERS ARE MEASURED AGAINST THE LARGEST LEGITIMATE SHAPES
|
|
1580
|
-
rather than picked. A five-server pretty-printed `mcp.json` is 54
|
|
1581
|
-
lines and 923 characters; a realistic `settings.json` is 14 and 264;
|
|
1582
|
-
this project's own AGENTS.md is 4,033 characters in its first 60
|
|
1583
|
-
lines. All three render whole, which is what keeps the marker
|
|
1584
|
-
meaningful on the files where it does appear.
|
|
1585
|
-
|
|
1586
|
-
THE MARKER IS NOT A COURTESY. A cut nobody is told about turns a
|
|
1587
|
-
bounded prompt into one that misrepresents what it covers, and the
|
|
1588
|
-
grant is over the whole file either way -- which is why the sentence
|
|
1589
|
-
says so rather than stopping at an ellipsis.
|
|
1590
|
-
"""
|
|
1591
|
-
shown: list = []
|
|
1592
|
-
used = 0
|
|
1593
|
-
hidden_chars = 0
|
|
1594
|
-
for line in source:
|
|
1595
|
-
if len(shown) >= _SECTION_LINES or used >= _SECTION_CHARS:
|
|
1596
|
-
hidden_chars += len(line)
|
|
1597
|
-
continue
|
|
1598
|
-
room = _SECTION_CHARS - used
|
|
1599
|
-
if len(line) > room:
|
|
1600
|
-
# CUT WITHIN THE LINE rather than dropping it whole. A file
|
|
1601
|
-
# flooded on one enormous line has exactly one line, and it
|
|
1602
|
-
# is the line the section exists to show.
|
|
1603
|
-
shown.append(line[:room] + "…")
|
|
1604
|
-
used += room
|
|
1605
|
-
hidden_chars += len(line) - room
|
|
1606
|
-
continue
|
|
1607
|
-
shown.append(line)
|
|
1608
|
-
used += len(line)
|
|
1609
|
-
out = [prefix + line for line in shown]
|
|
1610
|
-
hidden_lines = len(source) - len(shown)
|
|
1611
|
-
if hidden_lines or hidden_chars:
|
|
1612
|
-
# Always plural. The grammar is a blemish; a marker whose wording
|
|
1613
|
-
# depends on the count is one a test can match by luck.
|
|
1614
|
-
out.append(
|
|
1615
|
-
f"{prefix}… {hidden_lines} more lines and {hidden_chars} more "
|
|
1616
|
-
f"characters of {what} are not shown here. Trust covers the "
|
|
1617
|
-
f"whole file, not the part printed.")
|
|
1618
|
-
return out
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
def describe_project_content(project_path: str) -> str:
|
|
1622
|
-
"""Human-readable summary shown in the trust prompt, so approving
|
|
1623
|
-
trust is an informed decision -- a project's settings can choose the
|
|
1624
|
-
provider and multiply pipeline cost by turning on ensemble_mode over
|
|
1625
|
-
config.yaml's roster (#213: ensemble_n is ignored), and an agent's
|
|
1626
|
-
or a skill's description can reach the system prompt (#131) --
|
|
1627
|
-
where, each entry says (#259, #260).
|
|
1628
|
-
|
|
1629
|
-
EVERY SECTION IS BOUNDED, through `_bounded` above (A54, batch 155).
|
|
1630
|
-
Two of the four used to be pasted whole and the hub was bounded on
|
|
1631
|
-
one axis of two, which let a cloned project decide how long its own
|
|
1632
|
-
consent screen was."""
|
|
1633
|
-
root = workspace_trust.venastine_dir(project_path)
|
|
1634
|
-
files = workspace_trust.content_files(project_path)
|
|
1635
|
-
lines = [f"Project content ({os.path.realpath(project_path)}):"]
|
|
1636
|
-
# THE LISTING ITSELF IS BOUNDED TOO. `content_files()` is unchanged
|
|
1637
|
-
# and still returns every covered path -- the hash and
|
|
1638
|
-
# workspace_trust's "the loader must read no file the trust listing
|
|
1639
|
-
# omits" invariant are about that function, not about this rendering.
|
|
1640
|
-
# What is bounded is what the PROMPT prints, and 20,000 padding files
|
|
1641
|
-
# under .venastine/ were 20,001 lines of it.
|
|
1642
|
-
lines += _bounded(files, " - ", "the file listing")
|
|
1643
|
-
|
|
1644
|
-
# §44. The hub is shown VERBATIM, and it is the strongest instance of
|
|
1645
|
-
# this function's own stated criterion -- "the ones whose contents
|
|
1646
|
-
# change what runs". settings.json picks a provider and mcp.json names
|
|
1647
|
-
# a command; AGENTS.md is prose that goes into the system prompt of
|
|
1648
|
-
# every opted-in agent, which is the plainest form of "tells the model
|
|
1649
|
-
# what to do" this prompt can be asked to cover. Bounded to the first
|
|
1650
|
-
# few lines because a project's context document is not small and an
|
|
1651
|
-
# unbounded paste is a prompt nobody reads to the end of.
|
|
1652
|
-
hub = os.path.join(os.path.realpath(project_path),
|
|
1653
|
-
workspace_trust.PROJECT_CONTEXT_FILENAME)
|
|
1654
|
-
if os.path.exists(hub):
|
|
1655
|
-
# Every read in this function is of content nobody has consented
|
|
1656
|
-
# to yet, so each is regular-files-only (batch 219, #214): a hub
|
|
1657
|
-
# linked to /dev/zero would otherwise be read until memory ran out.
|
|
1658
|
-
try:
|
|
1659
|
-
body = read_regular_text(hub)
|
|
1660
|
-
except (OSError, UnicodeDecodeError) as e:
|
|
1661
|
-
lines.append(
|
|
1662
|
-
f"{workspace_trust.PROJECT_CONTEXT_FILENAME}: could not be "
|
|
1663
|
-
f"read ({e})")
|
|
1664
|
-
else:
|
|
1665
|
-
lines.append(
|
|
1666
|
-
f"{workspace_trust.PROJECT_CONTEXT_FILENAME} would enter "
|
|
1667
|
-
f"every opted-in agent's system prompt; it begins:")
|
|
1668
|
-
lines += _bounded(
|
|
1669
|
-
body.splitlines(), " | ",
|
|
1670
|
-
workspace_trust.PROJECT_CONTEXT_FILENAME)
|
|
1671
|
-
|
|
1672
|
-
# settings.json and mcp.json are shown VERBATIM, the rest by name.
|
|
1673
|
-
# These two are the ones whose contents change what runs: settings can
|
|
1674
|
-
# pick the provider and multiply pipeline cost by turning on
|
|
1675
|
-
# ensemble_mode (ensemble_n is ignored; the roster is config.yaml's), and
|
|
1676
|
-
# mcp.json names a local command to execute. Approving trust without
|
|
1677
|
-
# seeing that command is not an informed decision, which is the whole
|
|
1678
|
-
# reason §17 moved project MCP config under .venastine/ instead of
|
|
1679
|
-
# leaving it at the project root outside the trust boundary.
|
|
1680
|
-
for fname, label in (("settings.json", "settings.json contents:"),
|
|
1681
|
-
("mcp.json", "mcp.json contents (these commands would run):")):
|
|
1682
|
-
path = os.path.join(root, fname)
|
|
1683
|
-
if not os.path.exists(path):
|
|
1684
|
-
continue
|
|
1685
|
-
try:
|
|
1686
|
-
body = read_regular_text(path)
|
|
1687
|
-
except (OSError, UnicodeDecodeError) as e:
|
|
1688
|
-
# This function runs ONLY for untrusted projects, i.e. content
|
|
1689
|
-
# that is adversarial by definition. UnicodeDecodeError is a
|
|
1690
|
-
# ValueError, not an OSError, so without it here a UTF-16
|
|
1691
|
-
# settings.json kills startup before the trust prompt renders
|
|
1692
|
-
# -- including the non-TTY notice and the --trust-project path,
|
|
1693
|
-
# making trust ungrantable for that project.
|
|
1694
|
-
lines.append(f"{fname}: could not be read ({e})")
|
|
1695
|
-
continue
|
|
1696
|
-
lines.append(label)
|
|
1697
|
-
lines += _bounded(body.splitlines(), " | ", fname)
|
|
1698
|
-
|
|
1699
|
-
# §32 A6. Shown AFTER the two config files rather than beside the
|
|
1700
|
-
# file list, because the list answers "what is here" and this
|
|
1701
|
-
# answers "what would it say to the model" -- the same order the
|
|
1702
|
-
# two questions are asked in.
|
|
1703
|
-
entries = _catalog_entries(root, files,
|
|
1704
|
-
_shadowing_names(project_path))
|
|
1705
|
-
if entries:
|
|
1706
|
-
lines.append("what each project agent and skill would tell the "
|
|
1707
|
-
"model:")
|
|
1708
|
-
# A5 bounds each description at `max_catalog_text_chars` and
|
|
1709
|
-
# nothing bounded the NUMBER of them: 3,000 project agents were
|
|
1710
|
-
# 1 MB of prompt in 6,002 lines, every entry individually legal.
|
|
1711
|
-
rows = []
|
|
1712
|
-
for kind, name, description, reach in entries:
|
|
1713
|
-
rows.append(f"{kind[:-1]} {name}: {description}")
|
|
1714
|
-
rows += [f" {line}" for line in reach]
|
|
1715
|
-
lines += _bounded(rows, " | ", "the agent and skill descriptions")
|
|
1716
|
-
return "\n".join(lines)
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
# ---------------------------------------------------------------------------
|
|
1720
|
-
# ---- §45 (SQ6): the scoring knobs -----------------------------------------
|
|
1721
|
-
# ---------------------------------------------------------------------------
|
|
1722
|
-
#
|
|
1723
|
-
# WHY THESE TWO SECTIONS RESOLVE DIFFERENTLY FROM COMPACTION, and it is a
|
|
1724
|
-
# deliberate amendment rather than an oversight. `effective_compaction`
|
|
1725
|
-
# RAISES on an incoherent value, because a bad trigger burns real model
|
|
1726
|
-
# calls on every turn of every conversation. A bad scoring weight costs
|
|
1727
|
-
# the shape of one number in one report -- and refusing to start a
|
|
1728
|
-
# ten-pass run over a mistyped weight is the worse failure. So a known key
|
|
1729
|
-
# with an out-of-range value FALLS BACK to its own default and warns; only
|
|
1730
|
-
# that one value is discarded, never the whole section.
|
|
1731
|
-
#
|
|
1732
|
-
# Unknown KEYS still raise, unchanged. That is a schema question rather
|
|
1733
|
-
# than a range one, and §14's loud-rejection rule is what makes adding a
|
|
1734
|
-
# setting a deliberate act.
|
|
1735
|
-
#
|
|
1736
|
-
# TOP-LEVEL SECTIONS, not `research.confidence`. `_NESTED_SETTINGS`
|
|
1737
|
-
# supports exactly one level of nesting and both the validator and the
|
|
1738
|
-
# cross-tier merge iterate it; deepening that merge is the code path
|
|
1739
|
-
# review finding F2 already broke once, and these need no new mechanism to
|
|
1740
|
-
# sit beside `compaction` and `tui`.
|
|
1741
|
-
|
|
1742
|
-
# key -> (module attribute it defaults from, validator)
|
|
1743
|
-
#
|
|
1744
|
-
# The defaults live BESIDE THE FORMULA rather than in config.yaml: the
|
|
1745
|
-
# tunable-weights block at the top of confidence_scoring.py documents what
|
|
1746
|
-
# each number does and why it is that number, and a second copy here is a
|
|
1747
|
-
# second thing to keep in step. Imported lazily inside the resolver, the
|
|
1748
|
-
# same shape effective_compaction uses for `config_schema`.
|
|
1749
|
-
_UNIT = ("a number between 0 and 1", lambda v: isinstance(v, NUMBER_TYPES)
|
|
1750
|
-
and not isinstance(v, bool) and 0.0 <= v <= 1.0)
|
|
1751
|
-
_POSITIVE_INT = ("a positive whole number",
|
|
1752
|
-
lambda v: isinstance(v, int) and not isinstance(v, bool) and v > 0)
|
|
1753
|
-
_NON_NEGATIVE_INT = ("a whole number of 0 or more",
|
|
1754
|
-
lambda v: isinstance(v, int) and not isinstance(v, bool) and v >= 0)
|
|
1755
|
-
|
|
1756
|
-
_CONFIDENCE_KNOBS = {
|
|
1757
|
-
"grounding_weight_factor": _UNIT,
|
|
1758
|
-
"critic_weight_factor": _UNIT,
|
|
1759
|
-
"assumption_flag_penalty": _UNIT,
|
|
1760
|
-
"non_factual_score_cap": _UNIT,
|
|
1761
|
-
"source_quality_floor": _UNIT,
|
|
1762
|
-
"authority_full_credit": _UNIT,
|
|
1763
|
-
}
|
|
1764
|
-
|
|
1765
|
-
_SOURCE_SCORING_KNOBS = {
|
|
1766
|
-
"similarity_floor": _UNIT,
|
|
1767
|
-
"similarity_ceiling": _UNIT,
|
|
1768
|
-
"authority_adjustment_cap": _UNIT,
|
|
1769
|
-
"window_chars": _POSITIVE_INT,
|
|
1770
|
-
"max_windows": _POSITIVE_INT,
|
|
1771
|
-
"claim_min_chars": _NON_NEGATIVE_INT,
|
|
1772
|
-
"top_k": _POSITIVE_INT,
|
|
1773
|
-
"venue_weight": _UNIT,
|
|
1774
|
-
"citation_weight": _UNIT,
|
|
1775
|
-
"author_weight": _UNIT,
|
|
1776
|
-
"h_saturation": _POSITIVE_INT,
|
|
1777
|
-
"min_cohort_size": _POSITIVE_INT,
|
|
1778
|
-
"min_citation_age_days": _NON_NEGATIVE_INT,
|
|
1779
|
-
}
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
def _confidence_defaults() -> dict:
|
|
1783
|
-
from core.reasoning import confidence_scoring as cs
|
|
1784
|
-
|
|
1785
|
-
return {
|
|
1786
|
-
"grounding_weight_factor": cs.GROUNDING_WEIGHT_FACTOR,
|
|
1787
|
-
"critic_weight_factor": cs.CRITIC_WEIGHT_FACTOR,
|
|
1788
|
-
"assumption_flag_penalty": cs.ASSUMPTION_FLAG_PENALTY,
|
|
1789
|
-
"non_factual_score_cap": cs.NON_FACTUAL_SCORE_CAP,
|
|
1790
|
-
"source_quality_floor": cs.SOURCE_QUALITY_FLOOR,
|
|
1791
|
-
"authority_full_credit": cs.AUTHORITY_FULL_CREDIT,
|
|
1792
|
-
"grounding_weights": dict(cs.GROUNDING_WEIGHTS),
|
|
1793
|
-
"tier_thresholds": {name: value for value, name in cs.TIER_THRESHOLDS},
|
|
1794
|
-
}
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
def _source_scoring_defaults() -> dict:
|
|
1798
|
-
from core.reasoning import source_scoring as ss
|
|
1799
|
-
|
|
1800
|
-
cfg = config_schema.current()
|
|
1801
|
-
return {
|
|
1802
|
-
"similarity_floor": None, # None means "use the per-model table"
|
|
1803
|
-
"similarity_ceiling": None,
|
|
1804
|
-
"authority_adjustment_cap": cfg.authority_adjustment_cap,
|
|
1805
|
-
"window_chars": ss.WINDOW_CHARS,
|
|
1806
|
-
"max_windows": ss.MAX_WINDOWS,
|
|
1807
|
-
"claim_min_chars": ss.CLAIM_MIN_CHARS,
|
|
1808
|
-
"top_k": ss.TOP_K,
|
|
1809
|
-
"venue_weight": cfg.scholar_venue_weight,
|
|
1810
|
-
"citation_weight": cfg.scholar_citation_weight,
|
|
1811
|
-
"author_weight": cfg.scholar_author_weight,
|
|
1812
|
-
"h_saturation": cfg.scholar_h_saturation,
|
|
1813
|
-
"min_cohort_size": cfg.scholar_min_cohort_size,
|
|
1814
|
-
"min_citation_age_days": cfg.scholar_min_citation_age_days,
|
|
1815
|
-
"domain_overrides": {},
|
|
1816
|
-
}
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
def _apply_scalar_knobs(values, supplied, knobs, section, warn):
|
|
1820
|
-
"""Overlay `supplied` onto `values`, dropping what does not validate.
|
|
1821
|
-
|
|
1822
|
-
PER KEY, not per section. One mistyped weight must not discard the
|
|
1823
|
-
nine beside it that were right -- a user retuning six numbers and
|
|
1824
|
-
fat-fingering the seventh should get six retuned numbers and one
|
|
1825
|
-
sentence about the seventh, not silently get none of them.
|
|
1826
|
-
"""
|
|
1827
|
-
for key, value in supplied.items():
|
|
1828
|
-
if key not in knobs:
|
|
1829
|
-
continue
|
|
1830
|
-
description, is_valid = knobs[key]
|
|
1831
|
-
if is_valid(value):
|
|
1832
|
-
values[key] = value
|
|
1833
|
-
elif warn:
|
|
1834
|
-
logger.warning(
|
|
1835
|
-
"%s.%s must be %s, got %r -- using the default %r for this "
|
|
1836
|
-
"run.", section, key, description, value, values[key])
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
def _apply_grounding_weights(values, supplied, warn):
|
|
1840
|
-
"""`grounding_weights` -- a dict whose KEYS are a closed vocabulary."""
|
|
1841
|
-
if not isinstance(supplied, dict):
|
|
1842
|
-
if warn and supplied is not None:
|
|
1843
|
-
logger.warning("confidence.grounding_weights must be an object, "
|
|
1844
|
-
"got %r -- using the defaults for this run.",
|
|
1845
|
-
type(supplied).__name__)
|
|
1846
|
-
return
|
|
1847
|
-
merged = dict(values["grounding_weights"])
|
|
1848
|
-
for status, weight in supplied.items():
|
|
1849
|
-
if status not in GROUNDING_STATUSES:
|
|
1850
|
-
if warn:
|
|
1851
|
-
logger.warning(
|
|
1852
|
-
"confidence.grounding_weights has an unknown status %r "
|
|
1853
|
-
"-- expected one of %s. Ignored for this run.",
|
|
1854
|
-
status, ", ".join(sorted(GROUNDING_STATUSES)))
|
|
1855
|
-
continue
|
|
1856
|
-
if isinstance(weight, NUMBER_TYPES) and not isinstance(weight, bool) \
|
|
1857
|
-
and 0.0 <= weight <= 1.0:
|
|
1858
|
-
merged[status] = weight
|
|
1859
|
-
elif warn:
|
|
1860
|
-
logger.warning(
|
|
1861
|
-
"confidence.grounding_weights[%r] must be a number between 0 "
|
|
1862
|
-
"and 1, got %r -- using the default for this run.",
|
|
1863
|
-
status, weight)
|
|
1864
|
-
values["grounding_weights"] = merged
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
def _apply_tier_thresholds(values, supplied, warn):
|
|
1868
|
-
"""`tier_thresholds` -- three numbers that must STAY ORDERED.
|
|
1869
|
-
|
|
1870
|
-
The relational check `effective_compaction` makes for warning_margin,
|
|
1871
|
-
applied to the one relationship here whose breakage is not
|
|
1872
|
-
self-correcting: thresholds out of order make a tier unreachable, so a
|
|
1873
|
-
claim scoring 0.9 could be published LOW while the table a reader
|
|
1874
|
-
consults says HIGH starts at 0.8. Out of order, the whole table falls
|
|
1875
|
-
back rather than a single value -- unlike the scalars above, because
|
|
1876
|
-
these three are only meaningful together.
|
|
1877
|
-
"""
|
|
1878
|
-
if not isinstance(supplied, dict):
|
|
1879
|
-
if warn and supplied is not None:
|
|
1880
|
-
logger.warning("confidence.tier_thresholds must be an object, "
|
|
1881
|
-
"got %r -- using the defaults for this run.",
|
|
1882
|
-
type(supplied).__name__)
|
|
1883
|
-
return
|
|
1884
|
-
merged = dict(values["tier_thresholds"])
|
|
1885
|
-
for name, value in supplied.items():
|
|
1886
|
-
if name not in merged:
|
|
1887
|
-
if warn:
|
|
1888
|
-
logger.warning(
|
|
1889
|
-
"confidence.tier_thresholds has an unknown tier %r -- "
|
|
1890
|
-
"expected one of %s. Ignored for this run.",
|
|
1891
|
-
name, ", ".join(merged))
|
|
1892
|
-
continue
|
|
1893
|
-
if isinstance(value, NUMBER_TYPES) and not isinstance(value, bool) \
|
|
1894
|
-
and 0.0 <= value <= 1.0:
|
|
1895
|
-
merged[name] = value
|
|
1896
|
-
elif warn:
|
|
1897
|
-
logger.warning(
|
|
1898
|
-
"confidence.tier_thresholds[%r] must be a number between 0 "
|
|
1899
|
-
"and 1, got %r -- using the defaults for this run.",
|
|
1900
|
-
name, value)
|
|
1901
|
-
|
|
1902
|
-
ordered = [merged.get(name) for name in ("HIGH", "MEDIUM", "LOW")]
|
|
1903
|
-
if not (ordered[0] > ordered[1] > ordered[2]):
|
|
1904
|
-
if warn:
|
|
1905
|
-
logger.warning(
|
|
1906
|
-
"confidence.tier_thresholds must descend HIGH > MEDIUM > LOW, "
|
|
1907
|
-
"got %s -- using the defaults for this run. Out of order, a "
|
|
1908
|
-
"tier becomes unreachable and a published tier stops matching "
|
|
1909
|
-
"the table that defines it.", merged)
|
|
1910
|
-
return
|
|
1911
|
-
values["tier_thresholds"] = merged
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
def effective_confidence(warn: bool = False) -> dict:
|
|
1915
|
-
"""Pass 5's constants, as they are actually in force.
|
|
1916
|
-
|
|
1917
|
-
config default (from confidence_scoring's own tunable block) -> user
|
|
1918
|
-
settings.json -> trusted project settings.json. Nearest wins, the
|
|
1919
|
-
precedence the rest of the config system uses.
|
|
1920
|
-
|
|
1921
|
-
RESOLVED ONCE PER RUN by the caller, never per claim: score_claim is a
|
|
1922
|
-
pure function and stays one, the warnings are said once rather than
|
|
1923
|
-
once per claim (`effective_compaction`'s `warn=True` gating, for its
|
|
1924
|
-
reason), and a run's numbers cannot change halfway through.
|
|
1925
|
-
"""
|
|
1926
|
-
values = _confidence_defaults()
|
|
1927
|
-
supplied = get_settings().get("confidence") or {}
|
|
1928
|
-
if warn and "disagreement_penalty_factor" in supplied:
|
|
1929
|
-
logger.warning(
|
|
1930
|
-
"Ignoring confidence.disagreement_penalty_factor: a claim most "
|
|
1931
|
-
"ensemble candidates did not assert is sent to Pass 6a/6b to be "
|
|
1932
|
-
"checked now, and nothing is subtracted for it (ROADMAP_v3 §90). "
|
|
1933
|
-
"Remove the key from settings.json.")
|
|
1934
|
-
_apply_scalar_knobs(values, supplied, _CONFIDENCE_KNOBS, "confidence", warn)
|
|
1935
|
-
_apply_grounding_weights(values, supplied.get("grounding_weights"), warn)
|
|
1936
|
-
_apply_tier_thresholds(values, supplied.get("tier_thresholds"), warn)
|
|
1937
|
-
return values
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
def effective_source_scoring(warn: bool = False) -> dict:
|
|
1941
|
-
"""§45's scoring knobs, as they are actually in force.
|
|
1942
|
-
|
|
1943
|
-
`similarity_floor` / `similarity_ceiling` default to None, meaning
|
|
1944
|
-
"use the per-model calibration table". A user who sets them is saying
|
|
1945
|
-
their embedder's band is not what the table thinks, which outranks a
|
|
1946
|
-
table that cannot know about a model shipped after this release --
|
|
1947
|
-
`model_windows`' inversion, for its reason.
|
|
1948
|
-
"""
|
|
1949
|
-
values = _source_scoring_defaults()
|
|
1950
|
-
supplied = get_settings().get("source_scoring") or {}
|
|
1951
|
-
_apply_scalar_knobs(values, supplied, _SOURCE_SCORING_KNOBS,
|
|
1952
|
-
"source_scoring", warn)
|
|
1953
|
-
|
|
1954
|
-
overrides = supplied.get("domain_overrides")
|
|
1955
|
-
if isinstance(overrides, dict):
|
|
1956
|
-
kept = {}
|
|
1957
|
-
for host, score in overrides.items():
|
|
1958
|
-
if isinstance(host, str) and host and \
|
|
1959
|
-
isinstance(score, NUMBER_TYPES) and \
|
|
1960
|
-
not isinstance(score, bool) and 0.0 <= score <= 1.0:
|
|
1961
|
-
kept[host.lower().lstrip(".")] = float(score)
|
|
1962
|
-
elif warn:
|
|
1963
|
-
logger.warning(
|
|
1964
|
-
"source_scoring.domain_overrides[%r] must be a number "
|
|
1965
|
-
"between 0 and 1, got %r -- ignored for this run.",
|
|
1966
|
-
host, score)
|
|
1967
|
-
values["domain_overrides"] = kept
|
|
1968
|
-
elif overrides is not None and warn:
|
|
1969
|
-
logger.warning("source_scoring.domain_overrides must be an object, "
|
|
1970
|
-
"got %r -- ignored for this run.",
|
|
1971
|
-
type(overrides).__name__)
|
|
1972
|
-
|
|
1973
|
-
floor = values["similarity_floor"]
|
|
1974
|
-
ceiling = values["similarity_ceiling"]
|
|
1975
|
-
if floor is not None and ceiling is not None and floor >= ceiling:
|
|
1976
|
-
if warn:
|
|
1977
|
-
logger.warning(
|
|
1978
|
-
"source_scoring.similarity_floor (%s) must be below "
|
|
1979
|
-
"similarity_ceiling (%s) -- falling back to this embedder's "
|
|
1980
|
-
"own calibration for this run.", floor, ceiling)
|
|
1981
|
-
values["similarity_floor"] = values["similarity_ceiling"] = None
|
|
1982
|
-
return values
|
|
1
|
+
"""
|
|
2
|
+
core/config_loader.py
|
|
3
|
+
|
|
4
|
+
ROADMAP_v2 §14. Discovers and parses agent/skill `.md` files across the
|
|
5
|
+
three tiers (D8) -- harness (`agents/builtin/`, `skills/builtin/` shipped
|
|
6
|
+
with the repo), project (`.venastine/`, only if trusted per D17), user
|
|
7
|
+
(`~/.config/venastine/`) -- merges `settings.json` across tiers, and
|
|
8
|
+
exposes the skill catalog (frontmatter only, never bodies) for system
|
|
9
|
+
prompt injection.
|
|
10
|
+
|
|
11
|
+
Progressive disclosure: the model sees name+description of every
|
|
12
|
+
discovered skill in its system prompt and must call the `load_skill`
|
|
13
|
+
tool to view a body. Bodies never enter context unrequested.
|
|
14
|
+
|
|
15
|
+
Tier precedence: a harness-tier name can never be shadowed -- a
|
|
16
|
+
project/user file colliding with a builtin is rejected with a warning
|
|
17
|
+
(D18). Project vs user collisions resolve USER-wins (D29): the user tier is
|
|
18
|
+
what you authored; the project tier is what arrived with a directory
|
|
19
|
+
you cloned.
|
|
20
|
+
|
|
21
|
+
`initialize(project_path)` is called once at startup (main.py) and caches
|
|
22
|
+
everything; getters return the cached state. Tests call initialize() with
|
|
23
|
+
their own tmp dirs and reset() afterwards.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
import json
|
|
27
|
+
import logging
|
|
28
|
+
import os
|
|
29
|
+
import re
|
|
30
|
+
from dataclasses import dataclass
|
|
31
|
+
from typing import Optional
|
|
32
|
+
|
|
33
|
+
import yaml
|
|
34
|
+
|
|
35
|
+
import config_schema
|
|
36
|
+
from core import workspace_trust
|
|
37
|
+
from json_store import read_regular_text
|
|
38
|
+
|
|
39
|
+
logger = logging.getLogger(__name__)
|
|
40
|
+
|
|
41
|
+
# The repo root (this file lives in <root>/core/). The harness tier is
|
|
42
|
+
# shipped with the harness itself, so it resolves from here, not from cwd.
|
|
43
|
+
HARNESS_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
44
|
+
|
|
45
|
+
NUMBER_TYPES = (int, float)
|
|
46
|
+
|
|
47
|
+
# §45 (SQ6): the closed vocabulary `confidence.grounding_weights`
|
|
48
|
+
# keys against. Re-exported from base.py rather than re-typed, the
|
|
49
|
+
# way payload_validation does -- a second hand-written copy of a
|
|
50
|
+
# vocabulary is a second thing to keep in step.
|
|
51
|
+
from core.reasoning.base import GROUNDING_STATUSES # noqa: E402
|
|
52
|
+
|
|
53
|
+
_FRONTMATTER_DELIM = re.compile(r"^---\s*$", re.MULTILINE)
|
|
54
|
+
|
|
55
|
+
_KNOWN_SETTINGS = {
|
|
56
|
+
"default_provider": str,
|
|
57
|
+
"default_model": str,
|
|
58
|
+
# Batch 25 (#139). The effort level every path runs at unless something
|
|
59
|
+
# more specific speaks: CLI flag > this key > config.DEFAULT_EFFORT,
|
|
60
|
+
# with tui.effort still winning INSIDE the TUI (it predates this key
|
|
61
|
+
# and changing its meaning would surprise existing configs).
|
|
62
|
+
#
|
|
63
|
+
# Deliberately NOT an R12 by-name rejection, and the reason is worth
|
|
64
|
+
# stating so nobody "completes" the pattern: R12 rejects keys that are
|
|
65
|
+
# AUTHORITY. This one is cost -- the worst a cloned repo's settings.json
|
|
66
|
+
# can do here is make runs think harder and spend more tokens, which is
|
|
67
|
+
# compaction-settings territory, not granted-tools territory.
|
|
68
|
+
"effort": str,
|
|
69
|
+
# Batch 27 (#4). The optional per-run SPEND ceiling, in billed
|
|
70
|
+
# input+output tokens. None/absent -- the default -- means UNCAPPED:
|
|
71
|
+
# hard spend limits are the provider's dashboard's job, and a harness
|
|
72
|
+
# cap that misreads as a context limit is the exact defect #4 records.
|
|
73
|
+
# Cost knob, not authority, so effort's merge rule applies (normal
|
|
74
|
+
# tiers; deliberately NOT an R12 by-name rejection).
|
|
75
|
+
"max_token_budget": (int, type(None)),
|
|
76
|
+
"ensemble_mode": bool,
|
|
77
|
+
"ensemble_n": int,
|
|
78
|
+
"compaction": dict,
|
|
79
|
+
"tui": dict,
|
|
80
|
+
"research": dict,
|
|
81
|
+
# ROADMAP_v2 §45 (SQ6). TOP-LEVEL, not nested under `research`:
|
|
82
|
+
# _NESTED_SETTINGS supports exactly one level and both the validator
|
|
83
|
+
# and the cross-tier merge iterate it, so `research.confidence` would
|
|
84
|
+
# need the merge deepened -- the code path review finding F2 broke
|
|
85
|
+
# once. These are cost/quality knobs, not authority, so effort's merge
|
|
86
|
+
# rule applies: normal tiers, deliberately NOT an R12 by-name
|
|
87
|
+
# rejection. The two keys that ARE authority -- critic_model and
|
|
88
|
+
# embedder_model -- are rejected above.
|
|
89
|
+
"confidence": dict,
|
|
90
|
+
"source_scoring": dict,
|
|
91
|
+
}
|
|
92
|
+
# ROADMAP_v2 §25 (R12). The authorization MODE is persistable and the
|
|
93
|
+
# grant LIST deliberately is not.
|
|
94
|
+
#
|
|
95
|
+
# The asymmetry is the design. A persisted mode can only ever ADD prompts,
|
|
96
|
+
# so the worst a hostile settings.json can do is make your runs more
|
|
97
|
+
# annoying. A persisted grant list could only ever REMOVE them -- standing
|
|
98
|
+
# authorization for named tools, carried by any repo you clone, and
|
|
99
|
+
# settings.json is the one config file where project tier beats user tier.
|
|
100
|
+
#
|
|
101
|
+
# `granted_tools` is therefore rejected BY NAME rather than left to the
|
|
102
|
+
# unknown-key branch, because "unknown key" reads as an oversight to be
|
|
103
|
+
# fixed by adding support for it. See _validate_settings.
|
|
104
|
+
_KNOWN_RESEARCH = {
|
|
105
|
+
"approval_mode": str, # "none" (default) or "attended"
|
|
106
|
+
# §20/D9. A MODE, so R12's rule permits persisting it: the worst a
|
|
107
|
+
# hostile settings.json can do here is make a run stop and ask you
|
|
108
|
+
# about corrections it wants to make, which is more friction and never
|
|
109
|
+
# more authority. --no-review escapes it for one run.
|
|
110
|
+
"subagent_review": bool,
|
|
111
|
+
}
|
|
112
|
+
RESEARCH_APPROVAL_MODES = ("none", "attended")
|
|
113
|
+
_KNOWN_COMPACTION = {
|
|
114
|
+
"strength": int,
|
|
115
|
+
"keep_recent_tokens": int,
|
|
116
|
+
# ROADMAP_v2 §21 (M1). This was `buffer_tokens` while §21 was specified
|
|
117
|
+
# as "headroom before the model's context window". It is not headroom:
|
|
118
|
+
# the trigger is a WORKING-SET TARGET -- a window-derived threshold
|
|
119
|
+
# sits at a size a working thread never reaches, while ~40k keeps turns
|
|
120
|
+
# cheap and their tool steps intact. Renamed rather than redefined -- a
|
|
121
|
+
# key that keeps its name and changes its meaning is worse than one
|
|
122
|
+
# that moves, and these keys have been inert since §14 so nothing
|
|
123
|
+
# depends on the old spelling.
|
|
124
|
+
"trigger_tokens": int,
|
|
125
|
+
"warning_margin_tokens": int,
|
|
126
|
+
# §21 M5 / M2 / retry bound.
|
|
127
|
+
"keep_recent_turns": int,
|
|
128
|
+
"strategy": str,
|
|
129
|
+
"max_retries": int,
|
|
130
|
+
}
|
|
131
|
+
# ROADMAP_v2 §16. TUI preferences live here rather than in a dotfile of
|
|
132
|
+
# their own because unknown settings keys RAISE (§14 amendment 1) -- a
|
|
133
|
+
# preference the loader doesn't know about is a startup error, not a
|
|
134
|
+
# silently ignored line. That is the tradeoff the loud-rejection rule
|
|
135
|
+
# buys: adding a preference is a schema change, on purpose.
|
|
136
|
+
_KNOWN_TUI = {
|
|
137
|
+
"theme": str, # one of tui/themes.py's registered names
|
|
138
|
+
"animations": bool, # master switch for the raven + transitions
|
|
139
|
+
"effort": str, # persisted reasoning-effort level (§16)
|
|
140
|
+
"todo_position": str, # §23 slice 2: one of TODO_POSITIONS below
|
|
141
|
+
# §38 (O6): render a turn's reasoning inline, or collapse it to an
|
|
142
|
+
# animated one-line indicator. Defaults to True in tui/app.py rather
|
|
143
|
+
# than to a config.yaml key, following `animations` above -- config.yaml
|
|
144
|
+
# holds no TUI values.
|
|
145
|
+
"show_thinking": bool,
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
# §23 slice 2. Validated AT LOAD and raising, following
|
|
149
|
+
# research.approval_mode rather than tui.theme.
|
|
150
|
+
#
|
|
151
|
+
# tui.theme validates at use and falls back, for a stated reason: the loader
|
|
152
|
+
# cannot know the valid names without importing tui/themes.py, and a stale
|
|
153
|
+
# theme name should not stop the app from starting. Neither applies here --
|
|
154
|
+
# the vocabulary is three words that live in this file, and a position the
|
|
155
|
+
# renderer does not understand would silently put the panel somewhere the
|
|
156
|
+
# user did not ask for. tui.effort had neither check and that was a shipped
|
|
157
|
+
# bug (§16), which is the third precedent and the one that decided it.
|
|
158
|
+
TODO_POSITIONS = ("top", "bottom", "side")
|
|
159
|
+
|
|
160
|
+
# Settings whose value is a nested object. Both the validator and the
|
|
161
|
+
# cross-tier merge below iterate this rather than naming keys twice --
|
|
162
|
+
# `compaction` was the only member until §16 added `tui`, and hardcoding
|
|
163
|
+
# it in two places is what made the shallow-merge defect
|
|
164
|
+
# (review finding F2) possible in the first place.
|
|
165
|
+
_KNOWN_CONFIDENCE = {
|
|
166
|
+
"grounding_weight_factor": float,
|
|
167
|
+
"critic_weight_factor": float,
|
|
168
|
+
"assumption_flag_penalty": float,
|
|
169
|
+
# Retired by ROADMAP_v3 §90 and still KNOWN, as `ensemble_n` is (E4):
|
|
170
|
+
# unknown keys raise, so dropping it would stop every settings.json
|
|
171
|
+
# that sets it from loading. effective_confidence warns instead.
|
|
172
|
+
"disagreement_penalty_factor": float,
|
|
173
|
+
"non_factual_score_cap": float,
|
|
174
|
+
"source_quality_floor": float,
|
|
175
|
+
"authority_full_credit": float,
|
|
176
|
+
"grounding_weights": dict,
|
|
177
|
+
"tier_thresholds": dict,
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
_KNOWN_SOURCE_SCORING = {
|
|
181
|
+
"similarity_floor": float,
|
|
182
|
+
"similarity_ceiling": float,
|
|
183
|
+
"authority_adjustment_cap": float,
|
|
184
|
+
"window_chars": int,
|
|
185
|
+
"max_windows": int,
|
|
186
|
+
"claim_min_chars": int,
|
|
187
|
+
"top_k": int,
|
|
188
|
+
"domain_overrides": dict,
|
|
189
|
+
"venue_weight": float,
|
|
190
|
+
"citation_weight": float,
|
|
191
|
+
"author_weight": float,
|
|
192
|
+
"h_saturation": int,
|
|
193
|
+
"min_cohort_size": int,
|
|
194
|
+
"min_citation_age_days": int,
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
_NESTED_SETTINGS = {
|
|
198
|
+
"compaction": _KNOWN_COMPACTION,
|
|
199
|
+
"tui": _KNOWN_TUI,
|
|
200
|
+
"research": _KNOWN_RESEARCH,
|
|
201
|
+
"confidence": _KNOWN_CONFIDENCE,
|
|
202
|
+
"source_scoring": _KNOWN_SOURCE_SCORING,
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
@dataclass
|
|
207
|
+
class SkillDef:
|
|
208
|
+
name: str
|
|
209
|
+
description: str
|
|
210
|
+
# ROADMAP_v2 §19 (K2): the tools this skill's methodology DEPENDS on,
|
|
211
|
+
# not tools it grants. It cannot grant any -- ToolContext.allowed_tools
|
|
212
|
+
# is a whitelist that narrows, and D14 forbids widening -- so this is
|
|
213
|
+
# checked at activation and reported, never applied. See SkillManager.
|
|
214
|
+
additional_tools: list
|
|
215
|
+
body: str
|
|
216
|
+
tier: str
|
|
217
|
+
path: str
|
|
218
|
+
# §19 (K4): folder path from the tier root, posix separators, "" for a
|
|
219
|
+
# skill sitting directly in skills/. Metadata, not identity -- the name
|
|
220
|
+
# stays global, so load_skill, /skill and D18's collision rule are
|
|
221
|
+
# untouched by where a file lives. Used to GROUP the catalog, which is
|
|
222
|
+
# the point: progressive disclosure puts every skill's name and
|
|
223
|
+
# description in every system prompt, and a flat list of forty is
|
|
224
|
+
# exactly what that design exists to avoid.
|
|
225
|
+
category: str = ""
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
@dataclass
|
|
229
|
+
class AgentDef:
|
|
230
|
+
name: str
|
|
231
|
+
description: str
|
|
232
|
+
model: Optional[str]
|
|
233
|
+
provider: Optional[str]
|
|
234
|
+
allowed_tools: Optional[list]
|
|
235
|
+
approval_overrides: dict
|
|
236
|
+
use_project_context: bool
|
|
237
|
+
use_memory: bool
|
|
238
|
+
max_steps: Optional[int]
|
|
239
|
+
body: str
|
|
240
|
+
tier: str
|
|
241
|
+
path: str
|
|
242
|
+
# ROADMAP_v2 §32 (A3), #69. Whether spawn_subagent can actually
|
|
243
|
+
# FEED this agent -- i.e. whether a task string, as the first
|
|
244
|
+
# message of a fresh thread, is the input its body expects.
|
|
245
|
+
#
|
|
246
|
+
# None means UNDECLARED, and what that means depends on who wrote
|
|
247
|
+
# the file. For the harness tier it is a build error, caught by
|
|
248
|
+
# assert_spawnable_declared below: our own omission is a bug, and
|
|
249
|
+
# R13's rule applies -- omission has to be a detectable mistake
|
|
250
|
+
# rather than an inherited answer. For the user and project tiers
|
|
251
|
+
# it reads as False, because a third party's silence must get the
|
|
252
|
+
# SAFE answer: an agent written for /agent and advertised as
|
|
253
|
+
# spawnable does not fail, it under-performs, and the parent has
|
|
254
|
+
# no way to tell (which is #69's actual complaint).
|
|
255
|
+
#
|
|
256
|
+
# IT IS A PERMISSION SINCE A16 (ROADMAP_v3 section 62), and this
|
|
257
|
+
# comment said the opposite for as long as it was true. Consulted by
|
|
258
|
+
# prompts.system_prompts.agent_catalog_text AND by
|
|
259
|
+
# agents.subagent_tool.refusal_reason, which refuses a spawn of
|
|
260
|
+
# anything whose value is not True.
|
|
261
|
+
#
|
|
262
|
+
# WHY IT CHANGED. "Advertisement only" meant `spawn_subagent` would
|
|
263
|
+
# spawn any agent the loader had discovered, by name -- including one
|
|
264
|
+
# the asymmetry above had deliberately left off the catalog. So the
|
|
265
|
+
# measured consequence of a stranger's silence was INVISIBLE rather
|
|
266
|
+
# than unreachable: a project-tier agent omitting the field was absent
|
|
267
|
+
# from the prompt a user might read and callable by any text that named
|
|
268
|
+
# it. For the careless author the default was safe; for the adversarial
|
|
269
|
+
# one it was the quieter option, which inverts the ranking the SAFE
|
|
270
|
+
# answer above is reaching for. Its own sibling `spawn_targets` had
|
|
271
|
+
# already chosen refusal for the same class of problem.
|
|
272
|
+
#
|
|
273
|
+
# C6 still caps a child's tools and the depth limit still applies;
|
|
274
|
+
# those were never what this field was about. D24's rule
|
|
275
|
+
# ("advertising an uncallable tool is not harmless: the model keeps
|
|
276
|
+
# choosing it") still applies to the catalog's entries -- and now its
|
|
277
|
+
# mirror applies too: a catalog that hides a REACHABLE agent is the
|
|
278
|
+
# same defect read the other way.
|
|
279
|
+
spawnable: Optional[bool] = None
|
|
280
|
+
# WHICH agents this one may spawn. `spawnable` above is the other
|
|
281
|
+
# half of the same question -- that one is about being spawned, this
|
|
282
|
+
# one about spawning -- and they are separate fields because an agent
|
|
283
|
+
# can be either, both or neither.
|
|
284
|
+
#
|
|
285
|
+
# None means NO OPINION: every spawnable agent, which is what every
|
|
286
|
+
# definition written before this field got and therefore what it has
|
|
287
|
+
# to keep meaning. A list restricts, and only ever restricts: it
|
|
288
|
+
# composes by intersection on ToolContext exactly as allowed_tools
|
|
289
|
+
# does, so it cannot be widened by nesting.
|
|
290
|
+
#
|
|
291
|
+
# It is a REFUSAL, unlike `spawnable`'s advertisement. C6 already
|
|
292
|
+
# narrows a child's tools to the parent's, which is the wrong answer
|
|
293
|
+
# when the parent should not be delegating that work at all: `plan`
|
|
294
|
+
# spawning `build` produced a build stripped of `write` and `edit`,
|
|
295
|
+
# degraded exactly the way A4 describes and invisible from both ends.
|
|
296
|
+
# Read by agents.subagent_tool.refusal_reason (the refusal) and by
|
|
297
|
+
# prompts.system_prompts.agent_catalog_text (D24: an agent that will
|
|
298
|
+
# be refused is not advertised in the first place).
|
|
299
|
+
spawn_targets: Optional[list] = None
|
|
300
|
+
|
|
301
|
+
@property
|
|
302
|
+
def pins_model(self) -> bool:
|
|
303
|
+
"""Whether this file names EITHER half of its model pair.
|
|
304
|
+
|
|
305
|
+
One predicate, because three readers ask it and have to agree:
|
|
306
|
+
the subagent spawn, which takes the `/submodel` pair only for an
|
|
307
|
+
agent that pins neither half (ROADMAP_v3 §57, SM6); the note a
|
|
308
|
+
`/model` switch owes an active agent that pins; and `/submodel`'s
|
|
309
|
+
list of the agents it does not reach. A half pin opts out WHOLE:
|
|
310
|
+
a model name means nothing against the wrong provider, so filling
|
|
311
|
+
the missing half from `/submodel` would build a pair nobody chose.
|
|
312
|
+
"""
|
|
313
|
+
return bool(self.model or self.provider)
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _user_config_dir() -> str:
|
|
317
|
+
# Call-time resolution so tests can redirect it without touching the
|
|
318
|
+
# real user config dir.
|
|
319
|
+
return os.path.expanduser("~/.config/venastine")
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _catalog_text(value) -> str:
|
|
323
|
+
"""One line, bounded, for a string that enters every system prompt.
|
|
324
|
+
|
|
325
|
+
Applied to a skill's and an agent's `name` and `description` at
|
|
326
|
+
PARSE TIME, for every tier -- ROADMAP_v2 §32 A5, #131.
|
|
327
|
+
|
|
328
|
+
TWO RULES, ONE FUNCTION, because they are one rule about one kind
|
|
329
|
+
of string:
|
|
330
|
+
|
|
331
|
+
COLLAPSE WHITESPACE, and this is the security half. The catalogs
|
|
332
|
+
render `- {name}: {description}`, so a value containing a
|
|
333
|
+
newline does not merely look untidy -- it LEAVES ITS BULLET.
|
|
334
|
+
A project skill described with a YAML block scalar holding
|
|
335
|
+
`## Available tools` renders as a top-level section of the
|
|
336
|
+
system prompt, indistinguishable from one this harness wrote.
|
|
337
|
+
Driven, and it worked.
|
|
338
|
+
|
|
339
|
+
CAP THE LENGTH, and this is the budget half. Progressive
|
|
340
|
+
disclosure puts every name and description in every prompt of
|
|
341
|
+
every run; without a bound, one project file sets that cost.
|
|
342
|
+
|
|
343
|
+
TRUNCATE RATHER THAN REFUSE THE FILE. `_parse_md_file` skips a
|
|
344
|
+
malformed definition, which is right for `additional_tools` --
|
|
345
|
+
a non-string element crashes its first consumer. A description is
|
|
346
|
+
advisory prose read by a model, so it degrades gracefully, and
|
|
347
|
+
losing a whole working skill over the length of its summary would
|
|
348
|
+
be the worse trade. The ellipsis is visible on purpose: a model
|
|
349
|
+
reading a cut-off summary should be able to tell.
|
|
350
|
+
|
|
351
|
+
EVERY TIER, not just project. A rule that applies only to
|
|
352
|
+
untrusted input is a second source of truth about what a catalog
|
|
353
|
+
entry is, and the harness tier is where a regression would be
|
|
354
|
+
least visible -- so the shipped files are held to it too, and
|
|
355
|
+
tests/test_catalog_text.py asserts they already comply.
|
|
356
|
+
"""
|
|
357
|
+
cap = config_schema.current().max_catalog_text_chars
|
|
358
|
+
text = " ".join(str(value).split())
|
|
359
|
+
if len(text) <= cap:
|
|
360
|
+
return text
|
|
361
|
+
return text[:cap - 1].rstrip() + "\u2026"
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def _parse_frontmatter(text: str) -> tuple[dict, str]:
|
|
365
|
+
"""Returns (frontmatter_dict, body). Uses a delimiter that must occupy
|
|
366
|
+
its own line -- NOT text.find("---"), which matches the first bare
|
|
367
|
+
'---' anywhere in the file, including a markdown horizontal rule
|
|
368
|
+
inside the body itself (a very common convention in longer
|
|
369
|
+
instructional content, which is exactly what agent/skill bodies are).
|
|
370
|
+
"""
|
|
371
|
+
matches = list(_FRONTMATTER_DELIM.finditer(text))
|
|
372
|
+
if len(matches) < 2:
|
|
373
|
+
raise ValueError("No valid YAML frontmatter block found (need two '---' lines)")
|
|
374
|
+
# The opening delimiter must be the FIRST thing in the file. Without
|
|
375
|
+
# this, a file with no frontmatter at all but two horizontal rules in
|
|
376
|
+
# its body has the prose between them handed to yaml.safe_load --
|
|
377
|
+
# which usually fails the "is it a mapping / does it have a name"
|
|
378
|
+
# checks downstream, but reports the wrong reason when it does.
|
|
379
|
+
if matches[0].start() != 0:
|
|
380
|
+
raise ValueError(
|
|
381
|
+
"File does not begin with a YAML frontmatter block "
|
|
382
|
+
"(the opening '---' must be the first line)")
|
|
383
|
+
frontmatter_text = text[matches[0].end():matches[1].start()]
|
|
384
|
+
body = text[matches[1].end():].strip()
|
|
385
|
+
return yaml.safe_load(frontmatter_text), body
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def _parse_md_file(path: str, kind: str, tier: str, category: str = ""):
|
|
389
|
+
"""Parse one agent/skill .md file. Returns a SkillDef/AgentDef, or
|
|
390
|
+
None (with a warning) for malformed files -- a broken definition
|
|
391
|
+
warns and skips rather than taking down startup."""
|
|
392
|
+
try:
|
|
393
|
+
# utf-8-sig, not utf-8: a BOM (Notepad's default on this project's
|
|
394
|
+
# platform) is decoded as a leading , so the frontmatter's
|
|
395
|
+
# ^--- no longer sits at position 0 and the file is skipped with
|
|
396
|
+
# the false diagnosis "does not begin with a YAML frontmatter
|
|
397
|
+
# block". utf-8-sig strips a BOM and is a no-op without one.
|
|
398
|
+
# Regular files only (batch 219, #214): this runs for the trust
|
|
399
|
+
# prompt's listing, before any grant, and a cloned skill linked to
|
|
400
|
+
# /dev/zero was read until memory ran out. The refusal is an
|
|
401
|
+
# OSError, so it warns and skips below like any broken definition.
|
|
402
|
+
text = read_regular_text(path)
|
|
403
|
+
fm, body = _parse_frontmatter(text)
|
|
404
|
+
except (OSError, ValueError, yaml.YAMLError) as e:
|
|
405
|
+
logger.warning("Skipping %s file %s: %s", kind, path, e)
|
|
406
|
+
return None
|
|
407
|
+
if not isinstance(fm, dict):
|
|
408
|
+
logger.warning("Skipping %s file %s: frontmatter is not a YAML mapping", kind, path)
|
|
409
|
+
return None
|
|
410
|
+
name = fm.get("name")
|
|
411
|
+
if not isinstance(name, str) or not name:
|
|
412
|
+
logger.warning("Skipping %s file %s: missing 'name' in frontmatter", kind, path)
|
|
413
|
+
return None
|
|
414
|
+
# #131: the NAME is interpolated into the same catalog line as the
|
|
415
|
+
# description (`- {name}: {description}`), so it is the same
|
|
416
|
+
# injection surface and gets the same treatment. Bounded AFTER the
|
|
417
|
+
# emptiness check, because an all-whitespace name collapses to ""
|
|
418
|
+
# and must be reported as missing rather than registered as a
|
|
419
|
+
# nameless definition that D18's collision rule cannot reason about.
|
|
420
|
+
name = _catalog_text(name)
|
|
421
|
+
if not name:
|
|
422
|
+
logger.warning("Skipping %s file %s: 'name' is only whitespace",
|
|
423
|
+
kind, path)
|
|
424
|
+
return None
|
|
425
|
+
if kind == "skills":
|
|
426
|
+
tools = fm.get("additional_tools") or []
|
|
427
|
+
if not isinstance(tools, list):
|
|
428
|
+
logger.warning("Skipping skill file %s: additional_tools is not a list", path)
|
|
429
|
+
return None
|
|
430
|
+
# Element types, not just the container (review §19-20 f6): the
|
|
431
|
+
# first consumer feeds each element to a startswith() check, so a
|
|
432
|
+
# non-string element parsed cleanly here and crashed /skill at
|
|
433
|
+
# first consumption -- after "Activated" was already shown.
|
|
434
|
+
if not all(isinstance(t, str) for t in tools):
|
|
435
|
+
logger.warning("Skipping skill file %s: additional_tools is not a list of strings", path)
|
|
436
|
+
return None
|
|
437
|
+
return SkillDef(
|
|
438
|
+
name=name,
|
|
439
|
+
description=_catalog_text(fm.get("description", "")),
|
|
440
|
+
additional_tools=tools,
|
|
441
|
+
body=body,
|
|
442
|
+
tier=tier,
|
|
443
|
+
path=path,
|
|
444
|
+
category=category,
|
|
445
|
+
)
|
|
446
|
+
allowed = fm.get("allowed_tools")
|
|
447
|
+
if allowed is not None and not isinstance(allowed, list):
|
|
448
|
+
logger.warning("Skipping agent file %s: allowed_tools is not a list", path)
|
|
449
|
+
return None
|
|
450
|
+
# ELEMENT TYPES TOO, not just the container. This is the lesson the
|
|
451
|
+
# skills branch above already paid for with `additional_tools`: a
|
|
452
|
+
# non-string element parses cleanly here and fails at first
|
|
453
|
+
# consumption, layers away from the file that caused it. THIS FIELD IS
|
|
454
|
+
# THE ONE THE RULE WAS NEVER APPLIED TO (batch 155) -- it sat under
|
|
455
|
+
# the comment stating it, with the container checked and the elements
|
|
456
|
+
# not.
|
|
457
|
+
#
|
|
458
|
+
# Its two consumers fail in OPPOSITE directions, both measured. A
|
|
459
|
+
# hashable non-string (`[read, 123]`) survives `set()` and then
|
|
460
|
+
# matches nothing, so the whitelist silently holds one fewer entry
|
|
461
|
+
# than it reads as -- the whitelist-typo class
|
|
462
|
+
# tests/test_shipped_roster.py documents for misspelled strings. An
|
|
463
|
+
# unhashable one (`[read, [write]]`, `[read, {write: true}]`) raises
|
|
464
|
+
# `TypeError: unhashable type` out of `AgentManager.active_context`
|
|
465
|
+
# and `child_context`, which are the single place this `Optional[list]`
|
|
466
|
+
# becomes `ToolContext.allowed_tools`'s declared `Optional[set[str]]`.
|
|
467
|
+
#
|
|
468
|
+
# SKIP, NOT DROP THE ELEMENT, for the reason `approval_overrides`
|
|
469
|
+
# states below: half a whitelist is worse than no file at all for a
|
|
470
|
+
# field whose job is deciding what an agent may call.
|
|
471
|
+
if allowed is not None and not all(isinstance(t, str) for t in allowed):
|
|
472
|
+
logger.warning(
|
|
473
|
+
"Skipping agent file %s: allowed_tools is not a list of strings",
|
|
474
|
+
path)
|
|
475
|
+
return None
|
|
476
|
+
# The consumer here is a set membership test against an agent NAME, so
|
|
477
|
+
# a nested list or a mapping raises where that set is built and a
|
|
478
|
+
# hashable non-string sits inert and never matches -- the same pair
|
|
479
|
+
# measured one field up.
|
|
480
|
+
targets = fm.get("spawn_targets")
|
|
481
|
+
if targets is not None and not isinstance(targets, list):
|
|
482
|
+
logger.warning(
|
|
483
|
+
"Skipping agent file %s: spawn_targets is not a list", path)
|
|
484
|
+
return None
|
|
485
|
+
if targets is not None and not all(isinstance(t, str) for t in targets):
|
|
486
|
+
logger.warning(
|
|
487
|
+
"Skipping agent file %s: spawn_targets is not a list of strings",
|
|
488
|
+
path)
|
|
489
|
+
return None
|
|
490
|
+
overrides = fm.get("approval_overrides") or {}
|
|
491
|
+
if not isinstance(overrides, dict):
|
|
492
|
+
logger.warning("Skipping agent file %s: approval_overrides is not a mapping", path)
|
|
493
|
+
return None
|
|
494
|
+
# THE VALUES TOO, not just the mapping (batch 150). The two scalar
|
|
495
|
+
# booleans below are validated for the reason stated there, and this
|
|
496
|
+
# field is the third one a restrictive agent definition uses to opt
|
|
497
|
+
# IN to prompting -- so stopping at the container left
|
|
498
|
+
# `web_search: "false"` parsing to the STRING, and
|
|
499
|
+
# requires_approval() returning 'false' from a function documented to
|
|
500
|
+
# return a bool. Measured: no value widens a gate, because approval
|
|
501
|
+
# ORs and every falsy value is already indistinguishable from the key
|
|
502
|
+
# being absent -- so this is the type boundary, not an escalation.
|
|
503
|
+
# Same warn-and-skip contract as every other typed field here; a
|
|
504
|
+
# per-key repair would leave half a policy, which is worse than no
|
|
505
|
+
# file at all for a field whose job is adding prompts.
|
|
506
|
+
if not all(isinstance(v, bool) for v in overrides.values()):
|
|
507
|
+
bad = sorted(k for k, v in overrides.items()
|
|
508
|
+
if not isinstance(v, bool))
|
|
509
|
+
logger.warning(
|
|
510
|
+
"Skipping agent file %s: approval_overrides values must be "
|
|
511
|
+
"booleans; %s %s not.",
|
|
512
|
+
path, ", ".join(repr(k) for k in bad),
|
|
513
|
+
"is" if len(bad) == 1 else "are")
|
|
514
|
+
return None
|
|
515
|
+
max_steps = fm.get("max_steps")
|
|
516
|
+
# REPAIR, NOT REJECT (#45, batch 16 -- explicit owner decision). This
|
|
517
|
+
# guard used to SKIP the whole file for a bad max_steps, which made it
|
|
518
|
+
# the one frontmatter field whose typo cost the agent entirely while
|
|
519
|
+
# every sibling produced a warning-and-carry-on. Worse, a negative int
|
|
520
|
+
# passed the type check and crashed `_run` with an AttributeError on
|
|
521
|
+
# `response.stop_reason` three layers away from the line that caused it.
|
|
522
|
+
#
|
|
523
|
+
# Now: any unusable value -- non-int, YAML's bool-reading of yes/on,
|
|
524
|
+
# or < 1 (`isinstance(True, int)` is True in Python) -- is repaired to
|
|
525
|
+
# None with a warning NAMING the original. None is what "not declared"
|
|
526
|
+
# means downstream, so every call site's existing
|
|
527
|
+
# `max_steps or config.MAX_ITERATIONS` fallback applies unchanged, and
|
|
528
|
+
# the warning's default figure and the one that takes effect are the
|
|
529
|
+
# same number by construction rather than by two copies staying in
|
|
530
|
+
# step. The belt for values that bypass this loader lives in
|
|
531
|
+
# core/loop._run, which raises instead of repairing: a programmatic
|
|
532
|
+
# caller passing a bad value directly has no file to lose, only a bug
|
|
533
|
+
# worth naming.
|
|
534
|
+
if max_steps is not None:
|
|
535
|
+
valid = (isinstance(max_steps, int)
|
|
536
|
+
and not isinstance(max_steps, bool)
|
|
537
|
+
and max_steps >= 1)
|
|
538
|
+
if not valid:
|
|
539
|
+
logger.warning(
|
|
540
|
+
"Agent file %s: max_steps must be a positive integer, got "
|
|
541
|
+
"%r; defaulting to %s (config.MAX_ITERATIONS).",
|
|
542
|
+
path, max_steps, config_schema.current().max_iterations)
|
|
543
|
+
max_steps = None
|
|
544
|
+
# Booleans are VALIDATED, not coerced. bool("false") is True, so
|
|
545
|
+
# `use_memory: "false"` would invert a deliberate opt-out with no
|
|
546
|
+
# warning -- and these two fields are exactly the ones a restrictive
|
|
547
|
+
# agent definition uses to opt OUT. Same warn-and-skip contract every
|
|
548
|
+
# other typed field in this function has.
|
|
549
|
+
flags = {}
|
|
550
|
+
for key, default in (("use_project_context", False), ("use_memory", True)):
|
|
551
|
+
value = fm.get(key, default)
|
|
552
|
+
if not isinstance(value, bool):
|
|
553
|
+
logger.warning(
|
|
554
|
+
"Skipping agent file %s: %s is not a boolean", path, key)
|
|
555
|
+
return None
|
|
556
|
+
flags[key] = value
|
|
557
|
+
spawnable = fm.get("spawnable")
|
|
558
|
+
if spawnable is not None and not isinstance(spawnable, bool):
|
|
559
|
+
logger.warning(
|
|
560
|
+
"Skipping agent file %s: spawnable is not a boolean", path)
|
|
561
|
+
return None
|
|
562
|
+
# STRINGS OR ABSENT, and these are the other two fields the rule above
|
|
563
|
+
# was never applied to (batch 155). Both were read straight out of
|
|
564
|
+
# frontmatter and handed to code that has always assumed a string, so
|
|
565
|
+
# a well-typed-but-wrong-SHAPED value loaded clean here and failed
|
|
566
|
+
# layers away with nothing naming the file. Measured against the real
|
|
567
|
+
# resolvers, nothing monkeypatched: `provider: 123` reaches
|
|
568
|
+
# `client.api_initialization` and raises `ValueError: Unknown
|
|
569
|
+
# provider: 123`, which names the provider and not the file that
|
|
570
|
+
# supplied it; `model: [a, b]` reaches the effort cache's
|
|
571
|
+
# `(provider_name, model)` key and raises `TypeError: unhashable
|
|
572
|
+
# type: 'list'`, which names neither. Both arrive there through
|
|
573
|
+
# `agents/subagent_tool.py` and `tui/app.py`, which pass
|
|
574
|
+
# `agent.model or ...` through verbatim.
|
|
575
|
+
#
|
|
576
|
+
# SKIP RATHER THAN REPAIR TO None, which is where this parts company
|
|
577
|
+
# with `max_steps` above (#45). None is a REAL value for both fields
|
|
578
|
+
# and means "inherit the session's", so repairing would leave the
|
|
579
|
+
# agent available and running on a model nobody chose for it. A file
|
|
580
|
+
# names a model when being a DIFFERENT model is the point, and §11's
|
|
581
|
+
# critic rationale is the sharpest form of that -- "a model checking
|
|
582
|
+
# its own output for errors shares that model's blind spots". A bad
|
|
583
|
+
# `max_steps` has no honest reading, so repairing it invents nothing;
|
|
584
|
+
# an unreadable `model` has one, and it is the wrong one.
|
|
585
|
+
for key in ("model", "provider"):
|
|
586
|
+
value = fm.get(key)
|
|
587
|
+
if value is not None and not isinstance(value, str):
|
|
588
|
+
logger.warning(
|
|
589
|
+
"Skipping agent file %s: %s is not a string", path, key)
|
|
590
|
+
return None
|
|
591
|
+
return AgentDef(
|
|
592
|
+
name=name,
|
|
593
|
+
description=_catalog_text(fm.get("description", "")),
|
|
594
|
+
model=fm.get("model"),
|
|
595
|
+
provider=fm.get("provider"),
|
|
596
|
+
allowed_tools=allowed,
|
|
597
|
+
approval_overrides=overrides,
|
|
598
|
+
use_project_context=flags["use_project_context"],
|
|
599
|
+
use_memory=flags["use_memory"],
|
|
600
|
+
spawnable=spawnable,
|
|
601
|
+
spawn_targets=targets,
|
|
602
|
+
max_steps=max_steps,
|
|
603
|
+
body=body,
|
|
604
|
+
tier=tier,
|
|
605
|
+
path=path,
|
|
606
|
+
)
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
def _tier_dirs(kind: str, project_path: str, trusted: bool) -> list:
|
|
610
|
+
"""Search order, and _discover() is first-wins, so this list IS the
|
|
611
|
+
precedence rule: harness > user > project.
|
|
612
|
+
|
|
613
|
+
ORDER CHANGED IN §17 (D29). It was harness > project > user, i.e. the
|
|
614
|
+
conventional "more specific config wins". That is the wrong default
|
|
615
|
+
here, because "project" does not mean "more specific" -- it means "it
|
|
616
|
+
arrived with a directory you cloned", which is the entire premise of
|
|
617
|
+
D17. The user tier is the one you actually authored, so it should
|
|
618
|
+
never be silently displaced by the tier you merely trusted once.
|
|
619
|
+
|
|
620
|
+
Harness stays first and un-overridable (D18, unchanged): a project can
|
|
621
|
+
still never impersonate a builtin.
|
|
622
|
+
|
|
623
|
+
Note this only ever breaks SAME-NAME ties. Non-colliding definitions
|
|
624
|
+
from every tier all load -- _discover() accumulates a union and
|
|
625
|
+
consults precedence only when a name repeats.
|
|
626
|
+
"""
|
|
627
|
+
dirs = [("harness", os.path.join(HARNESS_ROOT, kind, "builtin"))]
|
|
628
|
+
dirs.append(("user", os.path.join(_user_config_dir(), kind)))
|
|
629
|
+
if trusted:
|
|
630
|
+
project_dir = os.path.join(
|
|
631
|
+
workspace_trust.venastine_dir(project_path), kind)
|
|
632
|
+
# #18. A SYMLINK at the project tier root escapes the D17 hash.
|
|
633
|
+
# `os.walk` follows the path handed to it as `top` but not
|
|
634
|
+
# symlinked subdirectories, and the two sides of the trust
|
|
635
|
+
# boundary start from different places: workspace_trust walks from
|
|
636
|
+
# `.venastine`, so a symlinked `skills/` is a subdirectory it will
|
|
637
|
+
# not descend, while _md_files walks from `.venastine/skills`, so
|
|
638
|
+
# it IS the top and gets followed. Directory names never enter the
|
|
639
|
+
# hash either, so the link contributes nothing -- not even its own
|
|
640
|
+
# name. The pointed-to definitions therefore load as project-tier
|
|
641
|
+
# agents/skills while being absent from the trust prompt's listing
|
|
642
|
+
# AND from the hash, so their bodies can be rewritten freely after
|
|
643
|
+
# one grant and `is_trusted()` keeps returning True.
|
|
644
|
+
#
|
|
645
|
+
# Treated as ABSENT rather than loaded-and-warned, matching the
|
|
646
|
+
# module's posture for untrusted content generally. The payload
|
|
647
|
+
# here is not data but instructions: a project-tier body becomes
|
|
648
|
+
# system-prompt content for the user's sessions.
|
|
649
|
+
#
|
|
650
|
+
# Scoped to the PROJECT tier deliberately. Harness and user
|
|
651
|
+
# directories are not hashed by anything, so a symlink in them is
|
|
652
|
+
# not a trust question -- and symlinking `~/.config/venastine/`
|
|
653
|
+
# into a dotfiles repo is a legitimate thing an operator does to
|
|
654
|
+
# their own config.
|
|
655
|
+
if os.path.islink(project_dir):
|
|
656
|
+
logger.warning(
|
|
657
|
+
"Ignoring project %s directory %s: it is a symlink, and "
|
|
658
|
+
"symlinked tier roots are invisible to the workspace-trust "
|
|
659
|
+
"content hash (see issue #18). Move the definitions inside "
|
|
660
|
+
".venastine/ to load them.", kind, project_dir)
|
|
661
|
+
else:
|
|
662
|
+
dirs.append(("project", project_dir))
|
|
663
|
+
return dirs
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def _md_files(directory: str, recursive: bool):
|
|
667
|
+
"""(absolute path, category) for every .md file in a tier directory.
|
|
668
|
+
|
|
669
|
+
ROADMAP_v2 §19 (K4): SKILLS nest under category folders --
|
|
670
|
+
`skills/builtin/<category>/<skill>.md` -- and agents stay flat, since
|
|
671
|
+
`agents/builtin/` holds one file and speculative structure for it
|
|
672
|
+
would be structure nobody asked for. One branch here rather than a
|
|
673
|
+
second discovery function, so tier handling and collision handling
|
|
674
|
+
cannot diverge between the two kinds.
|
|
675
|
+
|
|
676
|
+
The category is DERIVED from the folder, never authored: a `category:`
|
|
677
|
+
frontmatter key would immediately be able to disagree with where the
|
|
678
|
+
file actually sits, and then two places would claim to know.
|
|
679
|
+
|
|
680
|
+
Walk order matches workspace_trust.content_files() -- dirs.sort() plus
|
|
681
|
+
sorted(files) -- because same-name collisions resolve first-wins, so a
|
|
682
|
+
nondeterministic order would make WHICH definition loads depend on the
|
|
683
|
+
filesystem.
|
|
684
|
+
"""
|
|
685
|
+
if not recursive:
|
|
686
|
+
return [
|
|
687
|
+
(os.path.join(directory, f), "")
|
|
688
|
+
for f in sorted(os.listdir(directory)) if f.endswith(".md")
|
|
689
|
+
]
|
|
690
|
+
out = []
|
|
691
|
+
for dirpath, dirs, files in os.walk(directory):
|
|
692
|
+
dirs.sort()
|
|
693
|
+
category = os.path.relpath(dirpath, directory).replace(os.sep, "/")
|
|
694
|
+
if category == ".":
|
|
695
|
+
category = ""
|
|
696
|
+
for fname in sorted(files):
|
|
697
|
+
if fname.endswith(".md"):
|
|
698
|
+
out.append((os.path.join(dirpath, fname), category))
|
|
699
|
+
return out
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
def _discover(kind: str, project_path: str, trusted: bool) -> dict:
|
|
703
|
+
found = {}
|
|
704
|
+
for tier, directory in _tier_dirs(kind, project_path, trusted):
|
|
705
|
+
if not os.path.isdir(directory):
|
|
706
|
+
continue
|
|
707
|
+
for path, category in _md_files(directory, recursive=(kind == "skills")):
|
|
708
|
+
defn = _parse_md_file(path, kind, tier, category)
|
|
709
|
+
if defn is None:
|
|
710
|
+
continue
|
|
711
|
+
if defn.name in found:
|
|
712
|
+
winner = found[defn.name]
|
|
713
|
+
if winner.tier == "harness":
|
|
714
|
+
logger.warning(
|
|
715
|
+
"%s-tier %s %r collides with a harness builtin; "
|
|
716
|
+
"the builtin wins and the %s file is ignored (D18).",
|
|
717
|
+
tier, kind, defn.name, tier,
|
|
718
|
+
)
|
|
719
|
+
else:
|
|
720
|
+
# Cross-tier, USER beats project for agents and skills:
|
|
721
|
+
# _tier_dirs returns [harness, user, project] and this
|
|
722
|
+
# loop is first-wins. That is D29, not D8 -- D8 is where
|
|
723
|
+
# definitions are discovered, D29 is the §17 inversion
|
|
724
|
+
# that put user ahead of project, because "project" does
|
|
725
|
+
# not mean "more specific", it means "it arrived with a
|
|
726
|
+
# directory you cloned". (This comment said the opposite
|
|
727
|
+
# and cited D8; it was pre-D29 text that survived the
|
|
728
|
+
# change. See audit #20 -- the warning below was always
|
|
729
|
+
# right, because it names the winner dynamically.)
|
|
730
|
+
#
|
|
731
|
+
# settings.json deliberately goes the OTHER way,
|
|
732
|
+
# project-over-user, at _load_merged_settings -- because
|
|
733
|
+
# the trust prompt shows its values verbatim. Two
|
|
734
|
+
# opposite rules in one module is exactly what a comment
|
|
735
|
+
# here should disambiguate.
|
|
736
|
+
#
|
|
737
|
+
# Same-tier is alphabetical-first-wins. None of this is
|
|
738
|
+
# guessable from the outside, so say so:
|
|
739
|
+
# Silently shadowing a definition is how someone spends
|
|
740
|
+
# an afternoon editing a file that is never loaded.
|
|
741
|
+
logger.warning(
|
|
742
|
+
"%s-tier %s %r at %s is shadowed by the %s-tier "
|
|
743
|
+
"definition at %s, which wins.",
|
|
744
|
+
tier, kind, defn.name, defn.path, winner.tier,
|
|
745
|
+
winner.path,
|
|
746
|
+
)
|
|
747
|
+
continue
|
|
748
|
+
found[defn.name] = defn
|
|
749
|
+
return found
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
def _type_ok(value, expected) -> bool:
|
|
753
|
+
# bool is a subclass of int in Python, so int-typed keys must reject
|
|
754
|
+
# booleans explicitly, and bool-typed keys must reject ints -- including
|
|
755
|
+
# through a tuple expected-type: isinstance(True, (int, type(None))) is
|
|
756
|
+
# True, and max_token_budget=True would otherwise sail through as 1.
|
|
757
|
+
expecteds = expected if isinstance(expected, tuple) else (expected,)
|
|
758
|
+
if isinstance(value, bool):
|
|
759
|
+
return bool in expecteds
|
|
760
|
+
ok = False
|
|
761
|
+
for e in expecteds:
|
|
762
|
+
if e is bool:
|
|
763
|
+
continue # a non-bool value can never match bool
|
|
764
|
+
if e is int:
|
|
765
|
+
ok = ok or isinstance(value, int)
|
|
766
|
+
else:
|
|
767
|
+
ok = ok or isinstance(value, e)
|
|
768
|
+
return ok
|
|
769
|
+
|
|
770
|
+
|
|
771
|
+
def _validate_settings(data, source: str) -> None:
|
|
772
|
+
"""Loud rejection of unknown keys (a typo must not masquerade as a
|
|
773
|
+
setting) and type checks for known ones. Compaction keys are known
|
|
774
|
+
but not consumed until §21 -- they validate here and warn at init."""
|
|
775
|
+
if not isinstance(data, dict):
|
|
776
|
+
raise ValueError(f"settings.json at {source} is not a JSON object")
|
|
777
|
+
for key, value in data.items():
|
|
778
|
+
if key == "ensemble_models":
|
|
779
|
+
# ROADMAP §10 revisit (E2), and R12's rule applied to a second
|
|
780
|
+
# kind of authority. Rejected BY NAME rather than as an unknown
|
|
781
|
+
# key, because the generic message reads as an oversight to be
|
|
782
|
+
# fixed by adding support for it.
|
|
783
|
+
#
|
|
784
|
+
# Turning ensemble mode on can only spend more of the provider
|
|
785
|
+
# the user already chose. A ROSTER chooses providers -- so a
|
|
786
|
+
# project's settings.json, which beats the user's, could point N
|
|
787
|
+
# research passes at endpoints the user never configured for this
|
|
788
|
+
# work and multiply the run's cost by the length of a list it
|
|
789
|
+
# supplied. §14 already flagged project-tier provider selection as
|
|
790
|
+
# a distinct grant; this is that grant times N.
|
|
791
|
+
raise ValueError(
|
|
792
|
+
f"settings.json at {source}: ensemble_models is deliberately "
|
|
793
|
+
f"not supported -- a roster chooses which providers N research "
|
|
794
|
+
f"passes call, and a project's settings.json beats the "
|
|
795
|
+
f"user's. Set ensemble_models in config.yaml instead "
|
|
796
|
+
f"(same posture as critic_model).")
|
|
797
|
+
if key == "shell_approval_mode":
|
|
798
|
+
# ROADMAP_v2 §28 (G7), and the third application of R12's rule.
|
|
799
|
+
# Rejected BY NAME for the reason the other two are: the
|
|
800
|
+
# generic unknown-key message reads as an oversight someone
|
|
801
|
+
# should fix by adding support, and this omission IS the
|
|
802
|
+
# design.
|
|
803
|
+
#
|
|
804
|
+
# A project's settings.json beats the user's (D29), and it
|
|
805
|
+
# arrives with a directory you cloned. This key decides
|
|
806
|
+
# whether shell commands are asked about at all -- so
|
|
807
|
+
# supporting it would let a cloned repo set "never" and turn
|
|
808
|
+
# `cat ~/.aws/credentials` into an unprompted host read. D17's
|
|
809
|
+
# trust gate is not a substitute: the same prompt covers a
|
|
810
|
+
# README-shaped AGENTS.md, and nobody reading it is deciding
|
|
811
|
+
# about their shell gate.
|
|
812
|
+
raise ValueError(
|
|
813
|
+
f"settings.json at {source}: shell_approval_mode is "
|
|
814
|
+
f"deliberately not supported -- it decides whether shell "
|
|
815
|
+
f"commands are approved at all, and a project's "
|
|
816
|
+
f"settings.json beats the user's. Set "
|
|
817
|
+
f"shell_approval_mode in config.yaml instead (same "
|
|
818
|
+
f"posture as ensemble_models and research.granted_tools).")
|
|
819
|
+
if key in ("critic_model", "embedder_model"):
|
|
820
|
+
# ROADMAP_v2 §45 (SQ7), and the fourth application of R12's
|
|
821
|
+
# rule. Both name a PROVIDER the pipeline will call: the critic
|
|
822
|
+
# sees every claim, and the embedder is sent claim text and the
|
|
823
|
+
# text of every page the run fetched. A project's settings.json
|
|
824
|
+
# beats the user's and arrives with a directory you cloned, so
|
|
825
|
+
# supporting either would let that directory choose where a
|
|
826
|
+
# research run's content is sent.
|
|
827
|
+
#
|
|
828
|
+
# By name rather than as an unknown key for the reason the
|
|
829
|
+
# other three are: the generic message reads as an oversight
|
|
830
|
+
# someone should fix by adding support, and this omission is
|
|
831
|
+
# the design. `/critic` and `/embedder` write to the user-tier
|
|
832
|
+
# store in core/pipeline_models.py, which is the route.
|
|
833
|
+
raise ValueError(
|
|
834
|
+
f"settings.json at {source}: {key} is deliberately not "
|
|
835
|
+
f"supported -- it names a provider this harness sends "
|
|
836
|
+
f"research content to, and a project's settings.json beats "
|
|
837
|
+
f"the user's. Use /critic or /embedder, or set "
|
|
838
|
+
f"critic_model / embedder_model in config.yaml "
|
|
839
|
+
f"(same posture as ensemble_models).")
|
|
840
|
+
if key not in _KNOWN_SETTINGS:
|
|
841
|
+
raise ValueError(f"settings.json at {source}: unknown key {key!r}")
|
|
842
|
+
expected = _KNOWN_SETTINGS[key]
|
|
843
|
+
if not _type_ok(value, expected):
|
|
844
|
+
expected_name = getattr(expected, "__name__", None) \
|
|
845
|
+
or "an int or null"
|
|
846
|
+
raise ValueError(
|
|
847
|
+
f"settings.json at {source}: key {key!r} must be {expected_name}, "
|
|
848
|
+
f"got {type(value).__name__}")
|
|
849
|
+
if key == "max_token_budget" and value is not None and value <= 0:
|
|
850
|
+
# #213. The type gate passes 0 and -5 -- they are ints -- and
|
|
851
|
+
# the loop's check is `used >= cap`, so a cap of zero or less
|
|
852
|
+
# let every turn spend ONE model call and then die reporting a
|
|
853
|
+
# budget problem for what is a typo in this file.
|
|
854
|
+
raise ValueError(
|
|
855
|
+
f"settings.json at {source}: max_token_budget must be a "
|
|
856
|
+
f"positive number of tokens, or null for no cap -- got "
|
|
857
|
+
f"{value}, which would stop every turn after its first "
|
|
858
|
+
f"model call.")
|
|
859
|
+
for section, known_keys in _NESTED_SETTINGS.items():
|
|
860
|
+
block = data.get(section)
|
|
861
|
+
if block is None:
|
|
862
|
+
continue
|
|
863
|
+
if not isinstance(block, dict):
|
|
864
|
+
raise ValueError(
|
|
865
|
+
f"settings.json at {source}: {section!r} must be an object")
|
|
866
|
+
for key, value in block.items():
|
|
867
|
+
if section == "research" and key == "granted_tools":
|
|
868
|
+
# §25 R12: rejected BY NAME, not as an unknown key. The
|
|
869
|
+
# generic message reads as an oversight someone should fix
|
|
870
|
+
# by adding support; this one says the omission is the
|
|
871
|
+
# design, so nobody "fixes" it into standing authorization
|
|
872
|
+
# that a cloned repo would carry.
|
|
873
|
+
raise ValueError(
|
|
874
|
+
f"settings.json at {source}: research.granted_tools is "
|
|
875
|
+
f"deliberately not supported -- a persisted grant list "
|
|
876
|
+
f"would authorise named tools for every future run, and "
|
|
877
|
+
f"a project's settings.json beats the user's. Grant per "
|
|
878
|
+
f"run with --grant / --grant-tools instead.")
|
|
879
|
+
if key not in known_keys:
|
|
880
|
+
raise ValueError(
|
|
881
|
+
f"settings.json at {source}: unknown {section} key {key!r}")
|
|
882
|
+
expected = known_keys[key]
|
|
883
|
+
if not _type_ok(value, expected):
|
|
884
|
+
raise ValueError(
|
|
885
|
+
f"settings.json at {source}: {section} key {key!r} must be "
|
|
886
|
+
f"{expected.__name__}, got {type(value).__name__}")
|
|
887
|
+
if section == "research" and key == "approval_mode" and \
|
|
888
|
+
value not in RESEARCH_APPROVAL_MODES:
|
|
889
|
+
raise ValueError(
|
|
890
|
+
f"settings.json at {source}: research.approval_mode must "
|
|
891
|
+
f"be one of {', '.join(RESEARCH_APPROVAL_MODES)}, "
|
|
892
|
+
f"got {value!r}")
|
|
893
|
+
if section == "tui" and key == "todo_position" and \
|
|
894
|
+
value not in TODO_POSITIONS:
|
|
895
|
+
raise ValueError(
|
|
896
|
+
f"settings.json at {source}: tui.todo_position must be "
|
|
897
|
+
f"one of {', '.join(TODO_POSITIONS)}, got {value!r}")
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
# ---------------------------------------------------------------------------
|
|
901
|
+
# ---- Compaction settings (ROADMAP_v2 §21, D27) ----------------------------
|
|
902
|
+
# ---------------------------------------------------------------------------
|
|
903
|
+
#
|
|
904
|
+
# Resolution lives HERE rather than in core/compaction.py because
|
|
905
|
+
# config_loader already owns the settings merge, and core/compaction.py
|
|
906
|
+
# reaches agents.manager, which imports this module -- resolving there and
|
|
907
|
+
# validating here would be a cycle. One function, called at startup so a
|
|
908
|
+
# bad value is a startup error, and called again per compaction so a
|
|
909
|
+
# per-invocation `/compact --strength 4` composes with the same rules.
|
|
910
|
+
|
|
911
|
+
# settings.json key -> the `config.yaml` field it defaults from.
|
|
912
|
+
#
|
|
913
|
+
# THESE ARE SCHEMA FIELD NAMES, NOT STRINGS LOOKED UP ON A MODULE. This map
|
|
914
|
+
# used to hold `"COMPACTION_TRIGGER_TOKENS"` and friends and reach them with
|
|
915
|
+
# `getattr(config, attr)` -- no default, so renaming or dropping any one of
|
|
916
|
+
# the seven turned `shipped_defaults()` into an AttributeError that surfaced
|
|
917
|
+
# through `initialize()` at startup, and nothing but this dictionary tied the
|
|
918
|
+
# two names together. `config_schema.HarnessConfig` is the source of truth
|
|
919
|
+
# now, so a missing field is a validation error naming the key and a renamed
|
|
920
|
+
# one is an AttributeError here at the same moment either way -- but the names
|
|
921
|
+
# below are checked against the model by
|
|
922
|
+
# `tests/test_config_loader.py::test_every_compaction_default_names_a_schema_field`.
|
|
923
|
+
_COMPACTION_DEFAULTS = {
|
|
924
|
+
"trigger_tokens": "compaction_trigger_tokens",
|
|
925
|
+
"warning_margin_tokens": "compaction_warning_margin_tokens",
|
|
926
|
+
"keep_recent_tokens": "compaction_keep_recent_tokens",
|
|
927
|
+
"keep_recent_turns": "compaction_keep_recent_turns",
|
|
928
|
+
"strength": "compaction_strength",
|
|
929
|
+
"max_retries": "compaction_max_retries",
|
|
930
|
+
"strategy": "compaction_strategy",
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
|
|
934
|
+
def shipped_defaults() -> dict:
|
|
935
|
+
"""Every nested knob at the value it takes when no settings.json
|
|
936
|
+
speaks. `{section: {key: default}}`, sections only -- the top-level
|
|
937
|
+
scalars are plain `config.yaml` keys and need no accessor.
|
|
938
|
+
|
|
939
|
+
EXISTS SO A TEMPLATE CANNOT QUOTE A STALE DEFAULT (§24 I15).
|
|
940
|
+
`/init --config` writes a settings.json of defaults, and a settings
|
|
941
|
+
file is unforgiving: `_validate_settings` raises on an unknown key, so
|
|
942
|
+
the scaffold has to be right the first time or the harness will not
|
|
943
|
+
start. Retyping these values there would be a second copy of them,
|
|
944
|
+
which is how the first one goes stale.
|
|
945
|
+
|
|
946
|
+
The two research-tuning builders live further down this file and are
|
|
947
|
+
resolved at call time, which is why this can sit beside the compaction
|
|
948
|
+
map it also reads.
|
|
949
|
+
"""
|
|
950
|
+
cfg = config_schema.current()
|
|
951
|
+
return {
|
|
952
|
+
"compaction": {key: getattr(cfg, field)
|
|
953
|
+
for key, field in _COMPACTION_DEFAULTS.items()},
|
|
954
|
+
"confidence": _confidence_defaults(),
|
|
955
|
+
"source_scoring": _source_scoring_defaults(),
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
|
|
959
|
+
def effective_compaction(overrides: Optional[dict] = None,
|
|
960
|
+
warn: bool = False) -> dict:
|
|
961
|
+
"""The compaction values actually in force.
|
|
962
|
+
|
|
963
|
+
A flat {key: value}, and DELIBERATELY STILL FLAT. Where each value
|
|
964
|
+
came from is `compaction_sources()` below, beside this rather than
|
|
965
|
+
inside it: twelve call sites subscript what this returns, so
|
|
966
|
+
`{key: (value, tier)}` breaks every one of them and a parallel dict
|
|
967
|
+
breaks none -- TECHNICAL_DEBT 12's first blocker, which is a decision
|
|
968
|
+
about shape and is unchanged by having built the second half.
|
|
969
|
+
|
|
970
|
+
This docstring used to open "and where they came from" and return a
|
|
971
|
+
flat dict, which is audit #91's finding; then it promised nothing and
|
|
972
|
+
pointed at the register. Now it points at the function that answers.
|
|
973
|
+
|
|
974
|
+
config.yaml default -> user settings.json -> trusted project
|
|
975
|
+
settings.json -> `overrides` (a per-invocation `/compact --strength 4`,
|
|
976
|
+
which applies to that one run and persists nothing). Nearest wins,
|
|
977
|
+
which is the precedence the rest of the config system already uses.
|
|
978
|
+
|
|
979
|
+
RELATIONSHIPS ARE VALIDATED, not just documented -- §21's own
|
|
980
|
+
instruction about warning_margin, which is the one value whose wrong
|
|
981
|
+
setting is not self-correcting: a margin at or above the trigger fires
|
|
982
|
+
the warning after the thing it warns about, which is no warning at all.
|
|
983
|
+
|
|
984
|
+
Raises ValueError on a value that cannot produce coherent trigger
|
|
985
|
+
math. That is the same posture settings.json already takes toward an
|
|
986
|
+
unknown key: a project's settings are content the user may not have
|
|
987
|
+
authored (D17's premise), and while nothing here can destroy anything
|
|
988
|
+
-- the archive is never edited by compaction, which is exactly the
|
|
989
|
+
property that makes this safe -- an absurd trigger would force
|
|
990
|
+
compaction constantly, and every compaction is a real model call the
|
|
991
|
+
user pays for.
|
|
992
|
+
"""
|
|
993
|
+
cfg = config_schema.current()
|
|
994
|
+
|
|
995
|
+
values, _ = _compaction_merge(overrides)
|
|
996
|
+
|
|
997
|
+
strength = values["strength"]
|
|
998
|
+
if strength not in cfg.compaction_target_ratios:
|
|
999
|
+
raise ValueError(
|
|
1000
|
+
f"compaction.strength must be one of "
|
|
1001
|
+
f"{sorted(cfg.compaction_target_ratios)}, got {strength!r}")
|
|
1002
|
+
if values["strategy"] not in cfg.compaction_strategies:
|
|
1003
|
+
raise ValueError(
|
|
1004
|
+
f"compaction.strategy must be one of "
|
|
1005
|
+
f"{', '.join(cfg.compaction_strategies)}, "
|
|
1006
|
+
f"got {values['strategy']!r}")
|
|
1007
|
+
if values["trigger_tokens"] < 1:
|
|
1008
|
+
raise ValueError(
|
|
1009
|
+
f"compaction.trigger_tokens must be positive, got "
|
|
1010
|
+
f"{values['trigger_tokens']}")
|
|
1011
|
+
if values["warning_margin_tokens"] >= values["trigger_tokens"]:
|
|
1012
|
+
raise ValueError(
|
|
1013
|
+
f"compaction.warning_margin_tokens ({values['warning_margin_tokens']}) "
|
|
1014
|
+
f"must be strictly less than trigger_tokens "
|
|
1015
|
+
f"({values['trigger_tokens']}), or the early warning fires after "
|
|
1016
|
+
f"the compaction it is warning about.")
|
|
1017
|
+
if values["keep_recent_tokens"] >= values["trigger_tokens"]:
|
|
1018
|
+
raise ValueError(
|
|
1019
|
+
f"compaction.keep_recent_tokens ({values['keep_recent_tokens']}) "
|
|
1020
|
+
f"must be strictly less than trigger_tokens "
|
|
1021
|
+
f"({values['trigger_tokens']}), or every message is protected at "
|
|
1022
|
+
f"the moment compaction fires and there is nothing to summarize.")
|
|
1023
|
+
for key in ("keep_recent_turns", "max_retries"):
|
|
1024
|
+
if values[key] < 0:
|
|
1025
|
+
raise ValueError(f"compaction.{key} must not be negative, "
|
|
1026
|
+
f"got {values[key]}")
|
|
1027
|
+
|
|
1028
|
+
# M1's arithmetic, enforced rather than left in a comment. The prompt
|
|
1029
|
+
# is re-billed on every step of a tool-using turn, so a turn starting
|
|
1030
|
+
# at T tokens and running k steps spends roughly k*T against the
|
|
1031
|
+
# CONFIGURED spend cap (settings.json max_token_budget). A trigger too
|
|
1032
|
+
# close to that cap lets a thread reach a size where the cap ends the
|
|
1033
|
+
# turn after one response -- compaction would then only ever fire on
|
|
1034
|
+
# the turn AFTER the one it should have saved.
|
|
1035
|
+
#
|
|
1036
|
+
# ONLY MEANINGFUL WHEN A CAP EXISTS (#4). Uncapped -- the default --
|
|
1037
|
+
# nothing competes with compaction: the trigger is a context-size
|
|
1038
|
+
# target and always was; the billing meter never gated it and now
|
|
1039
|
+
# gates nothing at all by default.
|
|
1040
|
+
# WARN ONLY WHEN ASKED, which is once at startup. This function is
|
|
1041
|
+
# on should_compact()'s path, so it runs once per step of every
|
|
1042
|
+
# turn -- an unconditional warning here would repeat a
|
|
1043
|
+
# configuration complaint dozens of times per conversation, which
|
|
1044
|
+
# is how a real warning becomes one nobody reads. The VALIDATION
|
|
1045
|
+
# above still raises on every call; only the advisory is gated.
|
|
1046
|
+
cap = spend_cap()
|
|
1047
|
+
if warn and cap is not None:
|
|
1048
|
+
headroom = cap / max(values["trigger_tokens"], 1)
|
|
1049
|
+
if headroom < 3:
|
|
1050
|
+
logger.warning(
|
|
1051
|
+
"compaction.trigger_tokens (%s) leaves room for only ~%.1f model "
|
|
1052
|
+
"calls within your configured spend cap max_token_budget (%s), "
|
|
1053
|
+
"because a tool-using turn re-sends its whole prompt each "
|
|
1054
|
+
"step. Turns on a nearly-full thread will stop early with "
|
|
1055
|
+
"token_budget_exceeded before compaction gets a chance. "
|
|
1056
|
+
"Lower the trigger or raise the cap.",
|
|
1057
|
+
values["trigger_tokens"], headroom, cap)
|
|
1058
|
+
return values
|
|
1059
|
+
|
|
1060
|
+
|
|
1061
|
+
def _compaction_merge(overrides: Optional[dict] = None) -> tuple[dict, dict]:
|
|
1062
|
+
"""`(values, sources)` for the seven compaction keys -- ONE ladder.
|
|
1063
|
+
|
|
1064
|
+
TECHNICAL_DEBT 12, batch 111. Both halves come out of the same three
|
|
1065
|
+
`update()` calls because the alternative is two ladders that have to
|
|
1066
|
+
agree about precedence, and a provenance report that disagrees with
|
|
1067
|
+
the value it describes is worse than no provenance: it is a confident
|
|
1068
|
+
wrong answer to "why is this number what it is".
|
|
1069
|
+
|
|
1070
|
+
`sources` names a tier for every key, always -- `"config.yaml"` when
|
|
1071
|
+
nothing outranks the file -- where `settings_sources()` records only
|
|
1072
|
+
what a settings file set. The difference is deliberate: that one
|
|
1073
|
+
answers "did anything override this", which absence can express, and
|
|
1074
|
+
this one answers "where did this number come from", which it cannot.
|
|
1075
|
+
"""
|
|
1076
|
+
values = shipped_defaults()["compaction"]
|
|
1077
|
+
sources = dict.fromkeys(values, "config.yaml")
|
|
1078
|
+
|
|
1079
|
+
tiers = settings_sources()
|
|
1080
|
+
for key, value in (get_settings().get("compaction") or {}).items():
|
|
1081
|
+
values[key] = value
|
|
1082
|
+
# The tier that WROTE it, not "settings.json": the merge upstream
|
|
1083
|
+
# kept them apart precisely so this line can tell them apart.
|
|
1084
|
+
sources[key] = tiers.get(f"compaction.{key}", "user")
|
|
1085
|
+
|
|
1086
|
+
for key, value in (overrides or {}).items():
|
|
1087
|
+
if value is not None:
|
|
1088
|
+
values[key] = value
|
|
1089
|
+
sources[key] = "override"
|
|
1090
|
+
|
|
1091
|
+
return values, sources
|
|
1092
|
+
|
|
1093
|
+
|
|
1094
|
+
def compaction_sources(overrides: Optional[dict] = None) -> dict:
|
|
1095
|
+
"""`{key: tier}` for the seven compaction keys -- where each value in
|
|
1096
|
+
force came from. TECHNICAL_DEBT 12, and D27's third implementation
|
|
1097
|
+
note, whose whole argument was that "compaction feels too aggressive"
|
|
1098
|
+
should be a one-line answer rather than a value three files away.
|
|
1099
|
+
|
|
1100
|
+
Tiers: `"config.yaml"`, `"user"`, `"project"`, `"override"` -- nearest
|
|
1101
|
+
wins, the same ladder `effective_compaction` walks, walked once by the
|
|
1102
|
+
function they share.
|
|
1103
|
+
|
|
1104
|
+
NO VALIDATION HERE, and that is not an oversight. This is a display
|
|
1105
|
+
call on a process whose configuration was validated at startup by
|
|
1106
|
+
`initialize()`; raising from a `/config` explanation would replace a
|
|
1107
|
+
sentence about a number with a traceback about a number.
|
|
1108
|
+
"""
|
|
1109
|
+
return _compaction_merge(overrides)[1]
|
|
1110
|
+
|
|
1111
|
+
|
|
1112
|
+
def _read_settings_file(path: str) -> dict:
|
|
1113
|
+
if not os.path.exists(path):
|
|
1114
|
+
return {}
|
|
1115
|
+
# utf-8-sig for the same reason as _parse_md_file: a BOM'd
|
|
1116
|
+
# settings.json makes json.load raise at the first character. And
|
|
1117
|
+
# regular files only, for the same reason too (batch 219).
|
|
1118
|
+
data = json.loads(read_regular_text(path)) # a JSONDecodeError is a ValueError
|
|
1119
|
+
_validate_settings(data, path)
|
|
1120
|
+
return data
|
|
1121
|
+
|
|
1122
|
+
|
|
1123
|
+
def user_settings() -> dict:
|
|
1124
|
+
"""The user tier's settings.json, alone and unmerged.
|
|
1125
|
+
|
|
1126
|
+
`get_settings()` answers what is IN FORCE, which is the wrong question
|
|
1127
|
+
for two callers. The merge below needs the user tier before the
|
|
1128
|
+
project tier lands on it, and `/init --config` needs it to say which
|
|
1129
|
+
of the user's own keys the file it is about to write will start
|
|
1130
|
+
deciding instead (§24 I16) -- a project key wins by PRESENCE, so a
|
|
1131
|
+
scaffold of pure defaults still displaces a customised user tier.
|
|
1132
|
+
|
|
1133
|
+
Resolved at call time, like every other path in this module.
|
|
1134
|
+
"""
|
|
1135
|
+
return _read_settings_file(
|
|
1136
|
+
os.path.join(_user_config_dir(), "settings.json"))
|
|
1137
|
+
|
|
1138
|
+
|
|
1139
|
+
def _load_merged_settings(project_path: str,
|
|
1140
|
+
trusted: bool) -> tuple[dict, dict]:
|
|
1141
|
+
"""`(values, sources)`. Resolution order: project (trusted) > user.
|
|
1142
|
+
Anything absent falls through to config.yaml defaults at the consumer.
|
|
1143
|
+
|
|
1144
|
+
Nested sections (`_NESTED_SETTINGS`) merge one level deeper than the
|
|
1145
|
+
rest. Every other setting is a scalar, so whole-value replacement IS
|
|
1146
|
+
per-key override for them; for a nested key, a plain dict.update()
|
|
1147
|
+
would let a project setting `buffer_tokens` silently discard the
|
|
1148
|
+
user's `strength` -- a per-key override everywhere else that becomes a
|
|
1149
|
+
wholesale reset here, purely because of the value's type.
|
|
1150
|
+
|
|
1151
|
+
Driven off _NESTED_SETTINGS rather than naming sections inline: §16
|
|
1152
|
+
added `tui` alongside `compaction`, and a second hardcoded section name
|
|
1153
|
+
here is exactly how the first one came to be missed.
|
|
1154
|
+
|
|
1155
|
+
`sources` MAPS EACH SETTING TO THE TIER THAT LAST WROTE IT, dotted for
|
|
1156
|
+
a nested key: `{"compaction.strength": "project"}`. TECHNICAL_DEBT 12,
|
|
1157
|
+
batch 111. It is built HERE, in the same pass as the merge, because
|
|
1158
|
+
this is the only moment the two tiers are still distinguishable -- the
|
|
1159
|
+
entry's own words for why provenance was missing were that it is "not
|
|
1160
|
+
omitted, it is destroyed by the merge", and a second function
|
|
1161
|
+
re-reading both files to work it out afterwards would be two
|
|
1162
|
+
traversals that must agree, which is the shape this module's own
|
|
1163
|
+
docstrings keep naming as how the first one drifts.
|
|
1164
|
+
|
|
1165
|
+
A value the user set and the project did not keeps `user`, which is
|
|
1166
|
+
the whole point of recording it per key rather than per file: one
|
|
1167
|
+
`settings.json` arriving with a cloned directory does not make every
|
|
1168
|
+
compaction value the project's.
|
|
1169
|
+
"""
|
|
1170
|
+
merged = user_settings()
|
|
1171
|
+
sources = _tier_of(merged, "user")
|
|
1172
|
+
if trusted:
|
|
1173
|
+
project = _read_settings_file(os.path.join(
|
|
1174
|
+
workspace_trust.venastine_dir(project_path), "settings.json"))
|
|
1175
|
+
deep = {
|
|
1176
|
+
section: {**merged.get(section, {}), **project.get(section, {})}
|
|
1177
|
+
for section in _NESTED_SETTINGS
|
|
1178
|
+
}
|
|
1179
|
+
merged.update(project)
|
|
1180
|
+
sources.update(_tier_of(project, "project"))
|
|
1181
|
+
for section, value in deep.items():
|
|
1182
|
+
if value:
|
|
1183
|
+
merged[section] = value
|
|
1184
|
+
return merged, sources
|
|
1185
|
+
|
|
1186
|
+
|
|
1187
|
+
def _tier_of(settings: dict, tier: str) -> dict:
|
|
1188
|
+
"""`{dotted key: tier}` for everything *settings* actually sets.
|
|
1189
|
+
|
|
1190
|
+
One level deep, for `_NESTED_SETTINGS`' reason and driven off the same
|
|
1191
|
+
table: a section is the only kind of value whose keys are settings in
|
|
1192
|
+
their own right, so a section is the only thing flattened. Its own
|
|
1193
|
+
name is recorded too -- a consumer that reads a whole section, as
|
|
1194
|
+
`tui` does, wants to know who supplied it.
|
|
1195
|
+
"""
|
|
1196
|
+
flat = {}
|
|
1197
|
+
for key, value in settings.items():
|
|
1198
|
+
flat[key] = tier
|
|
1199
|
+
if key in _NESTED_SETTINGS and isinstance(value, dict):
|
|
1200
|
+
for leaf in value:
|
|
1201
|
+
flat[f"{key}.{leaf}"] = tier
|
|
1202
|
+
return flat
|
|
1203
|
+
|
|
1204
|
+
|
|
1205
|
+
_state: Optional[dict] = None
|
|
1206
|
+
|
|
1207
|
+
|
|
1208
|
+
def initialize(project_path: str) -> None:
|
|
1209
|
+
"""Load everything once at startup. Untrusted project content is
|
|
1210
|
+
absent entirely -- not loaded-but-disabled (D17)."""
|
|
1211
|
+
global _state
|
|
1212
|
+
trusted = workspace_trust.is_trusted(project_path)
|
|
1213
|
+
if not trusted:
|
|
1214
|
+
logger.warning(
|
|
1215
|
+
"Project .venastine/ at %s is not trusted; project-level "
|
|
1216
|
+
"content (agents, skills, settings, AGENTS.md) will not load.",
|
|
1217
|
+
project_path,
|
|
1218
|
+
)
|
|
1219
|
+
settings, settings_tiers = _load_merged_settings(project_path, trusted)
|
|
1220
|
+
context = None
|
|
1221
|
+
if trusted:
|
|
1222
|
+
# §44: the project ROOT, not .venastine/. The hub is an ordinary
|
|
1223
|
+
# committed file now, so a project shared WITHOUT its configuration
|
|
1224
|
+
# is not a project with a dangling pointer -- which is the whole
|
|
1225
|
+
# reason it moved. workspace_trust still covers it: the file is in
|
|
1226
|
+
# content_files() and in the hash, because what reaches a system
|
|
1227
|
+
# prompt is what a grant has to be about.
|
|
1228
|
+
context_path = os.path.join(
|
|
1229
|
+
project_path, workspace_trust.PROJECT_CONTEXT_FILENAME)
|
|
1230
|
+
if os.path.exists(context_path):
|
|
1231
|
+
# Degrade, don't abort. UnicodeDecodeError is a ValueError, so
|
|
1232
|
+
# an unreadable AGENTS.md would reach main.load_project_config's
|
|
1233
|
+
# handler and SystemExit(1) EVERY invocation in this directory
|
|
1234
|
+
# -- including plain chat, which never reads the file. Every
|
|
1235
|
+
# other malformed content file in this module warns and skips.
|
|
1236
|
+
try:
|
|
1237
|
+
context = read_regular_text(context_path)
|
|
1238
|
+
except (OSError, UnicodeDecodeError) as e:
|
|
1239
|
+
logger.warning(
|
|
1240
|
+
"Could not read %s; continuing without project "
|
|
1241
|
+
"context: %s", context_path, e)
|
|
1242
|
+
_state = {
|
|
1243
|
+
"project_path": os.path.realpath(project_path),
|
|
1244
|
+
"trusted": trusted,
|
|
1245
|
+
"agents": _discover("agents", project_path, trusted),
|
|
1246
|
+
"skills": _discover("skills", project_path, trusted),
|
|
1247
|
+
"settings": settings,
|
|
1248
|
+
# Beside the values rather than folded into them, because twelve
|
|
1249
|
+
# call sites subscript that dict and `{key: (value, tier)}` breaks
|
|
1250
|
+
# every one (TECHNICAL_DEBT 12's first blocker, unchanged).
|
|
1251
|
+
"settings_sources": settings_tiers,
|
|
1252
|
+
"context": context,
|
|
1253
|
+
}
|
|
1254
|
+
# §32 A3. Before effective_compaction, because this one is about
|
|
1255
|
+
# THIS repository's own files rather than about the user's -- if
|
|
1256
|
+
# it fires, every other startup check is reporting on a build that
|
|
1257
|
+
# should not have shipped.
|
|
1258
|
+
assert_spawnable_declared(_state["agents"])
|
|
1259
|
+
# AFTER _state is assigned, because effective_compaction() reads
|
|
1260
|
+
# get_settings(). Called here so an incoherent compaction block is a
|
|
1261
|
+
# STARTUP error naming the file, rather than a ValueError from inside
|
|
1262
|
+
# a compaction three hours into a long thread -- which is the one
|
|
1263
|
+
# moment the user least wants to find out. §21's "reject at load time,
|
|
1264
|
+
# not incoherent trigger math later".
|
|
1265
|
+
effective_compaction(warn=True)
|
|
1266
|
+
|
|
1267
|
+
|
|
1268
|
+
def assert_spawnable_declared(agents: dict) -> None:
|
|
1269
|
+
"""Every HARNESS-tier agent must say whether it is spawnable.
|
|
1270
|
+
|
|
1271
|
+
ROADMAP_v2 §32 (A3). The same shape as
|
|
1272
|
+
tools.base.assert_grant_policy_declared and
|
|
1273
|
+
assert_budget_declared, and for the same stated reason: a field
|
|
1274
|
+
whose absence silently means something is a field nobody will
|
|
1275
|
+
remember to set. #69 exists because there was no field at all, so
|
|
1276
|
+
every agent inherited "spawnable" and four shipped ones were
|
|
1277
|
+
advertised as a delegation route that cannot carry them.
|
|
1278
|
+
|
|
1279
|
+
HARNESS TIER ONLY, and that asymmetry is the decision rather than
|
|
1280
|
+
an omission. These are files this project ships, so a missing
|
|
1281
|
+
declaration is a build error and CI is where it should surface.
|
|
1282
|
+
A user's or a project's agent gets the safe default instead --
|
|
1283
|
+
raising there would take down startup over somebody else's file,
|
|
1284
|
+
which is the trade _parse_md_file already refuses everywhere else
|
|
1285
|
+
in this module.
|
|
1286
|
+
"""
|
|
1287
|
+
undeclared = sorted(
|
|
1288
|
+
f"{a.name} ({a.path})" for a in agents.values()
|
|
1289
|
+
if a.tier == "harness" and a.spawnable is None)
|
|
1290
|
+
if undeclared:
|
|
1291
|
+
raise RuntimeError(
|
|
1292
|
+
"These harness agents do not declare `spawnable` in their "
|
|
1293
|
+
"frontmatter, so nothing can tell whether spawn_subagent "
|
|
1294
|
+
"is able to feed them (ROADMAP_v2 §32 A3): "
|
|
1295
|
+
+ ", ".join(undeclared))
|
|
1296
|
+
|
|
1297
|
+
|
|
1298
|
+
def reset() -> None:
|
|
1299
|
+
"""Clear the startup cache (tests)."""
|
|
1300
|
+
global _state
|
|
1301
|
+
_state = None
|
|
1302
|
+
|
|
1303
|
+
|
|
1304
|
+
def get_agents() -> dict:
|
|
1305
|
+
"""Empty before initialize(), matching every other getter here.
|
|
1306
|
+
|
|
1307
|
+
Uninitialized means "no discovery has run", which is materially the
|
|
1308
|
+
same state as "discovery ran and found nothing" for every consumer --
|
|
1309
|
+
both mean there are no agents to offer. get_agents/get_skills used to
|
|
1310
|
+
raise while get_settings/get_skill/skill_catalog_text/context_for_agent
|
|
1311
|
+
returned empty, so the answer to "is calling this before startup an
|
|
1312
|
+
error?" depended on which getter you happened to pick. Only main.py's
|
|
1313
|
+
__main__ calls initialize(); §16's TUI, §18 and §19 add more entry
|
|
1314
|
+
points, so one convention now beats six discoveries later.
|
|
1315
|
+
"""
|
|
1316
|
+
if _state is None:
|
|
1317
|
+
return {}
|
|
1318
|
+
return dict(_state["agents"])
|
|
1319
|
+
|
|
1320
|
+
|
|
1321
|
+
def get_skills() -> dict:
|
|
1322
|
+
"""Empty before initialize() -- see get_agents()."""
|
|
1323
|
+
if _state is None:
|
|
1324
|
+
return {}
|
|
1325
|
+
return dict(_state["skills"])
|
|
1326
|
+
|
|
1327
|
+
|
|
1328
|
+
def get_settings() -> dict:
|
|
1329
|
+
if _state is None:
|
|
1330
|
+
return {} # pre-init consumers fall through to config.yaml defaults
|
|
1331
|
+
return dict(_state["settings"])
|
|
1332
|
+
|
|
1333
|
+
|
|
1334
|
+
def settings_sources() -> dict:
|
|
1335
|
+
"""`{dotted setting: "user" | "project"}` for everything a settings
|
|
1336
|
+
file actually set. TECHNICAL_DEBT 12.
|
|
1337
|
+
|
|
1338
|
+
A setting absent here was not set by either file, which means the
|
|
1339
|
+
value in force came from `config.yaml` -- so the caller reads the
|
|
1340
|
+
absence rather than needing a third tier name in the mapping.
|
|
1341
|
+
|
|
1342
|
+
Empty pre-init for `get_settings()`'s reason, and the consequence is
|
|
1343
|
+
the same one: a caller that runs before `initialize()` is told nothing
|
|
1344
|
+
outranks the file, which is true of that moment.
|
|
1345
|
+
"""
|
|
1346
|
+
if _state is None:
|
|
1347
|
+
return {}
|
|
1348
|
+
return dict(_state["settings_sources"])
|
|
1349
|
+
|
|
1350
|
+
|
|
1351
|
+
def spend_cap() -> Optional[int]:
|
|
1352
|
+
"""The configured per-run spend ceiling (#4), or None for uncapped.
|
|
1353
|
+
|
|
1354
|
+
The single resolution point every path reaches through the loop
|
|
1355
|
+
wrappers' `_SPEND_UNSET` sentinel: chat turns, each research pass,
|
|
1356
|
+
/init, the reviewer and its retries. One number applies per _run()
|
|
1357
|
+
invocation -- a user turn, a pass, an init run -- because that is what
|
|
1358
|
+
a spend ceiling means; nothing here reasons about context size, which
|
|
1359
|
+
is thresholds()/context_limit()'s instrument (compaction has NEVER
|
|
1360
|
+
keyed off this figure).
|
|
1361
|
+
|
|
1362
|
+
None -- the default, with or without the key present -- runs uncapped:
|
|
1363
|
+
#4's whole point is that the harness's cumulative counter is a billing
|
|
1364
|
+
meter, and hard spend limits belong to the provider's dashboard.
|
|
1365
|
+
"""
|
|
1366
|
+
return get_settings().get("max_token_budget")
|
|
1367
|
+
|
|
1368
|
+
|
|
1369
|
+
def get_project_path() -> Optional[str]:
|
|
1370
|
+
"""The RESOLVED project path this session was initialized against.
|
|
1371
|
+
|
|
1372
|
+
ROADMAP_v2 §21b (M12/D25): a project-scoped memory is keyed to "the
|
|
1373
|
+
same resolved project path the workspace-trust store uses", so there
|
|
1374
|
+
has to be exactly one answer to "which project" across trust, config
|
|
1375
|
+
and memory. This is it -- the realpath already computed at
|
|
1376
|
+
initialize() time, not a fresh resolution that could differ.
|
|
1377
|
+
|
|
1378
|
+
None before initialize(), matching every other getter here. A caller
|
|
1379
|
+
that gets None has no project to scope to, which memories/manager.py
|
|
1380
|
+
treats as "global memories only" rather than as "show everything":
|
|
1381
|
+
guessing a project would surface another one's facts.
|
|
1382
|
+
"""
|
|
1383
|
+
if _state is None:
|
|
1384
|
+
return None
|
|
1385
|
+
return _state["project_path"]
|
|
1386
|
+
|
|
1387
|
+
|
|
1388
|
+
def get_skill(name: str) -> Optional[SkillDef]:
|
|
1389
|
+
if _state is None:
|
|
1390
|
+
return None
|
|
1391
|
+
return _state["skills"].get(name)
|
|
1392
|
+
|
|
1393
|
+
|
|
1394
|
+
def get_agent(name: str) -> Optional[AgentDef]:
|
|
1395
|
+
"""None before initialize() or for an unknown name -- see
|
|
1396
|
+
get_agents(). AgentManager (§18) is the consumer."""
|
|
1397
|
+
if _state is None:
|
|
1398
|
+
return None
|
|
1399
|
+
return _state["agents"].get(name)
|
|
1400
|
+
|
|
1401
|
+
|
|
1402
|
+
def context_for_agent(agent: Optional[AgentDef]) -> Optional[str]:
|
|
1403
|
+
"""AGENTS.md is opt-in per agent (use_project_context), so no agent
|
|
1404
|
+
(or an agent without the flag) never pays its token cost. Untrusted
|
|
1405
|
+
projects have context=None regardless."""
|
|
1406
|
+
if _state is None or agent is None or not agent.use_project_context:
|
|
1407
|
+
return None
|
|
1408
|
+
return _state["context"]
|
|
1409
|
+
|
|
1410
|
+
|
|
1411
|
+
def skill_catalog_text(active: Optional[list] = None) -> str:
|
|
1412
|
+
"""Frontmatter-only catalog for system prompt injection. Bodies are
|
|
1413
|
+
deliberately absent -- the model requests them via load_skill.
|
|
1414
|
+
Empty string when uninitialized or when no skills exist, so prompt
|
|
1415
|
+
assembly is a no-op append.
|
|
1416
|
+
|
|
1417
|
+
GROUPED BY CATEGORY (§19 K4). Uncategorised skills come first, then
|
|
1418
|
+
categories alphabetically, so a flat catalog reads exactly as it did
|
|
1419
|
+
before any category folder existed.
|
|
1420
|
+
|
|
1421
|
+
`active` names skills whose full body is already pinned into this
|
|
1422
|
+
prompt (§19 K1). They are MARKED rather than omitted: dropping them
|
|
1423
|
+
would make the catalog disagree with `/skill`'s listing, and leaving
|
|
1424
|
+
them unmarked invites the model to spend a turn calling load_skill for
|
|
1425
|
+
text it can already read.
|
|
1426
|
+
"""
|
|
1427
|
+
if _state is None or not _state["skills"]:
|
|
1428
|
+
return ""
|
|
1429
|
+
active_set = set(active or ())
|
|
1430
|
+
lines = [
|
|
1431
|
+
"## Available skills",
|
|
1432
|
+
"The skills below are available in this session. Only their "
|
|
1433
|
+
"summaries are listed here; call the load_skill tool with a "
|
|
1434
|
+
"skill name to view a skill's full instructions before "
|
|
1435
|
+
"following one.",
|
|
1436
|
+
]
|
|
1437
|
+
by_category: dict = {}
|
|
1438
|
+
for name in sorted(_state["skills"]):
|
|
1439
|
+
skill = _state["skills"][name]
|
|
1440
|
+
by_category.setdefault(skill.category, []).append(skill)
|
|
1441
|
+
# "" sorts before any real category name, which is the order wanted:
|
|
1442
|
+
# uncategorised first, then categories alphabetically.
|
|
1443
|
+
for category in sorted(by_category):
|
|
1444
|
+
if category:
|
|
1445
|
+
lines.append(f"### {category}")
|
|
1446
|
+
for skill in by_category[category]:
|
|
1447
|
+
suffix = (" [ACTIVE — full text below]"
|
|
1448
|
+
if skill.name in active_set else "")
|
|
1449
|
+
lines.append(f"- {skill.name}: {skill.description}{suffix}")
|
|
1450
|
+
return "\n".join(lines)
|
|
1451
|
+
|
|
1452
|
+
|
|
1453
|
+
def _catalog_entries(root: str, files: list, shadows=None) -> list:
|
|
1454
|
+
"""(kind, name, description, reach) for every agent/skill in the
|
|
1455
|
+
listing; `reach` is the lines saying where that description goes.
|
|
1456
|
+
|
|
1457
|
+
ROADMAP_v2 §32 A6, #131. The trust prompt's stated criterion is
|
|
1458
|
+
written in describe_project_content below -- the files shown
|
|
1459
|
+
verbatim are "the ones whose contents change what runs". An
|
|
1460
|
+
agent's and a skill's `description` meets it exactly: one `y` lets
|
|
1461
|
+
it into system prompts in this project, with no tool call and no
|
|
1462
|
+
further consent, and the reason they were left out was that nobody
|
|
1463
|
+
had noticed they qualify.
|
|
1464
|
+
|
|
1465
|
+
WHERE EACH ONE GOES, SINCE BATCH 215 (#259, #260, register items
|
|
1466
|
+
112 and 113, the owner's wording). The section used
|
|
1467
|
+
to say every description entered EVERY system prompt. A skill's is
|
|
1468
|
+
listed wherever load_skill is offered; an agent's only if it is
|
|
1469
|
+
spawnable, and the project tier's are not unless they say so (§32
|
|
1470
|
+
A3/A4); and a definition whose name a harness or user one already
|
|
1471
|
+
has never loads at all, D29's precedence being first-wins. Each is
|
|
1472
|
+
still LISTED -- the grant covers its bytes, and renaming the other
|
|
1473
|
+
makes it live with no new prompt -- but no longer described as
|
|
1474
|
+
reaching further than it does. `shadows` is `{kind: {name: tier}}`
|
|
1475
|
+
for the tiers above the project.
|
|
1476
|
+
|
|
1477
|
+
PARSED THROUGH _parse_md_file, so what is shown is the NORMALISED
|
|
1478
|
+
text -- the same value A5 will put in the prompt, not the raw
|
|
1479
|
+
frontmatter. That is the whole point of showing it: a summary
|
|
1480
|
+
displaying something other than what gets injected is a worse
|
|
1481
|
+
answer than showing nothing, because it invites trust in the
|
|
1482
|
+
wrong string. It also means the listing is bounded by
|
|
1483
|
+
construction, which is what makes printing it safe at all.
|
|
1484
|
+
|
|
1485
|
+
Runs on UNTRUSTED content by definition, so a file that will not
|
|
1486
|
+
parse is reported as unreadable rather than skipped in silence:
|
|
1487
|
+
'this file is here and I cannot tell you what it says' is a fact
|
|
1488
|
+
the person answering the prompt needs.
|
|
1489
|
+
"""
|
|
1490
|
+
shadows = shadows or {}
|
|
1491
|
+
out = []
|
|
1492
|
+
for rel in files:
|
|
1493
|
+
# §44: the listing is PROJECT-relative now, so the .venastine/
|
|
1494
|
+
# prefix is stripped before matching. Matching the raw string
|
|
1495
|
+
# would silently describe nothing -- the trust prompt would still
|
|
1496
|
+
# render, still list the files, and simply stop saying what any of
|
|
1497
|
+
# them would tell the model, which is the half #131 added.
|
|
1498
|
+
if not rel.startswith(".venastine/"):
|
|
1499
|
+
continue
|
|
1500
|
+
parts = rel[len(".venastine/"):].split("/")
|
|
1501
|
+
if len(parts) < 2 or parts[0] not in ("agents", "skills"):
|
|
1502
|
+
continue
|
|
1503
|
+
if not rel.lower().endswith(".md"):
|
|
1504
|
+
continue
|
|
1505
|
+
kind = parts[0]
|
|
1506
|
+
defn = _parse_md_file(os.path.join(root, *parts), kind,
|
|
1507
|
+
"project")
|
|
1508
|
+
if defn is None:
|
|
1509
|
+
out.append((kind, "/".join(parts),
|
|
1510
|
+
"(could not be read -- see the log)", []))
|
|
1511
|
+
continue
|
|
1512
|
+
out.append((kind, defn.name, defn.description,
|
|
1513
|
+
_reach(kind, defn, shadows.get(kind, {}))))
|
|
1514
|
+
return out
|
|
1515
|
+
|
|
1516
|
+
|
|
1517
|
+
def _reach(kind: str, defn, shadowed_by: dict) -> list:
|
|
1518
|
+
"""Where a project definition's description goes, as the lines the
|
|
1519
|
+
trust prompt prints under it (#259, #260)."""
|
|
1520
|
+
tier = shadowed_by.get(defn.name)
|
|
1521
|
+
if tier is not None:
|
|
1522
|
+
owner = ("the harness's own" if tier == "harness"
|
|
1523
|
+
else "your own (user configuration)")
|
|
1524
|
+
return [f'shadowed by {owner} "{defn.name}", which is what loads;',
|
|
1525
|
+
"this file is used only if that one goes away"]
|
|
1526
|
+
if kind == "skills":
|
|
1527
|
+
return ["listed wherever load_skill is offered"]
|
|
1528
|
+
if defn.spawnable:
|
|
1529
|
+
return ["spawnable: listed wherever spawn_subagent is offered"]
|
|
1530
|
+
return ["not spawnable: in no system prompt; /agent can still pick it"]
|
|
1531
|
+
|
|
1532
|
+
|
|
1533
|
+
def _shadowing_names(project_path: str) -> dict:
|
|
1534
|
+
"""`{kind: {name: tier}}` for the tiers that win over the project's,
|
|
1535
|
+
harness first, in `_discover`'s own order and with its own parser, so
|
|
1536
|
+
the trust prompt and the loader cannot disagree about which loads."""
|
|
1537
|
+
out: dict = {}
|
|
1538
|
+
for kind in ("agents", "skills"):
|
|
1539
|
+
names: dict = {}
|
|
1540
|
+
for tier, directory in _tier_dirs(kind, project_path, False):
|
|
1541
|
+
if not os.path.isdir(directory):
|
|
1542
|
+
continue
|
|
1543
|
+
for path, category in _md_files(directory,
|
|
1544
|
+
recursive=(kind == "skills")):
|
|
1545
|
+
defn = _parse_md_file(path, kind, tier, category)
|
|
1546
|
+
if defn is not None:
|
|
1547
|
+
names.setdefault(defn.name, tier)
|
|
1548
|
+
out[kind] = names
|
|
1549
|
+
return out
|
|
1550
|
+
|
|
1551
|
+
|
|
1552
|
+
#: How much of any ONE section of the trust prompt is printed. Enough to
|
|
1553
|
+
#: recognise what a file is for, short enough that the prompt stays
|
|
1554
|
+
#: readable -- the whole thing is what a grant covers, not what it prints.
|
|
1555
|
+
_SECTION_LINES = 60
|
|
1556
|
+
_SECTION_CHARS = 6000
|
|
1557
|
+
|
|
1558
|
+
|
|
1559
|
+
def _bounded(source: list, prefix: str, what: str) -> list:
|
|
1560
|
+
"""`source`'s lines, prefixed and cut to BOTH bounds, with a marker
|
|
1561
|
+
naming what was withheld.
|
|
1562
|
+
|
|
1563
|
+
THE CONSENT PROMPT IS THE ONE PLACE A PROJECT'S CONTENT IS SHOWN TO
|
|
1564
|
+
SOMEONE WHO HAS NOT TRUSTED IT YET (D17), so how long it is must not
|
|
1565
|
+
be the project's choice. Every section goes through here (A54) --
|
|
1566
|
+
the file listing, the hub, the two JSON files and the catalog --
|
|
1567
|
+
because capping some of them leaves the prompt exactly as floodable
|
|
1568
|
+
and the cap stops being worth having.
|
|
1569
|
+
|
|
1570
|
+
TWO BOUNDS, NOT ONE, AND THE PAIR IS THE POINT. §44 gave the hub a
|
|
1571
|
+
LINE bound alone, and measured against it: a 2 MB AGENTS.md written
|
|
1572
|
+
as one line printed all 2 MB in four lines, and twenty 100 KB lines
|
|
1573
|
+
printed 2 MB from inside the cap. A char bound alone fails the other
|
|
1574
|
+
way -- 30,000 short lines are 2.4 MB of prompt. The two JSON files
|
|
1575
|
+
had neither bound: 2 x 2.2 MB rendered as 4.5 MB in 30 lines, with
|
|
1576
|
+
the `mcp.json` command the section exists to show sitting at
|
|
1577
|
+
99.9994% of the way in.
|
|
1578
|
+
|
|
1579
|
+
THE NUMBERS ARE MEASURED AGAINST THE LARGEST LEGITIMATE SHAPES
|
|
1580
|
+
rather than picked. A five-server pretty-printed `mcp.json` is 54
|
|
1581
|
+
lines and 923 characters; a realistic `settings.json` is 14 and 264;
|
|
1582
|
+
this project's own AGENTS.md is 4,033 characters in its first 60
|
|
1583
|
+
lines. All three render whole, which is what keeps the marker
|
|
1584
|
+
meaningful on the files where it does appear.
|
|
1585
|
+
|
|
1586
|
+
THE MARKER IS NOT A COURTESY. A cut nobody is told about turns a
|
|
1587
|
+
bounded prompt into one that misrepresents what it covers, and the
|
|
1588
|
+
grant is over the whole file either way -- which is why the sentence
|
|
1589
|
+
says so rather than stopping at an ellipsis.
|
|
1590
|
+
"""
|
|
1591
|
+
shown: list = []
|
|
1592
|
+
used = 0
|
|
1593
|
+
hidden_chars = 0
|
|
1594
|
+
for line in source:
|
|
1595
|
+
if len(shown) >= _SECTION_LINES or used >= _SECTION_CHARS:
|
|
1596
|
+
hidden_chars += len(line)
|
|
1597
|
+
continue
|
|
1598
|
+
room = _SECTION_CHARS - used
|
|
1599
|
+
if len(line) > room:
|
|
1600
|
+
# CUT WITHIN THE LINE rather than dropping it whole. A file
|
|
1601
|
+
# flooded on one enormous line has exactly one line, and it
|
|
1602
|
+
# is the line the section exists to show.
|
|
1603
|
+
shown.append(line[:room] + "…")
|
|
1604
|
+
used += room
|
|
1605
|
+
hidden_chars += len(line) - room
|
|
1606
|
+
continue
|
|
1607
|
+
shown.append(line)
|
|
1608
|
+
used += len(line)
|
|
1609
|
+
out = [prefix + line for line in shown]
|
|
1610
|
+
hidden_lines = len(source) - len(shown)
|
|
1611
|
+
if hidden_lines or hidden_chars:
|
|
1612
|
+
# Always plural. The grammar is a blemish; a marker whose wording
|
|
1613
|
+
# depends on the count is one a test can match by luck.
|
|
1614
|
+
out.append(
|
|
1615
|
+
f"{prefix}… {hidden_lines} more lines and {hidden_chars} more "
|
|
1616
|
+
f"characters of {what} are not shown here. Trust covers the "
|
|
1617
|
+
f"whole file, not the part printed.")
|
|
1618
|
+
return out
|
|
1619
|
+
|
|
1620
|
+
|
|
1621
|
+
def describe_project_content(project_path: str) -> str:
|
|
1622
|
+
"""Human-readable summary shown in the trust prompt, so approving
|
|
1623
|
+
trust is an informed decision -- a project's settings can choose the
|
|
1624
|
+
provider and multiply pipeline cost by turning on ensemble_mode over
|
|
1625
|
+
config.yaml's roster (#213: ensemble_n is ignored), and an agent's
|
|
1626
|
+
or a skill's description can reach the system prompt (#131) --
|
|
1627
|
+
where, each entry says (#259, #260).
|
|
1628
|
+
|
|
1629
|
+
EVERY SECTION IS BOUNDED, through `_bounded` above (A54, batch 155).
|
|
1630
|
+
Two of the four used to be pasted whole and the hub was bounded on
|
|
1631
|
+
one axis of two, which let a cloned project decide how long its own
|
|
1632
|
+
consent screen was."""
|
|
1633
|
+
root = workspace_trust.venastine_dir(project_path)
|
|
1634
|
+
files = workspace_trust.content_files(project_path)
|
|
1635
|
+
lines = [f"Project content ({os.path.realpath(project_path)}):"]
|
|
1636
|
+
# THE LISTING ITSELF IS BOUNDED TOO. `content_files()` is unchanged
|
|
1637
|
+
# and still returns every covered path -- the hash and
|
|
1638
|
+
# workspace_trust's "the loader must read no file the trust listing
|
|
1639
|
+
# omits" invariant are about that function, not about this rendering.
|
|
1640
|
+
# What is bounded is what the PROMPT prints, and 20,000 padding files
|
|
1641
|
+
# under .venastine/ were 20,001 lines of it.
|
|
1642
|
+
lines += _bounded(files, " - ", "the file listing")
|
|
1643
|
+
|
|
1644
|
+
# §44. The hub is shown VERBATIM, and it is the strongest instance of
|
|
1645
|
+
# this function's own stated criterion -- "the ones whose contents
|
|
1646
|
+
# change what runs". settings.json picks a provider and mcp.json names
|
|
1647
|
+
# a command; AGENTS.md is prose that goes into the system prompt of
|
|
1648
|
+
# every opted-in agent, which is the plainest form of "tells the model
|
|
1649
|
+
# what to do" this prompt can be asked to cover. Bounded to the first
|
|
1650
|
+
# few lines because a project's context document is not small and an
|
|
1651
|
+
# unbounded paste is a prompt nobody reads to the end of.
|
|
1652
|
+
hub = os.path.join(os.path.realpath(project_path),
|
|
1653
|
+
workspace_trust.PROJECT_CONTEXT_FILENAME)
|
|
1654
|
+
if os.path.exists(hub):
|
|
1655
|
+
# Every read in this function is of content nobody has consented
|
|
1656
|
+
# to yet, so each is regular-files-only (batch 219, #214): a hub
|
|
1657
|
+
# linked to /dev/zero would otherwise be read until memory ran out.
|
|
1658
|
+
try:
|
|
1659
|
+
body = read_regular_text(hub)
|
|
1660
|
+
except (OSError, UnicodeDecodeError) as e:
|
|
1661
|
+
lines.append(
|
|
1662
|
+
f"{workspace_trust.PROJECT_CONTEXT_FILENAME}: could not be "
|
|
1663
|
+
f"read ({e})")
|
|
1664
|
+
else:
|
|
1665
|
+
lines.append(
|
|
1666
|
+
f"{workspace_trust.PROJECT_CONTEXT_FILENAME} would enter "
|
|
1667
|
+
f"every opted-in agent's system prompt; it begins:")
|
|
1668
|
+
lines += _bounded(
|
|
1669
|
+
body.splitlines(), " | ",
|
|
1670
|
+
workspace_trust.PROJECT_CONTEXT_FILENAME)
|
|
1671
|
+
|
|
1672
|
+
# settings.json and mcp.json are shown VERBATIM, the rest by name.
|
|
1673
|
+
# These two are the ones whose contents change what runs: settings can
|
|
1674
|
+
# pick the provider and multiply pipeline cost by turning on
|
|
1675
|
+
# ensemble_mode (ensemble_n is ignored; the roster is config.yaml's), and
|
|
1676
|
+
# mcp.json names a local command to execute. Approving trust without
|
|
1677
|
+
# seeing that command is not an informed decision, which is the whole
|
|
1678
|
+
# reason §17 moved project MCP config under .venastine/ instead of
|
|
1679
|
+
# leaving it at the project root outside the trust boundary.
|
|
1680
|
+
for fname, label in (("settings.json", "settings.json contents:"),
|
|
1681
|
+
("mcp.json", "mcp.json contents (these commands would run):")):
|
|
1682
|
+
path = os.path.join(root, fname)
|
|
1683
|
+
if not os.path.exists(path):
|
|
1684
|
+
continue
|
|
1685
|
+
try:
|
|
1686
|
+
body = read_regular_text(path)
|
|
1687
|
+
except (OSError, UnicodeDecodeError) as e:
|
|
1688
|
+
# This function runs ONLY for untrusted projects, i.e. content
|
|
1689
|
+
# that is adversarial by definition. UnicodeDecodeError is a
|
|
1690
|
+
# ValueError, not an OSError, so without it here a UTF-16
|
|
1691
|
+
# settings.json kills startup before the trust prompt renders
|
|
1692
|
+
# -- including the non-TTY notice and the --trust-project path,
|
|
1693
|
+
# making trust ungrantable for that project.
|
|
1694
|
+
lines.append(f"{fname}: could not be read ({e})")
|
|
1695
|
+
continue
|
|
1696
|
+
lines.append(label)
|
|
1697
|
+
lines += _bounded(body.splitlines(), " | ", fname)
|
|
1698
|
+
|
|
1699
|
+
# §32 A6. Shown AFTER the two config files rather than beside the
|
|
1700
|
+
# file list, because the list answers "what is here" and this
|
|
1701
|
+
# answers "what would it say to the model" -- the same order the
|
|
1702
|
+
# two questions are asked in.
|
|
1703
|
+
entries = _catalog_entries(root, files,
|
|
1704
|
+
_shadowing_names(project_path))
|
|
1705
|
+
if entries:
|
|
1706
|
+
lines.append("what each project agent and skill would tell the "
|
|
1707
|
+
"model:")
|
|
1708
|
+
# A5 bounds each description at `max_catalog_text_chars` and
|
|
1709
|
+
# nothing bounded the NUMBER of them: 3,000 project agents were
|
|
1710
|
+
# 1 MB of prompt in 6,002 lines, every entry individually legal.
|
|
1711
|
+
rows = []
|
|
1712
|
+
for kind, name, description, reach in entries:
|
|
1713
|
+
rows.append(f"{kind[:-1]} {name}: {description}")
|
|
1714
|
+
rows += [f" {line}" for line in reach]
|
|
1715
|
+
lines += _bounded(rows, " | ", "the agent and skill descriptions")
|
|
1716
|
+
return "\n".join(lines)
|
|
1717
|
+
|
|
1718
|
+
|
|
1719
|
+
# ---------------------------------------------------------------------------
|
|
1720
|
+
# ---- §45 (SQ6): the scoring knobs -----------------------------------------
|
|
1721
|
+
# ---------------------------------------------------------------------------
|
|
1722
|
+
#
|
|
1723
|
+
# WHY THESE TWO SECTIONS RESOLVE DIFFERENTLY FROM COMPACTION, and it is a
|
|
1724
|
+
# deliberate amendment rather than an oversight. `effective_compaction`
|
|
1725
|
+
# RAISES on an incoherent value, because a bad trigger burns real model
|
|
1726
|
+
# calls on every turn of every conversation. A bad scoring weight costs
|
|
1727
|
+
# the shape of one number in one report -- and refusing to start a
|
|
1728
|
+
# ten-pass run over a mistyped weight is the worse failure. So a known key
|
|
1729
|
+
# with an out-of-range value FALLS BACK to its own default and warns; only
|
|
1730
|
+
# that one value is discarded, never the whole section.
|
|
1731
|
+
#
|
|
1732
|
+
# Unknown KEYS still raise, unchanged. That is a schema question rather
|
|
1733
|
+
# than a range one, and §14's loud-rejection rule is what makes adding a
|
|
1734
|
+
# setting a deliberate act.
|
|
1735
|
+
#
|
|
1736
|
+
# TOP-LEVEL SECTIONS, not `research.confidence`. `_NESTED_SETTINGS`
|
|
1737
|
+
# supports exactly one level of nesting and both the validator and the
|
|
1738
|
+
# cross-tier merge iterate it; deepening that merge is the code path
|
|
1739
|
+
# review finding F2 already broke once, and these need no new mechanism to
|
|
1740
|
+
# sit beside `compaction` and `tui`.
|
|
1741
|
+
|
|
1742
|
+
# key -> (module attribute it defaults from, validator)
|
|
1743
|
+
#
|
|
1744
|
+
# The defaults live BESIDE THE FORMULA rather than in config.yaml: the
|
|
1745
|
+
# tunable-weights block at the top of confidence_scoring.py documents what
|
|
1746
|
+
# each number does and why it is that number, and a second copy here is a
|
|
1747
|
+
# second thing to keep in step. Imported lazily inside the resolver, the
|
|
1748
|
+
# same shape effective_compaction uses for `config_schema`.
|
|
1749
|
+
_UNIT = ("a number between 0 and 1", lambda v: isinstance(v, NUMBER_TYPES)
|
|
1750
|
+
and not isinstance(v, bool) and 0.0 <= v <= 1.0)
|
|
1751
|
+
_POSITIVE_INT = ("a positive whole number",
|
|
1752
|
+
lambda v: isinstance(v, int) and not isinstance(v, bool) and v > 0)
|
|
1753
|
+
_NON_NEGATIVE_INT = ("a whole number of 0 or more",
|
|
1754
|
+
lambda v: isinstance(v, int) and not isinstance(v, bool) and v >= 0)
|
|
1755
|
+
|
|
1756
|
+
_CONFIDENCE_KNOBS = {
|
|
1757
|
+
"grounding_weight_factor": _UNIT,
|
|
1758
|
+
"critic_weight_factor": _UNIT,
|
|
1759
|
+
"assumption_flag_penalty": _UNIT,
|
|
1760
|
+
"non_factual_score_cap": _UNIT,
|
|
1761
|
+
"source_quality_floor": _UNIT,
|
|
1762
|
+
"authority_full_credit": _UNIT,
|
|
1763
|
+
}
|
|
1764
|
+
|
|
1765
|
+
_SOURCE_SCORING_KNOBS = {
|
|
1766
|
+
"similarity_floor": _UNIT,
|
|
1767
|
+
"similarity_ceiling": _UNIT,
|
|
1768
|
+
"authority_adjustment_cap": _UNIT,
|
|
1769
|
+
"window_chars": _POSITIVE_INT,
|
|
1770
|
+
"max_windows": _POSITIVE_INT,
|
|
1771
|
+
"claim_min_chars": _NON_NEGATIVE_INT,
|
|
1772
|
+
"top_k": _POSITIVE_INT,
|
|
1773
|
+
"venue_weight": _UNIT,
|
|
1774
|
+
"citation_weight": _UNIT,
|
|
1775
|
+
"author_weight": _UNIT,
|
|
1776
|
+
"h_saturation": _POSITIVE_INT,
|
|
1777
|
+
"min_cohort_size": _POSITIVE_INT,
|
|
1778
|
+
"min_citation_age_days": _NON_NEGATIVE_INT,
|
|
1779
|
+
}
|
|
1780
|
+
|
|
1781
|
+
|
|
1782
|
+
def _confidence_defaults() -> dict:
|
|
1783
|
+
from core.reasoning import confidence_scoring as cs
|
|
1784
|
+
|
|
1785
|
+
return {
|
|
1786
|
+
"grounding_weight_factor": cs.GROUNDING_WEIGHT_FACTOR,
|
|
1787
|
+
"critic_weight_factor": cs.CRITIC_WEIGHT_FACTOR,
|
|
1788
|
+
"assumption_flag_penalty": cs.ASSUMPTION_FLAG_PENALTY,
|
|
1789
|
+
"non_factual_score_cap": cs.NON_FACTUAL_SCORE_CAP,
|
|
1790
|
+
"source_quality_floor": cs.SOURCE_QUALITY_FLOOR,
|
|
1791
|
+
"authority_full_credit": cs.AUTHORITY_FULL_CREDIT,
|
|
1792
|
+
"grounding_weights": dict(cs.GROUNDING_WEIGHTS),
|
|
1793
|
+
"tier_thresholds": {name: value for value, name in cs.TIER_THRESHOLDS},
|
|
1794
|
+
}
|
|
1795
|
+
|
|
1796
|
+
|
|
1797
|
+
def _source_scoring_defaults() -> dict:
|
|
1798
|
+
from core.reasoning import source_scoring as ss
|
|
1799
|
+
|
|
1800
|
+
cfg = config_schema.current()
|
|
1801
|
+
return {
|
|
1802
|
+
"similarity_floor": None, # None means "use the per-model table"
|
|
1803
|
+
"similarity_ceiling": None,
|
|
1804
|
+
"authority_adjustment_cap": cfg.authority_adjustment_cap,
|
|
1805
|
+
"window_chars": ss.WINDOW_CHARS,
|
|
1806
|
+
"max_windows": ss.MAX_WINDOWS,
|
|
1807
|
+
"claim_min_chars": ss.CLAIM_MIN_CHARS,
|
|
1808
|
+
"top_k": ss.TOP_K,
|
|
1809
|
+
"venue_weight": cfg.scholar_venue_weight,
|
|
1810
|
+
"citation_weight": cfg.scholar_citation_weight,
|
|
1811
|
+
"author_weight": cfg.scholar_author_weight,
|
|
1812
|
+
"h_saturation": cfg.scholar_h_saturation,
|
|
1813
|
+
"min_cohort_size": cfg.scholar_min_cohort_size,
|
|
1814
|
+
"min_citation_age_days": cfg.scholar_min_citation_age_days,
|
|
1815
|
+
"domain_overrides": {},
|
|
1816
|
+
}
|
|
1817
|
+
|
|
1818
|
+
|
|
1819
|
+
def _apply_scalar_knobs(values, supplied, knobs, section, warn):
|
|
1820
|
+
"""Overlay `supplied` onto `values`, dropping what does not validate.
|
|
1821
|
+
|
|
1822
|
+
PER KEY, not per section. One mistyped weight must not discard the
|
|
1823
|
+
nine beside it that were right -- a user retuning six numbers and
|
|
1824
|
+
fat-fingering the seventh should get six retuned numbers and one
|
|
1825
|
+
sentence about the seventh, not silently get none of them.
|
|
1826
|
+
"""
|
|
1827
|
+
for key, value in supplied.items():
|
|
1828
|
+
if key not in knobs:
|
|
1829
|
+
continue
|
|
1830
|
+
description, is_valid = knobs[key]
|
|
1831
|
+
if is_valid(value):
|
|
1832
|
+
values[key] = value
|
|
1833
|
+
elif warn:
|
|
1834
|
+
logger.warning(
|
|
1835
|
+
"%s.%s must be %s, got %r -- using the default %r for this "
|
|
1836
|
+
"run.", section, key, description, value, values[key])
|
|
1837
|
+
|
|
1838
|
+
|
|
1839
|
+
def _apply_grounding_weights(values, supplied, warn):
|
|
1840
|
+
"""`grounding_weights` -- a dict whose KEYS are a closed vocabulary."""
|
|
1841
|
+
if not isinstance(supplied, dict):
|
|
1842
|
+
if warn and supplied is not None:
|
|
1843
|
+
logger.warning("confidence.grounding_weights must be an object, "
|
|
1844
|
+
"got %r -- using the defaults for this run.",
|
|
1845
|
+
type(supplied).__name__)
|
|
1846
|
+
return
|
|
1847
|
+
merged = dict(values["grounding_weights"])
|
|
1848
|
+
for status, weight in supplied.items():
|
|
1849
|
+
if status not in GROUNDING_STATUSES:
|
|
1850
|
+
if warn:
|
|
1851
|
+
logger.warning(
|
|
1852
|
+
"confidence.grounding_weights has an unknown status %r "
|
|
1853
|
+
"-- expected one of %s. Ignored for this run.",
|
|
1854
|
+
status, ", ".join(sorted(GROUNDING_STATUSES)))
|
|
1855
|
+
continue
|
|
1856
|
+
if isinstance(weight, NUMBER_TYPES) and not isinstance(weight, bool) \
|
|
1857
|
+
and 0.0 <= weight <= 1.0:
|
|
1858
|
+
merged[status] = weight
|
|
1859
|
+
elif warn:
|
|
1860
|
+
logger.warning(
|
|
1861
|
+
"confidence.grounding_weights[%r] must be a number between 0 "
|
|
1862
|
+
"and 1, got %r -- using the default for this run.",
|
|
1863
|
+
status, weight)
|
|
1864
|
+
values["grounding_weights"] = merged
|
|
1865
|
+
|
|
1866
|
+
|
|
1867
|
+
def _apply_tier_thresholds(values, supplied, warn):
|
|
1868
|
+
"""`tier_thresholds` -- three numbers that must STAY ORDERED.
|
|
1869
|
+
|
|
1870
|
+
The relational check `effective_compaction` makes for warning_margin,
|
|
1871
|
+
applied to the one relationship here whose breakage is not
|
|
1872
|
+
self-correcting: thresholds out of order make a tier unreachable, so a
|
|
1873
|
+
claim scoring 0.9 could be published LOW while the table a reader
|
|
1874
|
+
consults says HIGH starts at 0.8. Out of order, the whole table falls
|
|
1875
|
+
back rather than a single value -- unlike the scalars above, because
|
|
1876
|
+
these three are only meaningful together.
|
|
1877
|
+
"""
|
|
1878
|
+
if not isinstance(supplied, dict):
|
|
1879
|
+
if warn and supplied is not None:
|
|
1880
|
+
logger.warning("confidence.tier_thresholds must be an object, "
|
|
1881
|
+
"got %r -- using the defaults for this run.",
|
|
1882
|
+
type(supplied).__name__)
|
|
1883
|
+
return
|
|
1884
|
+
merged = dict(values["tier_thresholds"])
|
|
1885
|
+
for name, value in supplied.items():
|
|
1886
|
+
if name not in merged:
|
|
1887
|
+
if warn:
|
|
1888
|
+
logger.warning(
|
|
1889
|
+
"confidence.tier_thresholds has an unknown tier %r -- "
|
|
1890
|
+
"expected one of %s. Ignored for this run.",
|
|
1891
|
+
name, ", ".join(merged))
|
|
1892
|
+
continue
|
|
1893
|
+
if isinstance(value, NUMBER_TYPES) and not isinstance(value, bool) \
|
|
1894
|
+
and 0.0 <= value <= 1.0:
|
|
1895
|
+
merged[name] = value
|
|
1896
|
+
elif warn:
|
|
1897
|
+
logger.warning(
|
|
1898
|
+
"confidence.tier_thresholds[%r] must be a number between 0 "
|
|
1899
|
+
"and 1, got %r -- using the defaults for this run.",
|
|
1900
|
+
name, value)
|
|
1901
|
+
|
|
1902
|
+
ordered = [merged.get(name) for name in ("HIGH", "MEDIUM", "LOW")]
|
|
1903
|
+
if not (ordered[0] > ordered[1] > ordered[2]):
|
|
1904
|
+
if warn:
|
|
1905
|
+
logger.warning(
|
|
1906
|
+
"confidence.tier_thresholds must descend HIGH > MEDIUM > LOW, "
|
|
1907
|
+
"got %s -- using the defaults for this run. Out of order, a "
|
|
1908
|
+
"tier becomes unreachable and a published tier stops matching "
|
|
1909
|
+
"the table that defines it.", merged)
|
|
1910
|
+
return
|
|
1911
|
+
values["tier_thresholds"] = merged
|
|
1912
|
+
|
|
1913
|
+
|
|
1914
|
+
def effective_confidence(warn: bool = False) -> dict:
|
|
1915
|
+
"""Pass 5's constants, as they are actually in force.
|
|
1916
|
+
|
|
1917
|
+
config default (from confidence_scoring's own tunable block) -> user
|
|
1918
|
+
settings.json -> trusted project settings.json. Nearest wins, the
|
|
1919
|
+
precedence the rest of the config system uses.
|
|
1920
|
+
|
|
1921
|
+
RESOLVED ONCE PER RUN by the caller, never per claim: score_claim is a
|
|
1922
|
+
pure function and stays one, the warnings are said once rather than
|
|
1923
|
+
once per claim (`effective_compaction`'s `warn=True` gating, for its
|
|
1924
|
+
reason), and a run's numbers cannot change halfway through.
|
|
1925
|
+
"""
|
|
1926
|
+
values = _confidence_defaults()
|
|
1927
|
+
supplied = get_settings().get("confidence") or {}
|
|
1928
|
+
if warn and "disagreement_penalty_factor" in supplied:
|
|
1929
|
+
logger.warning(
|
|
1930
|
+
"Ignoring confidence.disagreement_penalty_factor: a claim most "
|
|
1931
|
+
"ensemble candidates did not assert is sent to Pass 6a/6b to be "
|
|
1932
|
+
"checked now, and nothing is subtracted for it (ROADMAP_v3 §90). "
|
|
1933
|
+
"Remove the key from settings.json.")
|
|
1934
|
+
_apply_scalar_knobs(values, supplied, _CONFIDENCE_KNOBS, "confidence", warn)
|
|
1935
|
+
_apply_grounding_weights(values, supplied.get("grounding_weights"), warn)
|
|
1936
|
+
_apply_tier_thresholds(values, supplied.get("tier_thresholds"), warn)
|
|
1937
|
+
return values
|
|
1938
|
+
|
|
1939
|
+
|
|
1940
|
+
def effective_source_scoring(warn: bool = False) -> dict:
|
|
1941
|
+
"""§45's scoring knobs, as they are actually in force.
|
|
1942
|
+
|
|
1943
|
+
`similarity_floor` / `similarity_ceiling` default to None, meaning
|
|
1944
|
+
"use the per-model calibration table". A user who sets them is saying
|
|
1945
|
+
their embedder's band is not what the table thinks, which outranks a
|
|
1946
|
+
table that cannot know about a model shipped after this release --
|
|
1947
|
+
`model_windows`' inversion, for its reason.
|
|
1948
|
+
"""
|
|
1949
|
+
values = _source_scoring_defaults()
|
|
1950
|
+
supplied = get_settings().get("source_scoring") or {}
|
|
1951
|
+
_apply_scalar_knobs(values, supplied, _SOURCE_SCORING_KNOBS,
|
|
1952
|
+
"source_scoring", warn)
|
|
1953
|
+
|
|
1954
|
+
overrides = supplied.get("domain_overrides")
|
|
1955
|
+
if isinstance(overrides, dict):
|
|
1956
|
+
kept = {}
|
|
1957
|
+
for host, score in overrides.items():
|
|
1958
|
+
if isinstance(host, str) and host and \
|
|
1959
|
+
isinstance(score, NUMBER_TYPES) and \
|
|
1960
|
+
not isinstance(score, bool) and 0.0 <= score <= 1.0:
|
|
1961
|
+
kept[host.lower().lstrip(".")] = float(score)
|
|
1962
|
+
elif warn:
|
|
1963
|
+
logger.warning(
|
|
1964
|
+
"source_scoring.domain_overrides[%r] must be a number "
|
|
1965
|
+
"between 0 and 1, got %r -- ignored for this run.",
|
|
1966
|
+
host, score)
|
|
1967
|
+
values["domain_overrides"] = kept
|
|
1968
|
+
elif overrides is not None and warn:
|
|
1969
|
+
logger.warning("source_scoring.domain_overrides must be an object, "
|
|
1970
|
+
"got %r -- ignored for this run.",
|
|
1971
|
+
type(overrides).__name__)
|
|
1972
|
+
|
|
1973
|
+
floor = values["similarity_floor"]
|
|
1974
|
+
ceiling = values["similarity_ceiling"]
|
|
1975
|
+
if floor is not None and ceiling is not None and floor >= ceiling:
|
|
1976
|
+
if warn:
|
|
1977
|
+
logger.warning(
|
|
1978
|
+
"source_scoring.similarity_floor (%s) must be below "
|
|
1979
|
+
"similarity_ceiling (%s) -- falling back to this embedder's "
|
|
1980
|
+
"own calibration for this run.", floor, ceiling)
|
|
1981
|
+
values["similarity_floor"] = values["similarity_ceiling"] = None
|
|
1982
|
+
return values
|