@event4u/agent-config 2.18.0 → 2.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agent-src/commands/agent-status.md +29 -0
- package/.agent-src/commands/onboard.md +221 -81
- package/.agent-src/commands/refine-ticket.md +3 -0
- package/.agent-src/packs/README.md +49 -0
- package/.agent-src/packs/agency-delivery.yml +63 -0
- package/.agent-src/packs/content-engine.yml +53 -0
- package/.agent-src/packs/founder-mvp.yml +51 -0
- package/.agent-src/personas/README.md +8 -0
- package/.agent-src/presets/README.md +26 -0
- package/.agent-src/presets/balanced.yml +34 -0
- package/.agent-src/presets/fast.yml +31 -0
- package/.agent-src/presets/strict.yml +38 -0
- package/.agent-src/profiles/README.md +29 -0
- package/.agent-src/profiles/agency.yml +27 -0
- package/.agent-src/profiles/content_creator.yml +25 -0
- package/.agent-src/profiles/developer.yml +26 -0
- package/.agent-src/profiles/finance.yml +24 -0
- package/.agent-src/profiles/founder.yml +25 -0
- package/.agent-src/profiles/ops.yml +25 -0
- package/.agent-src/rules/no-cheap-questions.md +25 -17
- package/.agent-src/skills/adr-create/SKILL.md +78 -68
- package/.agent-src/skills/refine-ticket/SKILL.md +3 -0
- package/.agent-src/skills/subagent-orchestration/SKILL.md +33 -0
- package/.agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/.agent-src/templates/skill-archive-note.md +101 -0
- package/.agent-src/user-types/README.md +124 -0
- package/.agent-src/user-types/_template/user-type.md +95 -0
- package/.agent-src/user-types/galabau-field-crew.md +100 -0
- package/.agent-src/user-types/metalworking-shop.md +105 -0
- package/.agent-src/user-types/truck-driver.md +113 -0
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +91 -30
- package/README.md +68 -72
- package/config/agent-settings.template.yml +22 -0
- package/docs/adrs/caveman/0001-default-off-until-bench.md +93 -0
- package/docs/adrs/caveman/README.md +9 -0
- package/docs/adrs/cost/0001-hard-stop-hook.md +114 -0
- package/docs/adrs/cost/README.md +9 -0
- package/docs/adrs/memory/0001-consumer-side-snapshot.md +111 -0
- package/docs/adrs/memory/README.md +9 -0
- package/docs/adrs/router/0001-three-tier-routing.md +119 -0
- package/docs/adrs/router/README.md +9 -0
- package/docs/adrs/schema/0001-json-schema-frontmatter.md +102 -0
- package/docs/adrs/schema/README.md +9 -0
- package/docs/adrs/smoke/0001-per-tier-smoke-scripts.md +99 -0
- package/docs/adrs/smoke/README.md +9 -0
- package/docs/architecture/current-onboard-baseline.md +126 -0
- package/docs/architecture/current-safety-behavior.md +137 -0
- package/docs/archive/CHANGELOG-pre-2.16.0.md +48 -0
- package/docs/contracts/adr-layout.md +108 -0
- package/docs/contracts/adr-mcp-runtime.md +128 -0
- package/docs/contracts/adr-user-types-axis.md +127 -0
- package/docs/contracts/benchmark-corpus-spec.md +97 -0
- package/docs/contracts/benchmark-report-schema.md +111 -0
- package/docs/contracts/command-clusters.md +1 -0
- package/docs/contracts/command-taxonomy.md +137 -0
- package/docs/contracts/compression-default-kill-criterion.md +69 -0
- package/docs/contracts/config-presets.md +144 -0
- package/docs/contracts/cost-dashboard.md +143 -0
- package/docs/contracts/cost-enforcement.md +134 -0
- package/docs/contracts/file-ownership-matrix.json +0 -7
- package/docs/contracts/mcp-tool-inventory.md +53 -0
- package/docs/contracts/measurement-baseline.md +102 -0
- package/docs/contracts/namespace.md +125 -0
- package/docs/contracts/profile-system.md +142 -0
- package/docs/contracts/safety-model.md +129 -0
- package/docs/contracts/smoke-contracts.md +144 -0
- package/docs/contracts/user-type-schema.md +146 -0
- package/docs/contracts/workflow-packs.md +121 -0
- package/docs/decisions/ADR-010-profile-pack-preset-boundary.md +132 -0
- package/docs/decisions/INDEX.md +1 -0
- package/docs/featured-commands.md +27 -0
- package/docs/parity/bench-ruflo.json +58 -0
- package/docs/parity/bench.json +41 -0
- package/docs/parity/ruflo.md +46 -0
- package/docs/profiles.md +91 -0
- package/docs/recruits/_template.md +81 -0
- package/package.json +1 -1
- package/scripts/_cli/cmd_explain.py +250 -0
- package/scripts/_lib/bench_cost.py +138 -0
- package/scripts/_lib/bench_quality.py +118 -0
- package/scripts/_lib/bench_report.py +150 -0
- package/scripts/agent-config +13 -0
- package/scripts/audit_adr_coverage.py +175 -0
- package/scripts/audit_mcp_tools.py +146 -0
- package/scripts/bench_baseline_ready.py +108 -0
- package/scripts/bench_drift_check.py +151 -0
- package/scripts/bench_per_tool.py +216 -0
- package/scripts/bench_run.py +155 -0
- package/scripts/compress.py +48 -2
- package/scripts/config/__init__.py +9 -0
- package/scripts/config/presets.py +206 -0
- package/scripts/config/profiles.py +173 -0
- package/scripts/cost/budget.mjs +73 -12
- package/scripts/cost/preflight.mjs +89 -0
- package/scripts/lint_archived_skills.py +143 -0
- package/scripts/lint_bench_corpus.py +161 -0
- package/scripts/lint_namespace.py +135 -0
- package/scripts/schemas/user-type.schema.json +35 -0
- package/scripts/skill_linter.py +139 -4
- package/scripts/skill_overlap.py +204 -0
- package/scripts/skill_tools/audit_user_type_coverage.py +148 -0
- package/scripts/skill_usage_collect.py +191 -0
- package/scripts/skill_usage_report.py +162 -0
- package/scripts/smoke/kernel.sh +101 -0
- package/scripts/smoke/router.sh +129 -0
- package/scripts/smoke/schema.sh +71 -0
- package/scripts/smoke/skills.sh +101 -0
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"""Profile loader — step-15 Phase 1 item 1.
|
|
2
|
+
|
|
3
|
+
Resolves the active ``profile.id`` from the chain documented in
|
|
4
|
+
:mod:`docs.contracts.profile-system` and returns a structured
|
|
5
|
+
:class:`ResolvedProfile`. Pure, read-only, lazy-PyYAML.
|
|
6
|
+
|
|
7
|
+
Resolution chain (last writer wins):
|
|
8
|
+
|
|
9
|
+
1. Pack-supplied ``profile_id`` (Phase 2 item 7 — pack loader passes it
|
|
10
|
+
in via ``pack_profile_id``; ``None`` until packs land).
|
|
11
|
+
2. ``.agent-settings.yml`` top-level ``profile.id`` (and any user
|
|
12
|
+
overrides for ``audience`` / ``defaults`` / ``surface``).
|
|
13
|
+
3. Environment variable ``AGENT_CONFIG_PROFILE_ID``.
|
|
14
|
+
4. Runtime CLI flag — caller passes ``runtime_id``.
|
|
15
|
+
|
|
16
|
+
Falls back to ``developer`` **only** when no settings file exists yet
|
|
17
|
+
(fresh install before ``/onboard``). With a settings file present but
|
|
18
|
+
no ``profile`` block, the loader returns a structured warning state so
|
|
19
|
+
``/onboard`` can surface "audience not yet picked".
|
|
20
|
+
"""
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import logging
|
|
24
|
+
import os
|
|
25
|
+
from dataclasses import dataclass, field
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import Any
|
|
28
|
+
|
|
29
|
+
from scripts._lib import agent_settings
|
|
30
|
+
|
|
31
|
+
logger = logging.getLogger(__name__)
|
|
32
|
+
|
|
33
|
+
PROFILE_ID_ENV = "AGENT_CONFIG_PROFILE_ID"
|
|
34
|
+
SEED_PROFILE_IDS: tuple[str, ...] = (
|
|
35
|
+
"founder",
|
|
36
|
+
"developer",
|
|
37
|
+
"content_creator",
|
|
38
|
+
"agency",
|
|
39
|
+
"finance",
|
|
40
|
+
"ops",
|
|
41
|
+
)
|
|
42
|
+
DEFAULT_PROFILE_ID = "developer"
|
|
43
|
+
PROFILES_DIRNAME = ".agent-src.uncompressed/profiles"
|
|
44
|
+
|
|
45
|
+
SOURCE_PACK = "pack"
|
|
46
|
+
SOURCE_USER = "user-settings"
|
|
47
|
+
SOURCE_ENV = "env"
|
|
48
|
+
SOURCE_RUNTIME = "runtime"
|
|
49
|
+
SOURCE_DEFAULT = "default"
|
|
50
|
+
SOURCE_MISSING = "missing"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class ResolvedProfile:
|
|
55
|
+
"""Outcome of :func:`resolve_profile`. See profile-system contract."""
|
|
56
|
+
|
|
57
|
+
id: str
|
|
58
|
+
audience: dict[str, str] = field(default_factory=dict)
|
|
59
|
+
preset_id: str | None = None
|
|
60
|
+
personas: tuple[str, ...] = ()
|
|
61
|
+
skills_hint: tuple[str, ...] = ()
|
|
62
|
+
commands_hint: tuple[str, ...] = ()
|
|
63
|
+
docs_first_pointer: str | None = None
|
|
64
|
+
source: str = SOURCE_DEFAULT
|
|
65
|
+
warning: str | None = None
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class ProfileError(Exception):
|
|
69
|
+
"""Raised when a profile id is referenced but its YAML cannot load."""
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _load_yaml(path: Path) -> dict[str, Any]:
|
|
73
|
+
try:
|
|
74
|
+
import yaml # type: ignore[import-not-found]
|
|
75
|
+
except ImportError:
|
|
76
|
+
logger.info("PyYAML unavailable; profile %s returned empty", path)
|
|
77
|
+
return {}
|
|
78
|
+
try:
|
|
79
|
+
text = path.read_text(encoding="utf-8")
|
|
80
|
+
except OSError as exc:
|
|
81
|
+
logger.warning("profile read failed for %s: %s", path, exc)
|
|
82
|
+
return {}
|
|
83
|
+
try:
|
|
84
|
+
data = yaml.safe_load(text) or {}
|
|
85
|
+
except yaml.YAMLError as exc:
|
|
86
|
+
logger.warning("profile parse failed for %s: %s", path, exc)
|
|
87
|
+
return {}
|
|
88
|
+
return data if isinstance(data, dict) else {}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _profile_file(project_root: Path, profile_id: str) -> Path:
|
|
92
|
+
return project_root / PROFILES_DIRNAME / f"{profile_id}.yml"
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _build_resolved(
|
|
96
|
+
profile_id: str,
|
|
97
|
+
raw: dict[str, Any],
|
|
98
|
+
*,
|
|
99
|
+
source: str,
|
|
100
|
+
warning: str | None = None,
|
|
101
|
+
) -> ResolvedProfile:
|
|
102
|
+
block = raw.get("profile") or {}
|
|
103
|
+
audience_raw = block.get("audience") or {}
|
|
104
|
+
defaults = block.get("defaults") or {}
|
|
105
|
+
surface = block.get("surface") or {}
|
|
106
|
+
audience = {str(k): str(v) for k, v in audience_raw.items()}
|
|
107
|
+
personas = tuple(str(p) for p in (defaults.get("personas") or []))
|
|
108
|
+
skills_hint = tuple(str(s) for s in (defaults.get("skills_hint") or []))
|
|
109
|
+
commands_hint = tuple(str(c) for c in (surface.get("commands_hint") or []))
|
|
110
|
+
docs_pointer = surface.get("docs_first_pointer")
|
|
111
|
+
return ResolvedProfile(
|
|
112
|
+
id=profile_id,
|
|
113
|
+
audience=audience,
|
|
114
|
+
preset_id=defaults.get("preset_id"),
|
|
115
|
+
personas=personas,
|
|
116
|
+
skills_hint=skills_hint,
|
|
117
|
+
commands_hint=commands_hint,
|
|
118
|
+
docs_first_pointer=str(docs_pointer) if docs_pointer else None,
|
|
119
|
+
source=source,
|
|
120
|
+
warning=warning,
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _pick_id(
|
|
125
|
+
pack_profile_id: str | None,
|
|
126
|
+
user_settings: dict[str, Any],
|
|
127
|
+
runtime_id: str | None,
|
|
128
|
+
) -> tuple[str | None, str]:
|
|
129
|
+
if runtime_id:
|
|
130
|
+
return runtime_id, SOURCE_RUNTIME
|
|
131
|
+
env_id = os.environ.get(PROFILE_ID_ENV)
|
|
132
|
+
if env_id:
|
|
133
|
+
return env_id, SOURCE_ENV
|
|
134
|
+
block = user_settings.get("profile") if isinstance(user_settings, dict) else None
|
|
135
|
+
if isinstance(block, dict) and block.get("id"):
|
|
136
|
+
return str(block["id"]), SOURCE_USER
|
|
137
|
+
if pack_profile_id:
|
|
138
|
+
return pack_profile_id, SOURCE_PACK
|
|
139
|
+
return None, SOURCE_MISSING
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def resolve_profile(
|
|
143
|
+
*,
|
|
144
|
+
project_root: Path,
|
|
145
|
+
user_settings: dict[str, Any] | None = None,
|
|
146
|
+
pack_profile_id: str | None = None,
|
|
147
|
+
runtime_id: str | None = None,
|
|
148
|
+
) -> ResolvedProfile:
|
|
149
|
+
"""Return the active :class:`ResolvedProfile` for the current session."""
|
|
150
|
+
settings = user_settings or {}
|
|
151
|
+
settings_file = project_root / agent_settings.DEFAULT_PROJECT_FILE
|
|
152
|
+
profile_id, source = _pick_id(pack_profile_id, settings, runtime_id)
|
|
153
|
+
if profile_id is None:
|
|
154
|
+
if settings_file.exists():
|
|
155
|
+
return ResolvedProfile(
|
|
156
|
+
id=DEFAULT_PROFILE_ID,
|
|
157
|
+
source=SOURCE_MISSING,
|
|
158
|
+
warning=(
|
|
159
|
+
"no profile.id in .agent-settings.yml — run /onboard to "
|
|
160
|
+
"pick an audience deliberately"
|
|
161
|
+
),
|
|
162
|
+
)
|
|
163
|
+
return _build_resolved(
|
|
164
|
+
DEFAULT_PROFILE_ID,
|
|
165
|
+
_load_yaml(_profile_file(project_root, DEFAULT_PROFILE_ID)),
|
|
166
|
+
source=SOURCE_DEFAULT,
|
|
167
|
+
)
|
|
168
|
+
yaml_path = _profile_file(project_root, profile_id)
|
|
169
|
+
if not yaml_path.exists():
|
|
170
|
+
raise ProfileError(
|
|
171
|
+
f"profile.id={profile_id!r} ({source}) but {yaml_path} not found",
|
|
172
|
+
)
|
|
173
|
+
return _build_resolved(profile_id, _load_yaml(yaml_path), source=source)
|
package/scripts/cost/budget.mjs
CHANGED
|
@@ -14,10 +14,62 @@ import { dirname } from 'node:path';
|
|
|
14
14
|
|
|
15
15
|
const STORE = process.env.BUDGET_STORE || 'agents/cost-tracking/sessions.jsonl';
|
|
16
16
|
const CONFIG = process.env.BUDGET_CONFIG || 'agents/cost-tracking/budget.json';
|
|
17
|
+
const SETTINGS = process.env.AGENT_SETTINGS || '.agent-settings.yml';
|
|
18
|
+
|
|
19
|
+
// Minimal YAML reader for the `cost:` block — avoids a yaml dep. Reads
|
|
20
|
+
// only the keys this script needs (cost.budgets.{daily,weekly,monthly},
|
|
21
|
+
// cost.enforcement) from the well-formed two-space-indent template.
|
|
22
|
+
function loadSettingsCost() {
|
|
23
|
+
if (!existsSync(SETTINGS)) return null;
|
|
24
|
+
let inCost = false, inBudgets = false;
|
|
25
|
+
const out = { budgets: {}, enforcement: null };
|
|
26
|
+
for (const raw of readFileSync(SETTINGS, 'utf-8').split('\n')) {
|
|
27
|
+
const line = raw.replace(/\s+$/, '');
|
|
28
|
+
if (!line || line.startsWith('#')) continue;
|
|
29
|
+
if (/^[a-z_]+:/.test(line)) inCost = inBudgets = false;
|
|
30
|
+
if (line === 'cost:') { inCost = true; continue; }
|
|
31
|
+
if (!inCost) continue;
|
|
32
|
+
if (/^ budgets:/.test(line)) { inBudgets = true; continue; }
|
|
33
|
+
if (inBudgets && /^ [a-z]+:/.test(line)) {
|
|
34
|
+
const [k, v] = line.trim().split(':').map((s) => s.trim());
|
|
35
|
+
const n = parseFloat(v);
|
|
36
|
+
if (Number.isFinite(n) && n > 0) out.budgets[k] = n;
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
if (/^ enforcement:/.test(line)) {
|
|
40
|
+
out.enforcement = line.split(':')[1].trim().replace(/['"]/g, '');
|
|
41
|
+
inBudgets = false;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
const hasAny = Object.keys(out.budgets).length || out.enforcement;
|
|
45
|
+
return hasAny ? out : null;
|
|
46
|
+
}
|
|
17
47
|
|
|
18
48
|
function loadConfig() {
|
|
49
|
+
// Settings file wins when it carries any cost.* values.
|
|
50
|
+
const fromSettings = loadSettingsCost();
|
|
51
|
+
if (fromSettings) {
|
|
52
|
+
const period = process.env.BUDGET_PERIOD || 'all';
|
|
53
|
+
const periodKey = ({ today: 'daily', week: 'weekly', month: 'monthly' })[period];
|
|
54
|
+
const budget_usd = periodKey ? fromSettings.budgets[periodKey] : (
|
|
55
|
+
fromSettings.budgets.monthly || fromSettings.budgets.weekly || fromSettings.budgets.daily
|
|
56
|
+
);
|
|
57
|
+
if (Number.isFinite(budget_usd) && budget_usd > 0) {
|
|
58
|
+
return {
|
|
59
|
+
budget_usd,
|
|
60
|
+
enforcement: fromSettings.enforcement || 'advisory',
|
|
61
|
+
source: 'agent-settings.yml',
|
|
62
|
+
setAt: null,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
}
|
|
19
66
|
if (!existsSync(CONFIG)) return null;
|
|
20
|
-
try {
|
|
67
|
+
try {
|
|
68
|
+
const cfg = JSON.parse(readFileSync(CONFIG, 'utf-8'));
|
|
69
|
+
cfg.source = cfg.source || 'budget.json';
|
|
70
|
+
cfg.enforcement = cfg.enforcement || 'advisory';
|
|
71
|
+
return cfg;
|
|
72
|
+
} catch { return null; }
|
|
21
73
|
}
|
|
22
74
|
|
|
23
75
|
function saveConfig(cfg) {
|
|
@@ -123,18 +175,27 @@ function cmdCheck() {
|
|
|
123
175
|
threshold: alert.threshold,
|
|
124
176
|
recommended_action: recommendedAction(alert.level),
|
|
125
177
|
sessionCount: filtered.length,
|
|
178
|
+
enforcement: cfg.enforcement || 'advisory',
|
|
179
|
+
source: cfg.source || 'budget.json',
|
|
126
180
|
};
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
181
|
+
const hardStop = alert.level === 'HARD_STOP' && out.enforcement === 'hard-stop';
|
|
182
|
+
if (process.env.BUDGET_QUIET === '1') {
|
|
183
|
+
console.log(JSON.stringify(out));
|
|
184
|
+
} else {
|
|
185
|
+
console.log(`# Budget check (period: ${period})\n`);
|
|
186
|
+
console.log('| Metric | Value |\n|---|---:|');
|
|
187
|
+
console.log(`| Budget | $${cfg.budget_usd.toFixed(2)} |`);
|
|
188
|
+
console.log(`| Spent | $${totalSpend.toFixed(2)} |`);
|
|
189
|
+
console.log(`| Remaining | $${out.remaining_usd.toFixed(2)} |`);
|
|
190
|
+
console.log(`| Utilization | ${out.utilization_pct.toFixed(1)}% |`);
|
|
191
|
+
console.log(`| Sessions counted | ${filtered.length} |`);
|
|
192
|
+
console.log(`| **Alert** | **${alert.emoji} ${alert.level}** |`);
|
|
193
|
+
console.log(`| Enforcement | ${out.enforcement} (source: ${out.source}) |`);
|
|
194
|
+
console.log(`\nAction: ${out.recommended_action}`);
|
|
195
|
+
}
|
|
196
|
+
// Only fail closed when enforcement is hard-stop; advisory mode reports
|
|
197
|
+
// the breach but exits clean so wrappers keep working.
|
|
198
|
+
if (hardStop) process.exit(1);
|
|
138
199
|
}
|
|
139
200
|
|
|
140
201
|
function main() {
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Cost preflight hook — process-entry gate per ADR docs/adrs/cost/0001-hard-stop-hook.md.
|
|
3
|
+
//
|
|
4
|
+
// Wraps scripts/cost/budget.mjs check. Reads cost.enforcement from
|
|
5
|
+
// .agent-settings.yml. Exits non-zero only when:
|
|
6
|
+
// - enforcement: hard-stop, AND
|
|
7
|
+
// - level: HARD_STOP
|
|
8
|
+
//
|
|
9
|
+
// Default behaviour without any budget configured: exit 0 (fail-open).
|
|
10
|
+
// Designed to be invoked by shell / CI wrappers BEFORE composing a
|
|
11
|
+
// turn: `task cost:preflight` or `node scripts/cost/preflight.mjs`.
|
|
12
|
+
|
|
13
|
+
import { spawnSync } from 'node:child_process';
|
|
14
|
+
|
|
15
|
+
const QUIET = process.env.PREFLIGHT_QUIET === '1';
|
|
16
|
+
|
|
17
|
+
function runBudgetCheck() {
|
|
18
|
+
const env = { ...process.env, BUDGET_QUIET: '1' };
|
|
19
|
+
const r = spawnSync('node', ['scripts/cost/budget.mjs', 'check'], { env, encoding: 'utf-8' });
|
|
20
|
+
if (r.error) return { ok: false, fatal: true, msg: String(r.error) };
|
|
21
|
+
const out = (r.stdout || '').trim();
|
|
22
|
+
if (!out) return { ok: true, unbudgeted: true };
|
|
23
|
+
// budget.mjs prints a single JSON line under BUDGET_QUIET=1 when a
|
|
24
|
+
// budget is set; otherwise it prints a no-budget plaintext notice.
|
|
25
|
+
if (!out.startsWith('{')) return { ok: true, unbudgeted: true };
|
|
26
|
+
try {
|
|
27
|
+
const data = JSON.parse(out);
|
|
28
|
+
// budget.mjs JSON shape carries `error: 'no budget configured'`
|
|
29
|
+
// when cost.budgets are all 0 / absent — treat as unbudgeted.
|
|
30
|
+
if (data.error || !Number.isFinite(data.budget_usd)) return { ok: true, unbudgeted: true };
|
|
31
|
+
return { ok: true, data, childExit: r.status ?? 0 };
|
|
32
|
+
} catch (e) {
|
|
33
|
+
return { ok: false, fatal: true, msg: `parse: ${e.message}`, raw: out };
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function refuse(data) {
|
|
38
|
+
if (QUIET) {
|
|
39
|
+
console.log(JSON.stringify({
|
|
40
|
+
preflight: 'refused',
|
|
41
|
+
reason: 'cost-hard-stop',
|
|
42
|
+
level: data.level,
|
|
43
|
+
utilization_pct: data.utilization_pct,
|
|
44
|
+
budget_usd: data.budget_usd,
|
|
45
|
+
spent_usd: data.spent_usd,
|
|
46
|
+
enforcement: data.enforcement,
|
|
47
|
+
source: data.source,
|
|
48
|
+
}));
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
console.error('# 🛑 Cost preflight — HARD STOP\n');
|
|
52
|
+
console.error('| Metric | Value |');
|
|
53
|
+
console.error('|---|---:|');
|
|
54
|
+
console.error(`| Budget | $${data.budget_usd.toFixed(2)} |`);
|
|
55
|
+
console.error(`| Spent | $${data.spent_usd.toFixed(2)} |`);
|
|
56
|
+
console.error(`| Utilization | ${data.utilization_pct.toFixed(1)}% |`);
|
|
57
|
+
console.error(`| Enforcement | ${data.enforcement} (source: ${data.source}) |`);
|
|
58
|
+
console.error('\nBypass (pick one — see docs/contracts/cost-enforcement.md):');
|
|
59
|
+
console.error(' 1. Raise the budget: edit .agent-settings.yml § cost.budgets.<period>');
|
|
60
|
+
console.error(' 2. Reset the ledger: node scripts/cost/track.mjs reset --confirm');
|
|
61
|
+
console.error(' 3. Disable enforcement: set cost.enforcement: advisory');
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function main() {
|
|
65
|
+
const r = runBudgetCheck();
|
|
66
|
+
if (!r.ok && r.fatal) {
|
|
67
|
+
// Fail-open on infra error — never block work because the hook itself broke.
|
|
68
|
+
if (!QUIET) console.error(`# cost-preflight: skipped (${r.msg})`);
|
|
69
|
+
process.exit(0);
|
|
70
|
+
}
|
|
71
|
+
if (r.unbudgeted) {
|
|
72
|
+
if (!QUIET) console.log('# cost-preflight: no budget configured — pass.');
|
|
73
|
+
process.exit(0);
|
|
74
|
+
}
|
|
75
|
+
const d = r.data;
|
|
76
|
+
const hardStop = d.level === 'HARD_STOP' && d.enforcement === 'hard-stop';
|
|
77
|
+
if (hardStop) {
|
|
78
|
+
refuse(d);
|
|
79
|
+
process.exit(1);
|
|
80
|
+
}
|
|
81
|
+
if (!QUIET) {
|
|
82
|
+
console.log(`# cost-preflight: ${d.level} (${d.utilization_pct.toFixed(1)}% of $${d.budget_usd.toFixed(2)}, enforcement=${d.enforcement})`);
|
|
83
|
+
} else {
|
|
84
|
+
console.log(JSON.stringify({ preflight: 'pass', level: d.level, enforcement: d.enforcement, utilization_pct: d.utilization_pct }));
|
|
85
|
+
}
|
|
86
|
+
process.exit(0);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
main();
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Lint archive notes under agents/archived-skills/.
|
|
3
|
+
|
|
4
|
+
Enforces the contract from
|
|
5
|
+
.agent-src.uncompressed/templates/skill-archive-note.md:
|
|
6
|
+
|
|
7
|
+
1. Every <slug>.md under agents/archived-skills/ has the six required
|
|
8
|
+
frontmatter fields with valid values.
|
|
9
|
+
2. `reason` is one of {unused, merged, superseded, deprecated}.
|
|
10
|
+
3. When `reason ∈ {merged, superseded}` the `replacement` slug exists
|
|
11
|
+
under .agent-src.uncompressed/skills/.
|
|
12
|
+
4. No archived slug still has a live SKILL.md (no zombies).
|
|
13
|
+
5. No live SKILL.md cites an archived slug as a router target in
|
|
14
|
+
its frontmatter `replaced_by:` field.
|
|
15
|
+
|
|
16
|
+
Hooked into `task ci` via `task lint-archived-skills`. Passes cleanly
|
|
17
|
+
against an empty agents/archived-skills/ (only README.md present).
|
|
18
|
+
|
|
19
|
+
Exit codes:
|
|
20
|
+
0 contract holds
|
|
21
|
+
1 one or more violations
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import re
|
|
26
|
+
import sys
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
QUIET = "--quiet" in sys.argv
|
|
30
|
+
|
|
31
|
+
REPO = Path(__file__).resolve().parents[1]
|
|
32
|
+
ARCHIVE_DIR = REPO / "agents" / "archived-skills"
|
|
33
|
+
SKILLS_DIR = REPO / ".agent-src.uncompressed" / "skills"
|
|
34
|
+
|
|
35
|
+
REQUIRED_FIELDS = ("slug", "archived_on", "last_seen_count", "reason", "replacement", "last_known_callers")
|
|
36
|
+
VALID_REASONS = frozenset({"unused", "merged", "superseded", "deprecated"})
|
|
37
|
+
DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def parse_frontmatter(text: str) -> dict[str, str] | None:
|
|
41
|
+
if not text.startswith("---\n"):
|
|
42
|
+
return None
|
|
43
|
+
end = text.find("\n---\n", 4)
|
|
44
|
+
if end == -1:
|
|
45
|
+
return None
|
|
46
|
+
fields: dict[str, str] = {}
|
|
47
|
+
for line in text[4:end].splitlines():
|
|
48
|
+
if ":" not in line or line.startswith(" ") or line.startswith("-"):
|
|
49
|
+
continue
|
|
50
|
+
k, _, v = line.partition(":")
|
|
51
|
+
fields[k.strip()] = v.strip().strip('"').strip("'")
|
|
52
|
+
return fields
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def archived_slugs() -> list[Path]:
|
|
56
|
+
return sorted(p for p in ARCHIVE_DIR.glob("*.md") if p.name != "README.md")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def live_skill_slugs() -> set[str]:
|
|
60
|
+
return {p.name for p in SKILLS_DIR.iterdir() if p.is_dir() and (p / "SKILL.md").exists()}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def main() -> int:
|
|
64
|
+
if not ARCHIVE_DIR.exists():
|
|
65
|
+
print(f"❌ lint_archived_skills: {ARCHIVE_DIR} missing", file=sys.stderr)
|
|
66
|
+
return 1
|
|
67
|
+
|
|
68
|
+
notes = archived_slugs()
|
|
69
|
+
live = live_skill_slugs()
|
|
70
|
+
errors: list[str] = []
|
|
71
|
+
|
|
72
|
+
archived_keys: set[str] = set()
|
|
73
|
+
for note in notes:
|
|
74
|
+
text = note.read_text(encoding="utf-8")
|
|
75
|
+
fm = parse_frontmatter(text)
|
|
76
|
+
slug_from_name = note.stem
|
|
77
|
+
|
|
78
|
+
if fm is None:
|
|
79
|
+
errors.append(f"{note.name}: missing or malformed frontmatter")
|
|
80
|
+
continue
|
|
81
|
+
|
|
82
|
+
missing = [f for f in REQUIRED_FIELDS if f not in fm]
|
|
83
|
+
if missing:
|
|
84
|
+
errors.append(f"{note.name}: missing required fields: {', '.join(missing)}")
|
|
85
|
+
continue
|
|
86
|
+
|
|
87
|
+
if fm["slug"] != slug_from_name:
|
|
88
|
+
errors.append(f"{note.name}: slug field '{fm['slug']}' != filename stem '{slug_from_name}'")
|
|
89
|
+
|
|
90
|
+
if not DATE_RE.match(fm["archived_on"]):
|
|
91
|
+
errors.append(f"{note.name}: archived_on '{fm['archived_on']}' is not YYYY-MM-DD")
|
|
92
|
+
|
|
93
|
+
if fm["reason"] not in VALID_REASONS:
|
|
94
|
+
errors.append(f"{note.name}: reason '{fm['reason']}' not in {sorted(VALID_REASONS)}")
|
|
95
|
+
|
|
96
|
+
try:
|
|
97
|
+
int(fm["last_seen_count"])
|
|
98
|
+
except ValueError:
|
|
99
|
+
errors.append(f"{note.name}: last_seen_count '{fm['last_seen_count']}' is not an integer")
|
|
100
|
+
|
|
101
|
+
replacement = fm["replacement"]
|
|
102
|
+
reason = fm["reason"]
|
|
103
|
+
if reason in {"merged", "superseded"}:
|
|
104
|
+
if replacement == "none" or not replacement:
|
|
105
|
+
errors.append(f"{note.name}: reason={reason} requires a replacement slug, got 'none'")
|
|
106
|
+
elif replacement not in live:
|
|
107
|
+
errors.append(f"{note.name}: replacement '{replacement}' not found under {SKILLS_DIR}")
|
|
108
|
+
elif reason in {"unused", "deprecated"}:
|
|
109
|
+
if replacement not in {"none", ""}:
|
|
110
|
+
if replacement not in live:
|
|
111
|
+
errors.append(f"{note.name}: replacement '{replacement}' not found under {SKILLS_DIR}")
|
|
112
|
+
|
|
113
|
+
if fm["slug"] in live:
|
|
114
|
+
errors.append(f"{note.name}: slug '{fm['slug']}' still has a live SKILL.md (zombie)")
|
|
115
|
+
|
|
116
|
+
archived_keys.add(fm["slug"])
|
|
117
|
+
|
|
118
|
+
# Cross-check: live skills must not list an archived slug as replaced_by.
|
|
119
|
+
for skill_dir in sorted(SKILLS_DIR.iterdir()):
|
|
120
|
+
skill_md = skill_dir / "SKILL.md"
|
|
121
|
+
if not skill_md.exists():
|
|
122
|
+
continue
|
|
123
|
+
text = skill_md.read_text(encoding="utf-8")
|
|
124
|
+
fm = parse_frontmatter(text)
|
|
125
|
+
if fm is None:
|
|
126
|
+
continue
|
|
127
|
+
rb = fm.get("replaced_by", "").strip()
|
|
128
|
+
if rb and rb in archived_keys:
|
|
129
|
+
errors.append(f"{skill_dir.name}/SKILL.md: replaced_by '{rb}' points at an archived slug")
|
|
130
|
+
|
|
131
|
+
if errors:
|
|
132
|
+
print(f"❌ lint_archived_skills: {len(errors)} violation(s) across {len(notes)} note(s)", file=sys.stderr)
|
|
133
|
+
for e in errors:
|
|
134
|
+
print(f" {e}", file=sys.stderr)
|
|
135
|
+
return 1
|
|
136
|
+
|
|
137
|
+
if not QUIET:
|
|
138
|
+
print(f"✅ lint_archived_skills: {len(notes)} archive note(s), contract holds")
|
|
139
|
+
return 0
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
if __name__ == "__main__":
|
|
143
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Lint benchmark corpora under tests/eval/corpus-*.yaml.
|
|
3
|
+
|
|
4
|
+
Enforces the contract from docs/contracts/benchmark-corpus-spec.md:
|
|
5
|
+
- Required top-level keys (version, corpus_id, prompts) present.
|
|
6
|
+
- version == 1.
|
|
7
|
+
- selection_accuracy_target in [0.0, 1.0].
|
|
8
|
+
- Per-prompt schema (id format, category enum, language enum,
|
|
9
|
+
expected_skills non-empty + referencing real skills, destructive
|
|
10
|
+
prompts carry expected_carve_outs, prompt text non-empty).
|
|
11
|
+
- No duplicate ids within a corpus.
|
|
12
|
+
|
|
13
|
+
Hooked into `task ci` via `task lint-bench`. Step-4 Phase 1 Step 3.
|
|
14
|
+
|
|
15
|
+
Exit codes:
|
|
16
|
+
0 contract holds across every corpus
|
|
17
|
+
1 one or more violations
|
|
18
|
+
2 invocation error (missing PyYAML, no corpora found)
|
|
19
|
+
|
|
20
|
+
Flags:
|
|
21
|
+
--quiet suppress per-file OK lines
|
|
22
|
+
--require-full also enforce 25-prompt composition (10/8/5/2)
|
|
23
|
+
"""
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import re
|
|
27
|
+
import sys
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
|
|
30
|
+
try:
|
|
31
|
+
import yaml
|
|
32
|
+
except ImportError:
|
|
33
|
+
sys.stderr.write("error: PyYAML required (pip install pyyaml)\n")
|
|
34
|
+
sys.exit(2)
|
|
35
|
+
|
|
36
|
+
QUIET = "--quiet" in sys.argv
|
|
37
|
+
REQUIRE_FULL = "--require-full" in sys.argv
|
|
38
|
+
|
|
39
|
+
REPO = Path(__file__).resolve().parents[1]
|
|
40
|
+
CORPUS_DIR = REPO / "tests" / "eval"
|
|
41
|
+
SKILLS_DIR = REPO / ".agent-src.uncompressed" / "skills"
|
|
42
|
+
|
|
43
|
+
VALID_CATEGORIES = frozenset({"canonical", "ambiguous", "destructive", "long-context"})
|
|
44
|
+
# Non-dev corpus (pre-spec) uses legacy categories — accept them so the
|
|
45
|
+
# new linter does not break that file. Migration is a follow-up.
|
|
46
|
+
LEGACY_CATEGORIES = frozenset({"content", "consulting", "finance", "ops", "safety"})
|
|
47
|
+
VALID_LANGUAGES = frozenset({"en", "de"})
|
|
48
|
+
VALID_VERSIONS = frozenset({1})
|
|
49
|
+
ID_RE = re.compile(r"^[a-z][a-z0-9-]*-\d{2}$")
|
|
50
|
+
FULL_COUNTS = {"canonical": 10, "ambiguous": 8, "destructive": 5, "long-context": 2}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def live_skills() -> set[str]:
|
|
54
|
+
return {p.name for p in SKILLS_DIR.iterdir() if p.is_dir() and (p / "SKILL.md").exists()}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def lint_corpus(path: Path, skills: set[str]) -> list[str]:
|
|
58
|
+
errors: list[str] = []
|
|
59
|
+
try:
|
|
60
|
+
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
|
61
|
+
except yaml.YAMLError as exc:
|
|
62
|
+
return [f"{path.name}: yaml_parse_error: {exc}"]
|
|
63
|
+
|
|
64
|
+
if not isinstance(data, dict):
|
|
65
|
+
return [f"{path.name}: missing_top_level: corpus must be a mapping"]
|
|
66
|
+
|
|
67
|
+
for key in ("version", "corpus_id", "prompts"):
|
|
68
|
+
if key not in data:
|
|
69
|
+
errors.append(f"{path.name}: missing_top_level: {key}")
|
|
70
|
+
|
|
71
|
+
if data.get("version") not in VALID_VERSIONS:
|
|
72
|
+
errors.append(f"{path.name}: unsupported_version: {data.get('version')!r}")
|
|
73
|
+
|
|
74
|
+
target = data.get("selection_accuracy_target")
|
|
75
|
+
if target is not None and not (isinstance(target, (int, float)) and 0.0 <= target <= 1.0):
|
|
76
|
+
errors.append(f"{path.name}: target_out_of_range: {target!r}")
|
|
77
|
+
|
|
78
|
+
prompts = data.get("prompts") or []
|
|
79
|
+
if not isinstance(prompts, list):
|
|
80
|
+
return errors + [f"{path.name}: missing_top_level: prompts must be a list"]
|
|
81
|
+
|
|
82
|
+
seen_ids: set[str] = set()
|
|
83
|
+
bucket_counts: dict[str, int] = {}
|
|
84
|
+
is_legacy = data.get("corpus_id") == "non-dev"
|
|
85
|
+
for idx, p in enumerate(prompts):
|
|
86
|
+
loc = f"{path.name}:#{idx}"
|
|
87
|
+
if not isinstance(p, dict):
|
|
88
|
+
errors.append(f"{loc}: bad_prompt_shape")
|
|
89
|
+
continue
|
|
90
|
+
pid = p.get("id")
|
|
91
|
+
if not isinstance(pid, str) or not ID_RE.match(pid):
|
|
92
|
+
errors.append(f"{loc}: bad_id_format: {pid!r}")
|
|
93
|
+
elif pid in seen_ids:
|
|
94
|
+
errors.append(f"{loc}: duplicate_id: {pid}")
|
|
95
|
+
else:
|
|
96
|
+
seen_ids.add(pid)
|
|
97
|
+
|
|
98
|
+
cat = p.get("category")
|
|
99
|
+
if cat not in VALID_CATEGORIES and not (is_legacy and cat in LEGACY_CATEGORIES):
|
|
100
|
+
errors.append(f"{loc}: bad_category: {cat!r}")
|
|
101
|
+
bucket_counts[cat] = bucket_counts.get(cat, 0) + 1
|
|
102
|
+
|
|
103
|
+
lang = p.get("language", "en")
|
|
104
|
+
if lang not in VALID_LANGUAGES:
|
|
105
|
+
errors.append(f"{loc}: bad_language: {lang!r}")
|
|
106
|
+
|
|
107
|
+
prompt_text = p.get("prompt", "")
|
|
108
|
+
if not isinstance(prompt_text, str) or not prompt_text.strip():
|
|
109
|
+
errors.append(f"{loc}: empty_prompt")
|
|
110
|
+
|
|
111
|
+
expected = p.get("expected_skills") or []
|
|
112
|
+
if not isinstance(expected, list) or not expected:
|
|
113
|
+
errors.append(f"{loc}: empty_expected")
|
|
114
|
+
else:
|
|
115
|
+
for slug in expected:
|
|
116
|
+
if slug not in skills:
|
|
117
|
+
errors.append(f"{loc}: unknown_skill: {slug}")
|
|
118
|
+
|
|
119
|
+
if cat == "destructive":
|
|
120
|
+
carve = p.get("expected_carve_outs") or []
|
|
121
|
+
if not isinstance(carve, list) or not carve:
|
|
122
|
+
errors.append(f"{loc}: missing_carve_out")
|
|
123
|
+
|
|
124
|
+
if REQUIRE_FULL and not is_legacy:
|
|
125
|
+
for bucket, want in FULL_COUNTS.items():
|
|
126
|
+
have = bucket_counts.get(bucket, 0)
|
|
127
|
+
if have != want:
|
|
128
|
+
errors.append(f"{path.name}: composition_drift: {bucket} have={have} want={want}")
|
|
129
|
+
|
|
130
|
+
return errors
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def main() -> int:
|
|
134
|
+
if not CORPUS_DIR.is_dir():
|
|
135
|
+
sys.stderr.write(f"error: corpus dir missing: {CORPUS_DIR}\n")
|
|
136
|
+
return 2
|
|
137
|
+
corpora = sorted(CORPUS_DIR.glob("corpus-*.yaml"))
|
|
138
|
+
if not corpora:
|
|
139
|
+
sys.stderr.write("error: no corpora found\n")
|
|
140
|
+
return 2
|
|
141
|
+
|
|
142
|
+
skills = live_skills()
|
|
143
|
+
all_errors: list[str] = []
|
|
144
|
+
for path in corpora:
|
|
145
|
+
errs = lint_corpus(path, skills)
|
|
146
|
+
if errs:
|
|
147
|
+
all_errors.extend(errs)
|
|
148
|
+
elif not QUIET:
|
|
149
|
+
print(f"✅ {path.name}: contract OK")
|
|
150
|
+
|
|
151
|
+
if all_errors:
|
|
152
|
+
for err in all_errors:
|
|
153
|
+
print(f"❌ {err}", file=sys.stderr)
|
|
154
|
+
return 1
|
|
155
|
+
if not QUIET:
|
|
156
|
+
print(f"✅ lint-bench: {len(corpora)} corpora clean")
|
|
157
|
+
return 0
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
if __name__ == "__main__":
|
|
161
|
+
sys.exit(main())
|