@event4u/agent-config 2.10.0 → 2.12.0

package/scripts/skill_linter.py

@@ -775,8 +775,14 @@ def lint_skill(path: Path, text: str) -> LintResult:
     # is *both* large AND prose-dominant OR ships ≥ 2 independently invocable
     # procedures. Reference catalogues (quality-tools 411 L / density 0.83)
     # pass; multi-procedure skills are flagged for split.
+    #
+    # Frontmatter opt-out: `meta_skill: true` exempts a skill from the size
+    # warn when the skill's purpose *is* breadth (skill-writing, agent-docs-
+    # writing, skill-reviewer, etc.). Meta-skills inherently bundle multiple
+    # procedures and inline examples.
     total_lines = len(text.splitlines())
-    if total_lines > 400:
+    is_meta_skill = bool(fm) and re.search(r"^meta_skill:\s*true\s*$", fm, re.MULTILINE)
+    if total_lines > 400 and not is_meta_skill:
         density = _density_score(text)
         procedures = _count_procedure_sections(text)
         if density < 0.6 or procedures >= 2:
@@ -832,6 +838,12 @@ def lint_skill(path: Path, text: str) -> LintResult:
                                f"{meaningful_steps} steps) — may lack its own executable workflow"))
             suggestions.append("Expand the skill so it remains executable without opening a guideline")
 
+    # --- evals.json schema validator ---
+    # When a skill ships sibling `evals/evals.json` (quantitative behavior
+    # eval per skill-writing § 7), validate its shape. Triggers.json is a
+    # separate concern handled elsewhere. All issues here are WARN.
+    issues.extend(validate_evals_json(path))
+
     return LintResult(
         file=str(path),
         artifact_type="skill",
@@ -841,6 +853,64 @@ def lint_skill(path: Path, text: str) -> LintResult:
     )
 
 
+def validate_evals_json(skill_path: Path) -> list[Issue]:
+    """Validate `{skill_dir}/evals/evals.json` against the schema declared
+    in `skill-writing` § 7. Returns WARN-level issues only; never blocks.
+    Skipped entirely when the file is absent."""
+    evals_path = skill_path.parent / "evals" / "evals.json"
+    if not evals_path.is_file():
+        return []
+    issues: list[Issue] = []
+    try:
+        data = json.loads(evals_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        return [Issue("warning", "evals_json_unreadable",
+                      f"evals/evals.json could not be parsed: {exc}")]
+    if not isinstance(data, dict):
+        return [Issue("warning", "evals_json_shape",
+                      "evals/evals.json root must be an object")]
+    if "skill" not in data or not isinstance(data["skill"], str):
+        issues.append(Issue("warning", "evals_json_missing_skill",
+                            "evals/evals.json must declare top-level 'skill' (string)"))
+    scenarios = data.get("scenarios")
+    if not isinstance(scenarios, list) or len(scenarios) < 1:
+        issues.append(Issue("warning", "evals_json_no_scenarios",
+                            "evals/evals.json must declare 'scenarios' (non-empty array)"))
+        return issues
+    valid_kinds = {"contains", "file_exists", "rubric"}
+    for idx, scenario in enumerate(scenarios):
+        loc = f"scenarios[{idx}]"
+        if not isinstance(scenario, dict):
+            issues.append(Issue("warning", "evals_json_scenario_shape",
+                                f"{loc} must be an object"))
+            continue
+        for key in ("id", "prompt"):
+            if key not in scenario or not isinstance(scenario[key], str) or not scenario[key].strip():
+                issues.append(Issue("warning", "evals_json_scenario_missing_field",
+                                    f"{loc} missing required string field '{key}'"))
+        assertions = scenario.get("assertions")
+        if not isinstance(assertions, list) or len(assertions) < 1:
+            issues.append(Issue("warning", "evals_json_scenario_no_assertions",
+                                f"{loc}.assertions must be a non-empty array"))
+            continue
+        for a_idx, assertion in enumerate(assertions):
+            a_loc = f"{loc}.assertions[{a_idx}]"
+            if not isinstance(assertion, dict):
+                issues.append(Issue("warning", "evals_json_assertion_shape",
+                                    f"{a_loc} must be an object"))
+                continue
+            kind = assertion.get("kind")
+            if kind not in valid_kinds:
+                issues.append(Issue("warning", "evals_json_assertion_kind",
+                                    f"{a_loc}.kind must be one of {sorted(valid_kinds)}, got {kind!r}"))
+                continue
+            required_field = {"contains": "value", "file_exists": "path", "rubric": "criterion"}[kind]
+            if required_field not in assertion or not isinstance(assertion[required_field], str):
+                issues.append(Issue("warning", "evals_json_assertion_missing_field",
+                                    f"{a_loc} (kind={kind}) missing required string field '{required_field}'"))
+    return issues
+
+
 def extract_frontmatter(text: str) -> Optional[str]:
     match = FRONTMATTER_PATTERN.search(text)
     return match.group(1) if match else None
@@ -2233,17 +2303,26 @@ def lint_type_boundaries(path: Path, text: str, artifact_type: str) -> List[Issu
         # Check frontmatter skills field
         frontmatter = extract_frontmatter(text)
         has_skills_field = False
+        # Commands tagged `type: orchestrator` aggregate other commands /
+        # routers — they intentionally do not declare a `skills:` list and
+        # are exempt from the no-skill-reference check. The tag is the
+        # contract; no hard-coded path list.
+        is_orchestrator = False
         if frontmatter:
             skills_match = re.search(r'skills:\s*\[(.+)\]', frontmatter)
             has_skills_field = bool(skills_match and skills_match.group(1).strip())
+            type_match = re.search(r'^type:\s*[\'"]?orchestrator[\'"]?\s*$',
+                                   frontmatter, re.MULTILINE)
+            is_orchestrator = bool(type_match)
 
         # Also check body for skill references
         has_skill_ref = bool(re.search(r'skill|SKILL\.md', text))
 
-        if not has_skills_field and not has_skill_ref:
+        if not has_skills_field and not has_skill_ref and not is_orchestrator:
             issues.append(Issue("warning", "command_missing_skill_references",
                                 "Command does not reference any skills — "
-                                "commands should orchestrate skills, not contain domain logic"))
+                                "commands should orchestrate skills, not contain domain logic "
+                                "(use `type: orchestrator` in frontmatter to exempt routers)"))
 
     # --- Skill: validation should be concrete, not vague ---
     if artifact_type == "skill":

package/scripts/smoke_quickstart.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""Structural smoke-test for the README Quickstart path.
+
+Verifies the 3-step Quickstart from a fresh-project perspective:
+
+  1. `scripts/install.py --project <tmpdir>` produces a usable
+     `.agent-settings.yml` with the documented default `cost_profile`.
+  2. The decision_engine block (P2.x of road-to-productization) parses
+     cleanly through the same engine parser the runtime uses.
+  3. The work-engine state-file format (`agents/state/<id>.json`) is
+     emit-ready — schema for `decision_result` matches the contract.
+
+What it does NOT do:
+  - Invoke a real LLM agent (CI doesn't run a model). The end-to-end
+    `/onboard → /work → decision_result` chain still requires the host
+    agent. This smoke test asserts the *mechanics* the agent depends
+    on, so a Quickstart break is caught before the agent ever runs.
+
+Exit codes: 0 = green; 1 = one or more checks failed; 2 = setup error.
+"""
+from __future__ import annotations
+
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+INSTALLER = ROOT / "scripts" / "install.py"
+TEMPLATE = ROOT / "config" / "agent-settings.template.yml"
+
+EXPECTED_DEFAULT_PROFILE = "balanced"
+
+
+def _fail(msg: str) -> int:
+    print(f"::error::{msg}", file=sys.stderr)
+    return 1
+
+
+def _check_installer_runs(tmpdir: Path) -> tuple[int, Path | None]:
+    """Step 1 — run installer against a fresh tmpdir."""
+    cmd = [
+        sys.executable,
+        str(INSTALLER),
+        "--project",
+        str(tmpdir),
+        "--package",
+        str(ROOT),
+        "--skip-bridges",
+    ]
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=60)
+    except subprocess.TimeoutExpired:
+        return _fail("installer timed out after 60s"), None
+    if result.returncode != 0:
+        return (
+            _fail(f"installer exited {result.returncode}\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}"),
+            None,
+        )
+    settings = tmpdir / ".agent-settings.yml"
+    if not settings.exists():
+        return _fail(".agent-settings.yml not written by installer"), None
+    return 0, settings
+
+
+def _check_default_profile(settings: Path) -> int:
+    """Step 2 — assert default cost_profile matches the contract."""
+    import yaml
+
+    parsed = yaml.safe_load(settings.read_text(encoding="utf-8"))
+    if not isinstance(parsed, dict):
+        return _fail(f"{settings.name}: top-level is not a YAML mapping")
+    profile = parsed.get("cost_profile")
+    if profile != EXPECTED_DEFAULT_PROFILE:
+        return _fail(
+            f"cost_profile drift: docs/contracts/cost-profile-defaults.md "
+            f"declares '{EXPECTED_DEFAULT_PROFILE}', settings has '{profile!r}'"
+        )
+    return 0
+
+
+def _check_decision_engine_block(settings: Path) -> int:
+    """Step 3 — decision_engine block parses through the engine parser."""
+    sys.path.insert(0, str(ROOT / ".agent-src.uncompressed" / "templates" / "scripts"))
+    try:
+        from work_engine.scoring.decision_engine import (  # type: ignore[import-not-found]
+            DecisionEngineSettings,
+            parse as parse_decision_engine,
+        )
+    except ImportError as exc:
+        return _fail(f"decision_engine module not importable: {exc}")
+
+    import yaml
+
+    parsed = yaml.safe_load(settings.read_text(encoding="utf-8"))
+    block = parsed.get("decision_engine") if isinstance(parsed, dict) else None
+    try:
+        settings_obj = parse_decision_engine(block)
+    except Exception as exc:  # noqa: BLE001 — surface the schema error
+        return _fail(f"decision_engine block rejected by parser: {exc}")
+    if not isinstance(settings_obj, DecisionEngineSettings):
+        return _fail("parser returned non-DecisionEngineSettings instance")
+    return 0
+
+
+def main() -> int:
+    if not INSTALLER.exists():
+        print(f"::error::installer not found at {INSTALLER}", file=sys.stderr)
+        return 2
+    if not TEMPLATE.exists():
+        print(f"::error::template not found at {TEMPLATE}", file=sys.stderr)
+        return 2
+
+    failures = 0
+    tmpdir = Path(tempfile.mkdtemp(prefix="agent-config-quickstart-"))
+    try:
+        rc, settings = _check_installer_runs(tmpdir)
+        failures += rc
+        if settings is not None:
+            failures += _check_default_profile(settings)
+            failures += _check_decision_engine_block(settings)
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
+    if failures:
+        print(f"\n❌  smoke-quickstart: {failures} check(s) failed", file=sys.stderr)
+        return 1
+    print("✅  smoke-quickstart: install → settings → decision_engine green")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/validate_decision_engine.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""
+Decision-engine settings validator (road-to-productization P2).
+
+Walks every ``agent-settings.yml`` / ``agent-settings.template.yml``
+under the repo, parses any ``decision_engine`` block via the canonical
+``work_engine.scoring.decision_engine.parse`` schema, and surfaces:
+
+- hard errors → exit 1 (unknown keys, invalid enum values, bad types).
+- warnings    → exit 0 with a ``::warning::`` line per finding
+  (gates active but ``hooks.enabled`` is false → gates won't fire).
+
+Contract: ``docs/contracts/decision-engine-gates.md``. Wired into
+``task ci`` via ``taskfiles/ci-fast.yml`` so configuration drift is
+caught before a Decision Engine surprise lands in main.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+try:
+    import yaml
+except ImportError:  # pragma: no cover — bootstrap guard
+    print("::error::PyYAML not installed; cannot validate decision_engine block")
+    sys.exit(3)
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+TEMPLATE_SCRIPTS = REPO_ROOT / ".agent-src.uncompressed" / "templates" / "scripts"
+if str(TEMPLATE_SCRIPTS) not in sys.path:
+    sys.path.insert(0, str(TEMPLATE_SCRIPTS))
+
+from work_engine.scoring.decision_engine import (  # noqa: E402
+    DecisionEngineConfigError,
+    parse,
+)
+
+# Files we always validate, even if they don't exist (template is
+# canonical — its absence is itself a regression).
+TEMPLATE_PATH = REPO_ROOT / "config" / "agent-settings.template.yml"
+# Project-level overrides developers may have on disk locally.
+LOCAL_PATHS = [REPO_ROOT / ".agent-settings.yml"]
+
+
+def _load_yaml(path: Path) -> dict | None:
+    if not path.is_file():
+        return None
+    try:
+        raw = yaml.safe_load(path.read_text(encoding="utf-8"))
+    except yaml.YAMLError as exc:
+        print(f"::error file={path}::malformed YAML: {exc}")
+        return {}
+    if raw is None:
+        return {}
+    if not isinstance(raw, dict):
+        print(f"::error file={path}::top-level must be a mapping")
+        return {}
+    return raw
+
+
+def _validate(path: Path, doc: dict) -> tuple[int, int]:
+    """Return ``(errors, warnings)`` counts for ``doc``."""
+    errors = 0
+    warnings = 0
+    block = doc.get("decision_engine")
+    if block is None:
+        return 0, 0
+    try:
+        settings = parse(block)
+    except DecisionEngineConfigError as exc:
+        rel = path.relative_to(REPO_ROOT)
+        print(f"::error file={rel}::decision_engine: {exc}")
+        return 1, 0
+    if settings.any_gate_active:
+        hooks_block = doc.get("hooks") or {}
+        if isinstance(hooks_block, dict) and hooks_block.get("enabled") is False:
+            rel = path.relative_to(REPO_ROOT)
+            print(
+                f"::warning file={rel}::decision_engine gates configured "
+                "(min_confidence/block_on_risk/require_memory_hits) but "
+                "hooks.enabled=false — gates will not fire. Either enable "
+                "hooks or remove the gate keys."
+            )
+            warnings += 1
+    return errors, warnings
+
+
+def main() -> int:
+    total_errors = 0
+    total_warnings = 0
+    paths: list[Path] = []
+    if TEMPLATE_PATH.is_file():
+        paths.append(TEMPLATE_PATH)
+    else:
+        print(f"::error file={TEMPLATE_PATH}::template missing")
+        return 1
+    for candidate in LOCAL_PATHS:
+        if candidate.is_file():
+            paths.append(candidate)
+    for path in paths:
+        doc = _load_yaml(path)
+        if doc is None:
+            continue
+        errors, warnings = _validate(path, doc)
+        total_errors += errors
+        total_warnings += warnings
+    if total_errors:
+        return 1
+    if total_warnings:
+        # Warnings already printed; CI treats exit 0 + ::warning:: as
+        # green-with-note. Surface a summary for human readers.
+        print(
+            f"decision_engine: {total_warnings} warning(s); see ::warning:: lines above"
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except Exception as exc:  # noqa: BLE001
+        print(f"::error::validate_decision_engine internal error: {exc}")
+        sys.exit(3)