@softspark/ai-toolkit 4.21.0 → 4.22.1

package/scripts/surface_manifest.py

@@ -0,0 +1,246 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024-2026 Lukasz Krzemien (biuro@softspark.eu)
+# Source: https://github.com/softspark/ai-toolkit
+
+"""Public surface manifest — the enforceable half of BACKWARD_COMPATIBILITY.md.
+
+`BACKWARD_COMPATIBILITY.md` names the surfaces users depend on. Prose stops
+nothing: rename a skill and every gate stays green. This turns the list into a
+committed snapshot and a test.
+
+The check is deliberately one-directional:
+
+    entry in the manifest, missing from the tree  -> FAIL   (a removal)
+    entry in the tree, missing from the manifest  -> pass   (an addition)
+
+Additions are free because a surface nobody has installed yet has no users to
+break. Removals fail because someone out there pinned the version that had it.
+
+The manifest is NOT regenerated automatically. If it were, deleting a skill would
+delete its manifest entry in the same breath and the check would detect nothing.
+`--update` is a deliberate act: run it, read the diff, and any line that
+disappeared is a breaking change owing an entry in DECISIONS.md.
+
+Stdlib-only.
+
+Usage:
+    python3 scripts/surface_manifest.py            # check, human-readable
+    python3 scripts/surface_manifest.py --json     # check, machine-readable
+    python3 scripts/surface_manifest.py --update   # rewrite app/surface.json
+    python3 scripts/surface_manifest.py --toolkit-dir /path
+
+Exit codes:
+    0  no protected surface was removed
+    1  a removal detected (or the manifest is missing on a check run)
+    2  usage error
+"""
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _common import toolkit_dir as default_toolkit_dir
+
+MANIFEST_RELPATH = Path("app") / "surface.json"
+
+FM_FIELD_RE = re.compile(r"^([a-z][a-z0-9-]*):")
+CLI_COMMAND_RE = re.compile(r"^\s+'?([a-z][a-z0-9-]*)'?\s*:")
+
+
+def _frontmatter_fields(path: Path) -> set[str]:
+    parts = path.read_text(encoding="utf-8").split("---")
+    if len(parts) < 3:
+        return set()
+    return {
+        m.group(1)
+        for line in parts[1].splitlines()
+        if (m := FM_FIELD_RE.match(line))
+    }
+
+
+def collect_surface(tk_dir: Path) -> dict:
+    """Read the current public surface out of the tree."""
+    skills_dir = tk_dir / "app" / "skills"
+    agents_dir = tk_dir / "app" / "agents"
+
+    skills: list[str] = []
+    skill_fields: set[str] = set()
+    for d in sorted(skills_dir.iterdir()) if skills_dir.is_dir() else []:
+        if not d.is_dir() or d.name.startswith("_") or not (d / "SKILL.md").is_file():
+            continue
+        skills.append(d.name)
+        skill_fields |= _frontmatter_fields(d / "SKILL.md")
+
+    agents: list[str] = []
+    agent_fields: set[str] = set()
+    for f in sorted(agents_dir.glob("*.md")) if agents_dir.is_dir() else []:
+        agents.append(f.stem)
+        agent_fields |= _frontmatter_fields(f)
+
+    hooks_dir = tk_dir / "app" / "hooks"
+    hook_scripts = sorted(
+        f.name for f in hooks_dir.glob("*.sh") if not f.name.startswith("_")
+    ) if hooks_dir.is_dir() else []
+
+    hook_events: list[str] = []
+    hooks_json = tk_dir / "app" / "hooks.json"
+    if hooks_json.is_file():
+        try:
+            hook_events = sorted(json.loads(hooks_json.read_text(encoding="utf-8")).get("hooks", {}))
+        except (json.JSONDecodeError, OSError):
+            hook_events = []
+
+    plugins_dir = tk_dir / "app" / "plugins"
+    packs = sorted(
+        d.name for d in plugins_dir.iterdir()
+        if d.is_dir() and not d.name.startswith(".")
+    ) if plugins_dir.is_dir() else []
+
+    kb_categories: list[str] = []
+    validate_py = tk_dir / "scripts" / "validate.py"
+    if validate_py.is_file():
+        text = validate_py.read_text(encoding="utf-8")
+        block = re.search(r"VALID_KB_CATEGORIES\s*=\s*frozenset\(\{(.*?)\}\)", text, re.S)
+        if block:
+            kb_categories = sorted(re.findall(r'"([a-z-]+)"', block.group(1)))
+
+    cli_commands: list[str] = []
+    cli = tk_dir / "bin" / "ai-toolkit.js"
+    if not cli.is_file():
+        candidates = sorted((tk_dir / "bin").glob("*.js")) if (tk_dir / "bin").is_dir() else []
+        cli = candidates[0] if candidates else cli
+    if cli.is_file():
+        # The COMMANDS map is what `--help` prints, so it is the published surface.
+        # An undocumented internal branch is not a promise and is not captured here.
+        block = re.search(r"const COMMANDS\s*=\s*\{(.*?)\n\};", cli.read_text(encoding="utf-8"), re.S)
+        if block:
+            cli_commands = sorted({
+                m.group(1) for line in block.group(1).splitlines()
+                if (m := CLI_COMMAND_RE.match(line))
+            })
+
+    return {
+        "skills": skills,
+        "agents": agents,
+        "skill_frontmatter_fields": sorted(skill_fields),
+        "agent_frontmatter_fields": sorted(agent_fields),
+        "hook_scripts": hook_scripts,
+        "hook_events": hook_events,
+        "plugin_packs": packs,
+        "kb_categories": kb_categories,
+        "cli_commands": cli_commands,
+    }
+
+
+def compare(manifest: dict, current: dict) -> dict:
+    """Removals fail, additions pass. See the module docstring for why."""
+    removed: dict[str, list[str]] = {}
+    added: dict[str, list[str]] = {}
+    for key, entries in manifest.items():
+        if key.startswith("_"):
+            continue
+        have = set(current.get(key, []))
+        gone = [e for e in entries if e not in have]
+        if gone:
+            removed[key] = gone
+        new = [e for e in current.get(key, []) if e not in set(entries)]
+        if new:
+            added[key] = new
+    return {"removed": removed, "added": added, "ok": not removed}
+
+
+def _write_manifest(path: Path, current: dict) -> None:
+    payload = {
+        "_comment": (
+            "Public surface snapshot. Removing an entry is a breaking change: see "
+            "BACKWARD_COMPATIBILITY.md, and record it in DECISIONS.md. Regenerate "
+            "deliberately with `python3 scripts/surface_manifest.py --update` and "
+            "read the diff — never as a reflex to green a red build."
+        ),
+        **current,
+    }
+    path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+
+def main(argv: list[str]) -> int:
+    update = False
+    as_json = False
+    tk_dir = default_toolkit_dir
+
+    idx = 0
+    while idx < len(argv):
+        arg = argv[idx]
+        if arg == "--update":
+            update = True
+        elif arg == "--json":
+            as_json = True
+        elif arg in ("-h", "--help"):
+            print(__doc__)
+            return 0
+        elif arg == "--toolkit-dir":
+            idx += 1
+            if idx >= len(argv):
+                print("ERROR: --toolkit-dir requires a value", file=sys.stderr)
+                return 2
+            tk_dir = Path(argv[idx])
+        elif arg.startswith("-"):
+            print(f"ERROR: unknown option: {arg}", file=sys.stderr)
+            return 2
+        else:
+            tk_dir = Path(arg)
+        idx += 1
+
+    tk_dir = tk_dir.resolve()
+    manifest_path = tk_dir / MANIFEST_RELPATH
+    current = collect_surface(tk_dir)
+
+    if update:
+        _write_manifest(manifest_path, current)
+        total = sum(len(v) for v in current.values())
+        print(f"Wrote {manifest_path.relative_to(tk_dir)} — {total} protected entries")
+        print("Read the diff. Every line removed is a breaking change.")
+        return 0
+
+    if not manifest_path.is_file():
+        print(f"ERROR: {MANIFEST_RELPATH} not found — run with --update to create it",
+              file=sys.stderr)
+        return 1
+
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    result = compare(manifest, current)
+
+    if as_json:
+        print(json.dumps(result, indent=2, ensure_ascii=False))
+        return 0 if result["ok"] else 1
+
+    print("## Public Surface")
+    if result["removed"]:
+        for key, entries in sorted(result["removed"].items()):
+            for entry in entries:
+                print(f"  ERROR: {key}: '{entry}' was removed from the public surface")
+        print()
+        print("A removal is a breaking change. Either restore it, or take the")
+        print("deprecation path in BACKWARD_COMPATIBILITY.md, record it in")
+        print("DECISIONS.md, and re-run with --update.")
+        print()
+        print("SURFACE CHECK FAILED")
+        return 1
+
+    protected = sum(len(v) for k, v in manifest.items() if not k.startswith("_"))
+    print(f"  OK: {protected} protected entries intact")
+    if result["added"]:
+        new_total = sum(len(v) for v in result["added"].values())
+        detail = ", ".join(f"{k} +{len(v)}" for k, v in sorted(result["added"].items()))
+        print(f"  Note: {new_total} new entries not yet protected ({detail})")
+        print("        Run --update before the next release to adopt them.")
+    print()
+    print("SURFACE CHECK PASSED")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/scripts/sync_badges.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024-2026 Lukasz Krzemien (biuro@softspark.eu)
+# Source: https://github.com/softspark/ai-toolkit
+
+"""Sync the README count badges to ground truth.
+
+`validate.py` errors when a README badge disagrees with the tree. The badges were
+maintained by hand, so every change that added a test or a skill turned the build
+red until someone edited a number — a tax paid per change, with a diagnosis cycle
+each time it was forgotten.
+
+This derives the three counts the same way `validate.py` does and rewrites them.
+It runs inside `npm run generate:all`, which `prepublishOnly` executes before
+`validate.py --strict`, so the badges are correct by the time anything checks them.
+
+Counting rules, matched to validate.py so the two cannot disagree:
+  agents  — *.md under app/agents/
+  skills  — directories under app/skills/ with a SKILL.md, excluding _-prefixed
+  tests   — `@test ` declarations across tests/*.bats
+
+Stdlib-only.
+
+Usage:
+    python3 scripts/sync_badges.py            # rewrite README.md badges
+    python3 scripts/sync_badges.py --check    # report drift, change nothing
+    python3 scripts/sync_badges.py --toolkit-dir /path
+
+Exit codes:
+    0  badges are correct (or were rewritten)
+    1  --check found drift
+    2  usage error
+"""
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _common import toolkit_dir as default_toolkit_dir
+
+TEST_RE = re.compile(r"^@test ", re.MULTILINE)
+
+
+def counts(tk_dir: Path) -> dict[str, int]:
+    agents_dir = tk_dir / "app" / "agents"
+    skills_dir = tk_dir / "app" / "skills"
+    tests_dir = tk_dir / "tests"
+
+    agents = len(list(agents_dir.glob("*.md"))) if agents_dir.is_dir() else 0
+    skills = sum(
+        1 for d in skills_dir.iterdir()
+        if d.is_dir() and not d.name.startswith("_") and (d / "SKILL.md").is_file()
+    ) if skills_dir.is_dir() else 0
+    tests = sum(
+        len(TEST_RE.findall(f.read_text(encoding="utf-8")))
+        for f in tests_dir.glob("*.bats")
+    ) if tests_dir.is_dir() else 0
+
+    return {"agents": agents, "skills": skills, "tests": tests}
+
+
+def sync(tk_dir: Path, check_only: bool) -> int:
+    readme = tk_dir / "README.md"
+    if not readme.is_file():
+        print("ERROR: README.md not found", file=sys.stderr)
+        return 2
+
+    text = readme.read_text(encoding="utf-8")
+    actual = counts(tk_dir)
+    drift: list[str] = []
+    updated = text
+
+    for label, value in actual.items():
+        pattern = re.compile(rf"({label}-)(\d+)")
+        match = pattern.search(updated)
+        if not match:
+            continue
+        if match.group(2) != str(value):
+            drift.append(f"{label}: badge {match.group(2)} != actual {value}")
+            updated = pattern.sub(rf"\g<1>{value}", updated, count=1)
+
+    if check_only:
+        for line in drift:
+            print(f"  DRIFT: {line}")
+        if drift:
+            print("BADGE CHECK FAILED — run `python3 scripts/sync_badges.py`")
+            return 1
+        print(f"  OK: badges match ({actual['agents']} agents, "
+              f"{actual['skills']} skills, {actual['tests']} tests)")
+        return 0
+
+    if drift:
+        readme.write_text(updated, encoding="utf-8")
+        for line in drift:
+            print(f"  synced {line}")
+    else:
+        print(f"  OK: badges already match ({actual['agents']} agents, "
+              f"{actual['skills']} skills, {actual['tests']} tests)")
+    return 0
+
+
+def main(argv: list[str]) -> int:
+    check_only = False
+    tk_dir = default_toolkit_dir
+
+    idx = 0
+    while idx < len(argv):
+        arg = argv[idx]
+        if arg == "--check":
+            check_only = True
+        elif arg in ("-h", "--help"):
+            print(__doc__)
+            return 0
+        elif arg == "--toolkit-dir":
+            idx += 1
+            if idx >= len(argv):
+                print("ERROR: --toolkit-dir requires a value", file=sys.stderr)
+                return 2
+            tk_dir = Path(argv[idx])
+        elif arg.startswith("-"):
+            print(f"ERROR: unknown option: {arg}", file=sys.stderr)
+            return 2
+        else:
+            tk_dir = Path(arg)
+        idx += 1
+
+    return sync(tk_dir.resolve(), check_only)
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/scripts/validate.py

@@ -45,6 +45,7 @@ VALID_TOOLS = frozenset({
 VALID_HOOK_EVENTS = frozenset({
     # Core lifecycle
     "SessionStart", "SessionEnd", "UserPromptSubmit", "Notification",
+    "MessageDisplay",
     # Tool lifecycle
     "PreToolUse", "PostToolUse", "PostToolUseFailure", "PostToolBatch",
     # Turn lifecycle
@@ -60,7 +61,7 @@ VALID_HOOK_EVENTS = frozenset({
     "TaskCreated", "TaskCompleted", "TeammateIdle",
     # Worktrees & environment
     "WorktreeCreate", "WorktreeRemove",
-    "CwdChanged", "FileChanged", "ConfigChange",
+    "CwdChanged", "FileChanged", "ConfigChange", "DirectoryAdded",
     # Setup / bootstrap
     "Setup", "InstructionsLoaded",
 })
@@ -107,6 +108,19 @@ VALID_KB_CATEGORIES = frozenset({
     "decisions", "runbooks", "planning",
 })
 
+# Skill body budget, in bytes after the frontmatter block.
+#
+# The body loads in full every time a trigger matches, including when it matches
+# by accident. Detail that only some runs need belongs in the skill's reference/
+# directory, reached from a pointer in the body.
+#
+# Ratchet: lower WARN by 2_000 each release until it reaches 12_000. Never lower a
+# threshold in the same change that something violates it — split the skill first,
+# then tighten. The step lives in kb/procedures/release-preparation-sop.md so it
+# does not rot as a comment nobody reads.
+SKILL_BODY_BUDGET_ERROR = 20_000
+SKILL_BODY_BUDGET_WARN = 18_000
+
 VALID_RULE_CATEGORIES = frozenset({
     "coding-style",
     "testing",
@@ -226,6 +240,22 @@ def _body_line_count(filepath: Path) -> int:
     return count
 
 
+def _body_bytes(filepath: Path) -> int:
+    """Byte size of the body — everything after the frontmatter block.
+
+    Unlike _body_line_count this anchors on the frontmatter delimiters at the top
+    of the file, so a `---` horizontal rule inside the body does not shift the
+    measurement.
+    """
+    text = filepath.read_text(encoding="utf-8")
+    lines = text.splitlines(keepends=True)
+    if lines and lines[0].strip() == "---":
+        for idx in range(1, len(lines)):
+            if lines[idx].strip() == "---":
+                return len("".join(lines[idx + 1:]).encode("utf-8"))
+    return len(text.encode("utf-8"))
+
+
 def _body_nonblank_count(filepath: Path) -> int:
     """Count non-blank lines after the second --- delimiter."""
     count = 0
@@ -332,11 +362,88 @@ def _validate_skill_frontmatter(tk_dir: Path, skill_path: Path,
     if body_lines > 500:
         vr.warn(f"skills/{name}/SKILL.md: body is {body_lines} lines (recommended < 500)")
 
+    body_bytes = _body_bytes(skill_file)
+    if body_bytes > SKILL_BODY_BUDGET_ERROR:
+        vr.error(
+            f"skills/{name}/SKILL.md: body is {body_bytes} bytes "
+            f"(limit {SKILL_BODY_BUDGET_ERROR}) - move detail into reference/"
+        )
+    elif body_bytes > SKILL_BODY_BUDGET_WARN:
+        vr.warn(
+            f"skills/{name}/SKILL.md: body is {body_bytes} bytes "
+            f"(budget {SKILL_BODY_BUDGET_WARN}) - move detail into reference/"
+        )
+
     desc_value = _fm_field(fm_lines, "description")
     if len(desc_value) > 1024:
         vr.warn(f"{name} - Description exceeds 1024 characters")
 
 
+def _validate_skill_script_invocations(skill_path: Path, vr: ValidationResult) -> None:
+    """Every documented run of a skill-owned script must resolve after install.
+
+    A skill's own scripts live beside it and are reachable only through
+    `${CLAUDE_SKILL_DIR}`. A relative path such as `python3 scripts/foo.py`
+    resolves against the user's working directory, and a repo-relative one such
+    as `app/skills/<name>/scripts/foo.py` against a tree the user does not have.
+    Both look correct in the source and fail for every installed user.
+
+    Nine skills shipped with one of those forms before this check existed.
+    `_validate_skill_references` never caught them because it asks whether the
+    file is on disk, not whether the documented command can find it.
+
+    Only executable invocations are checked. The frontmatter `scripts:` list
+    declares ownership and is correctly relative; prose that names a script in
+    backticks runs nothing.
+    """
+    name = skill_path.name
+    scripts_dir = skill_path / "scripts"
+    if not scripts_dir.is_dir():
+        return
+    owned = {p.name for p in scripts_dir.iterdir() if p.is_file()}
+    if not owned:
+        return
+
+    skill_file = skill_path / "SKILL.md"
+    text = skill_file.read_text(encoding="utf-8")
+
+    # Skip the frontmatter block: `scripts:` entries there are declarations.
+    lines = text.splitlines()
+    start = 0
+    if lines and lines[0].strip() == "---":
+        for idx in range(1, len(lines)):
+            if lines[idx].strip() == "---":
+                start = idx + 1
+                break
+
+    interpreters = {"python3": {".py"}, "python": {".py"}, "bash": {".sh"}, "sh": {".sh"}}
+    invocation = re.compile(r"\b(python3|python|bash|sh)\s+(\S+)")
+
+    for offset, line in enumerate(lines[start:], start=start + 1):
+        for interp, target in invocation.findall(line):
+            base = target.rsplit("/", 1)[-1].strip("\"'`")
+            if base not in owned:
+                continue  # a repo-level script such as scripts/validate.py
+            if "${CLAUDE_SKILL_DIR}" not in target:
+                vr.error(
+                    f"skills/{name}/SKILL.md:{offset}: runs its own '{base}' via "
+                    f"'{target}' - use ${{CLAUDE_SKILL_DIR}}/scripts/{base}, which "
+                    f"is the only path that resolves after install"
+                )
+            suffix = base[base.rfind("."):] if "." in base else ""
+            allowed = interpreters.get(interp, set())
+            if suffix and allowed and suffix not in allowed:
+                vr.error(
+                    f"skills/{name}/SKILL.md:{offset}: runs '{base}' with "
+                    f"'{interp}' - wrong interpreter for a {suffix} file"
+                )
+            elif interp == "python":
+                vr.warn(
+                    f"skills/{name}/SKILL.md:{offset}: uses 'python' for '{base}' "
+                    f"- prefer 'python3', 'python' is absent or Python 2 on many systems"
+                )
+
+
 def _validate_skill_references(tk_dir: Path, skill_path: Path,
                                fm_lines: list[str], vr: ValidationResult) -> None:
     """Validate agent refs, depends-on, context/agent co-occurrence, and reference links."""
@@ -370,6 +477,8 @@ def _validate_skill_references(tk_dir: Path, skill_path: Path,
     if _fm_has(fm_lines, "run-mode"):
         vr.warn(f"{name} - Uses deprecated 'run-mode' field (rename to 'context:')")
 
+    _validate_skill_script_invocations(skill_path, vr)
+
     ref_dir = skill_path / "reference"
     if ref_dir.is_dir():
         content = skill_file.read_text(encoding="utf-8")
@@ -384,6 +493,7 @@ def validate_skills(tk_dir: Path, vr: ValidationResult) -> int:
     print("## Skills")
     skills_dir = tk_dir / "app" / "skills"
     skill_count = 0
+    largest_body = (0, "")
 
     if not skills_dir.is_dir():
         vr.error("app/skills directory not found")
@@ -408,8 +518,15 @@ def validate_skills(tk_dir: Path, vr: ValidationResult) -> int:
         fm_lines = _parse_frontmatter_lines(skill_file)
         _validate_skill_frontmatter(tk_dir, skill_path, fm_lines, vr)
         _validate_skill_references(tk_dir, skill_path, fm_lines, vr)
+        largest_body = max(largest_body, (_body_bytes(skill_file), name))
 
     print(f"  Found: {skill_count} skills")
+    if skill_count:
+        size, worst = largest_body
+        print(
+            f"  Body budget: largest is {worst} at {size} bytes "
+            f"(warn {SKILL_BODY_BUDGET_WARN}, error {SKILL_BODY_BUDGET_ERROR})"
+        )
     print()
     return skill_count