@softspark/ai-toolkit 4.20.0 → 4.22.0

package/scripts/surface_manifest.py

@@ -0,0 +1,246 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024-2026 Lukasz Krzemien (biuro@softspark.eu)
+# Source: https://github.com/softspark/ai-toolkit
+
+"""Public surface manifest — the enforceable half of BACKWARD_COMPATIBILITY.md.
+
+`BACKWARD_COMPATIBILITY.md` names the surfaces users depend on. Prose stops
+nothing: rename a skill and every gate stays green. This turns the list into a
+committed snapshot and a test.
+
+The check is deliberately one-directional:
+
+    entry in the manifest, missing from the tree  -> FAIL   (a removal)
+    entry in the tree, missing from the manifest  -> pass   (an addition)
+
+Additions are free because a surface nobody has installed yet has no users to
+break. Removals fail because someone out there pinned the version that had it.
+
+The manifest is NOT regenerated automatically. If it were, deleting a skill would
+delete its manifest entry in the same breath and the check would detect nothing.
+`--update` is a deliberate act: run it, read the diff, and any line that
+disappeared is a breaking change owing an entry in DECISIONS.md.
+
+Stdlib-only.
+
+Usage:
+    python3 scripts/surface_manifest.py            # check, human-readable
+    python3 scripts/surface_manifest.py --json     # check, machine-readable
+    python3 scripts/surface_manifest.py --update   # rewrite app/surface.json
+    python3 scripts/surface_manifest.py --toolkit-dir /path
+
+Exit codes:
+    0  no protected surface was removed
+    1  a removal detected (or the manifest is missing on a check run)
+    2  usage error
+"""
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _common import toolkit_dir as default_toolkit_dir
+
+MANIFEST_RELPATH = Path("app") / "surface.json"
+
+FM_FIELD_RE = re.compile(r"^([a-z][a-z0-9-]*):")
+CLI_COMMAND_RE = re.compile(r"^\s+'?([a-z][a-z0-9-]*)'?\s*:")
+
+
+def _frontmatter_fields(path: Path) -> set[str]:
+    parts = path.read_text(encoding="utf-8").split("---")
+    if len(parts) < 3:
+        return set()
+    return {
+        m.group(1)
+        for line in parts[1].splitlines()
+        if (m := FM_FIELD_RE.match(line))
+    }
+
+
+def collect_surface(tk_dir: Path) -> dict:
+    """Read the current public surface out of the tree."""
+    skills_dir = tk_dir / "app" / "skills"
+    agents_dir = tk_dir / "app" / "agents"
+
+    skills: list[str] = []
+    skill_fields: set[str] = set()
+    for d in sorted(skills_dir.iterdir()) if skills_dir.is_dir() else []:
+        if not d.is_dir() or d.name.startswith("_") or not (d / "SKILL.md").is_file():
+            continue
+        skills.append(d.name)
+        skill_fields |= _frontmatter_fields(d / "SKILL.md")
+
+    agents: list[str] = []
+    agent_fields: set[str] = set()
+    for f in sorted(agents_dir.glob("*.md")) if agents_dir.is_dir() else []:
+        agents.append(f.stem)
+        agent_fields |= _frontmatter_fields(f)
+
+    hooks_dir = tk_dir / "app" / "hooks"
+    hook_scripts = sorted(
+        f.name for f in hooks_dir.glob("*.sh") if not f.name.startswith("_")
+    ) if hooks_dir.is_dir() else []
+
+    hook_events: list[str] = []
+    hooks_json = tk_dir / "app" / "hooks.json"
+    if hooks_json.is_file():
+        try:
+            hook_events = sorted(json.loads(hooks_json.read_text(encoding="utf-8")).get("hooks", {}))
+        except (json.JSONDecodeError, OSError):
+            hook_events = []
+
+    plugins_dir = tk_dir / "app" / "plugins"
+    packs = sorted(
+        d.name for d in plugins_dir.iterdir()
+        if d.is_dir() and not d.name.startswith(".")
+    ) if plugins_dir.is_dir() else []
+
+    kb_categories: list[str] = []
+    validate_py = tk_dir / "scripts" / "validate.py"
+    if validate_py.is_file():
+        text = validate_py.read_text(encoding="utf-8")
+        block = re.search(r"VALID_KB_CATEGORIES\s*=\s*frozenset\(\{(.*?)\}\)", text, re.S)
+        if block:
+            kb_categories = sorted(re.findall(r'"([a-z-]+)"', block.group(1)))
+
+    cli_commands: list[str] = []
+    cli = tk_dir / "bin" / "ai-toolkit.js"
+    if not cli.is_file():
+        candidates = sorted((tk_dir / "bin").glob("*.js")) if (tk_dir / "bin").is_dir() else []
+        cli = candidates[0] if candidates else cli
+    if cli.is_file():
+        # The COMMANDS map is what `--help` prints, so it is the published surface.
+        # An undocumented internal branch is not a promise and is not captured here.
+        block = re.search(r"const COMMANDS\s*=\s*\{(.*?)\n\};", cli.read_text(encoding="utf-8"), re.S)
+        if block:
+            cli_commands = sorted({
+                m.group(1) for line in block.group(1).splitlines()
+                if (m := CLI_COMMAND_RE.match(line))
+            })
+
+    return {
+        "skills": skills,
+        "agents": agents,
+        "skill_frontmatter_fields": sorted(skill_fields),
+        "agent_frontmatter_fields": sorted(agent_fields),
+        "hook_scripts": hook_scripts,
+        "hook_events": hook_events,
+        "plugin_packs": packs,
+        "kb_categories": kb_categories,
+        "cli_commands": cli_commands,
+    }
+
+
+def compare(manifest: dict, current: dict) -> dict:
+    """Removals fail, additions pass. See the module docstring for why."""
+    removed: dict[str, list[str]] = {}
+    added: dict[str, list[str]] = {}
+    for key, entries in manifest.items():
+        if key.startswith("_"):
+            continue
+        have = set(current.get(key, []))
+        gone = [e for e in entries if e not in have]
+        if gone:
+            removed[key] = gone
+        new = [e for e in current.get(key, []) if e not in set(entries)]
+        if new:
+            added[key] = new
+    return {"removed": removed, "added": added, "ok": not removed}
+
+
+def _write_manifest(path: Path, current: dict) -> None:
+    payload = {
+        "_comment": (
+            "Public surface snapshot. Removing an entry is a breaking change: see "
+            "BACKWARD_COMPATIBILITY.md, and record it in DECISIONS.md. Regenerate "
+            "deliberately with `python3 scripts/surface_manifest.py --update` and "
+            "read the diff — never as a reflex to green a red build."
+        ),
+        **current,
+    }
+    path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+
+def main(argv: list[str]) -> int:
+    update = False
+    as_json = False
+    tk_dir = default_toolkit_dir
+
+    idx = 0
+    while idx < len(argv):
+        arg = argv[idx]
+        if arg == "--update":
+            update = True
+        elif arg == "--json":
+            as_json = True
+        elif arg in ("-h", "--help"):
+            print(__doc__)
+            return 0
+        elif arg == "--toolkit-dir":
+            idx += 1
+            if idx >= len(argv):
+                print("ERROR: --toolkit-dir requires a value", file=sys.stderr)
+                return 2
+            tk_dir = Path(argv[idx])
+        elif arg.startswith("-"):
+            print(f"ERROR: unknown option: {arg}", file=sys.stderr)
+            return 2
+        else:
+            tk_dir = Path(arg)
+        idx += 1
+
+    tk_dir = tk_dir.resolve()
+    manifest_path = tk_dir / MANIFEST_RELPATH
+    current = collect_surface(tk_dir)
+
+    if update:
+        _write_manifest(manifest_path, current)
+        total = sum(len(v) for v in current.values())
+        print(f"Wrote {manifest_path.relative_to(tk_dir)} — {total} protected entries")
+        print("Read the diff. Every line removed is a breaking change.")
+        return 0
+
+    if not manifest_path.is_file():
+        print(f"ERROR: {MANIFEST_RELPATH} not found — run with --update to create it",
+              file=sys.stderr)
+        return 1
+
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    result = compare(manifest, current)
+
+    if as_json:
+        print(json.dumps(result, indent=2, ensure_ascii=False))
+        return 0 if result["ok"] else 1
+
+    print("## Public Surface")
+    if result["removed"]:
+        for key, entries in sorted(result["removed"].items()):
+            for entry in entries:
+                print(f"  ERROR: {key}: '{entry}' was removed from the public surface")
+        print()
+        print("A removal is a breaking change. Either restore it, or take the")
+        print("deprecation path in BACKWARD_COMPATIBILITY.md, record it in")
+        print("DECISIONS.md, and re-run with --update.")
+        print()
+        print("SURFACE CHECK FAILED")
+        return 1
+
+    protected = sum(len(v) for k, v in manifest.items() if not k.startswith("_"))
+    print(f"  OK: {protected} protected entries intact")
+    if result["added"]:
+        new_total = sum(len(v) for v in result["added"].values())
+        detail = ", ".join(f"{k} +{len(v)}" for k, v in sorted(result["added"].items()))
+        print(f"  Note: {new_total} new entries not yet protected ({detail})")
+        print("        Run --update before the next release to adopt them.")
+    print()
+    print("SURFACE CHECK PASSED")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/scripts/sync_badges.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024-2026 Lukasz Krzemien (biuro@softspark.eu)
+# Source: https://github.com/softspark/ai-toolkit
+
+"""Sync the README count badges to ground truth.
+
+`validate.py` errors when a README badge disagrees with the tree. The badges were
+maintained by hand, so every change that added a test or a skill turned the build
+red until someone edited a number — a tax paid per change, with a diagnosis cycle
+each time it was forgotten.
+
+This derives the three counts the same way `validate.py` does and rewrites them.
+It runs inside `npm run generate:all`, which `prepublishOnly` executes before
+`validate.py --strict`, so the badges are correct by the time anything checks them.
+
+Counting rules, matched to validate.py so the two cannot disagree:
+  agents  — *.md under app/agents/
+  skills  — directories under app/skills/ with a SKILL.md, excluding _-prefixed
+  tests   — `@test ` declarations across tests/*.bats
+
+Stdlib-only.
+
+Usage:
+    python3 scripts/sync_badges.py            # rewrite README.md badges
+    python3 scripts/sync_badges.py --check    # report drift, change nothing
+    python3 scripts/sync_badges.py --toolkit-dir /path
+
+Exit codes:
+    0  badges are correct (or were rewritten)
+    1  --check found drift
+    2  usage error
+"""
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _common import toolkit_dir as default_toolkit_dir
+
+TEST_RE = re.compile(r"^@test ", re.MULTILINE)
+
+
+def counts(tk_dir: Path) -> dict[str, int]:
+    agents_dir = tk_dir / "app" / "agents"
+    skills_dir = tk_dir / "app" / "skills"
+    tests_dir = tk_dir / "tests"
+
+    agents = len(list(agents_dir.glob("*.md"))) if agents_dir.is_dir() else 0
+    skills = sum(
+        1 for d in skills_dir.iterdir()
+        if d.is_dir() and not d.name.startswith("_") and (d / "SKILL.md").is_file()
+    ) if skills_dir.is_dir() else 0
+    tests = sum(
+        len(TEST_RE.findall(f.read_text(encoding="utf-8")))
+        for f in tests_dir.glob("*.bats")
+    ) if tests_dir.is_dir() else 0
+
+    return {"agents": agents, "skills": skills, "tests": tests}
+
+
+def sync(tk_dir: Path, check_only: bool) -> int:
+    readme = tk_dir / "README.md"
+    if not readme.is_file():
+        print("ERROR: README.md not found", file=sys.stderr)
+        return 2
+
+    text = readme.read_text(encoding="utf-8")
+    actual = counts(tk_dir)
+    drift: list[str] = []
+    updated = text
+
+    for label, value in actual.items():
+        pattern = re.compile(rf"({label}-)(\d+)")
+        match = pattern.search(updated)
+        if not match:
+            continue
+        if match.group(2) != str(value):
+            drift.append(f"{label}: badge {match.group(2)} != actual {value}")
+            updated = pattern.sub(rf"\g<1>{value}", updated, count=1)
+
+    if check_only:
+        for line in drift:
+            print(f"  DRIFT: {line}")
+        if drift:
+            print("BADGE CHECK FAILED — run `python3 scripts/sync_badges.py`")
+            return 1
+        print(f"  OK: badges match ({actual['agents']} agents, "
+              f"{actual['skills']} skills, {actual['tests']} tests)")
+        return 0
+
+    if drift:
+        readme.write_text(updated, encoding="utf-8")
+        for line in drift:
+            print(f"  synced {line}")
+    else:
+        print(f"  OK: badges already match ({actual['agents']} agents, "
+              f"{actual['skills']} skills, {actual['tests']} tests)")
+    return 0
+
+
+def main(argv: list[str]) -> int:
+    check_only = False
+    tk_dir = default_toolkit_dir
+
+    idx = 0
+    while idx < len(argv):
+        arg = argv[idx]
+        if arg == "--check":
+            check_only = True
+        elif arg in ("-h", "--help"):
+            print(__doc__)
+            return 0
+        elif arg == "--toolkit-dir":
+            idx += 1
+            if idx >= len(argv):
+                print("ERROR: --toolkit-dir requires a value", file=sys.stderr)
+                return 2
+            tk_dir = Path(argv[idx])
+        elif arg.startswith("-"):
+            print(f"ERROR: unknown option: {arg}", file=sys.stderr)
+            return 2
+        else:
+            tk_dir = Path(arg)
+        idx += 1
+
+    return sync(tk_dir.resolve(), check_only)
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/scripts/validate.py

@@ -45,6 +45,7 @@ VALID_TOOLS = frozenset({
 VALID_HOOK_EVENTS = frozenset({
     # Core lifecycle
     "SessionStart", "SessionEnd", "UserPromptSubmit", "Notification",
+    "MessageDisplay",
     # Tool lifecycle
     "PreToolUse", "PostToolUse", "PostToolUseFailure", "PostToolBatch",
     # Turn lifecycle
@@ -60,7 +61,7 @@ VALID_HOOK_EVENTS = frozenset({
     "TaskCreated", "TaskCompleted", "TeammateIdle",
     # Worktrees & environment
     "WorktreeCreate", "WorktreeRemove",
-    "CwdChanged", "FileChanged", "ConfigChange",
+    "CwdChanged", "FileChanged", "ConfigChange", "DirectoryAdded",
     # Setup / bootstrap
     "Setup", "InstructionsLoaded",
 })
@@ -95,11 +96,31 @@ HOOK_REQUIRED_FIELDS = {
     "mcp_tool": ("server", "tool", "arguments"),
 }
 
+# The taxonomy. `app/skills/documentation-standards/SKILL.md` documents the same
+# eight and is what authors read; this set is what rejects a typo. They are one
+# list in two places, so a change belongs in both.
+#
+# `decisions` and `runbooks` were missing until v4.21.0 while the kb-migration
+# SOP had been telling people to create those directories for months, so a
+# correctly-filed ADR failed validation.
 VALID_KB_CATEGORIES = frozenset({
     "reference", "howto", "procedures", "troubleshooting", "best-practices",
-    "planning",
+    "decisions", "runbooks", "planning",
 })
 
+# Skill body budget, in bytes after the frontmatter block.
+#
+# The body loads in full every time a trigger matches, including when it matches
+# by accident. Detail that only some runs need belongs in the skill's reference/
+# directory, reached from a pointer in the body.
+#
+# Ratchet: lower WARN by 2_000 each release until it reaches 12_000. Never lower a
+# threshold in the same change that something violates it — split the skill first,
+# then tighten. The step lives in kb/procedures/release-preparation-sop.md so it
+# does not rot as a comment nobody reads.
+SKILL_BODY_BUDGET_ERROR = 20_000
+SKILL_BODY_BUDGET_WARN = 18_000
+
 VALID_RULE_CATEGORIES = frozenset({
     "coding-style",
     "testing",
@@ -219,6 +240,22 @@ def _body_line_count(filepath: Path) -> int:
     return count
 
 
+def _body_bytes(filepath: Path) -> int:
+    """Byte size of the body — everything after the frontmatter block.
+
+    Unlike _body_line_count this anchors on the frontmatter delimiters at the top
+    of the file, so a `---` horizontal rule inside the body does not shift the
+    measurement.
+    """
+    text = filepath.read_text(encoding="utf-8")
+    lines = text.splitlines(keepends=True)
+    if lines and lines[0].strip() == "---":
+        for idx in range(1, len(lines)):
+            if lines[idx].strip() == "---":
+                return len("".join(lines[idx + 1:]).encode("utf-8"))
+    return len(text.encode("utf-8"))
+
+
 def _body_nonblank_count(filepath: Path) -> int:
     """Count non-blank lines after the second --- delimiter."""
     count = 0
@@ -325,6 +362,18 @@ def _validate_skill_frontmatter(tk_dir: Path, skill_path: Path,
     if body_lines > 500:
         vr.warn(f"skills/{name}/SKILL.md: body is {body_lines} lines (recommended < 500)")
 
+    body_bytes = _body_bytes(skill_file)
+    if body_bytes > SKILL_BODY_BUDGET_ERROR:
+        vr.error(
+            f"skills/{name}/SKILL.md: body is {body_bytes} bytes "
+            f"(limit {SKILL_BODY_BUDGET_ERROR}) - move detail into reference/"
+        )
+    elif body_bytes > SKILL_BODY_BUDGET_WARN:
+        vr.warn(
+            f"skills/{name}/SKILL.md: body is {body_bytes} bytes "
+            f"(budget {SKILL_BODY_BUDGET_WARN}) - move detail into reference/"
+        )
+
     desc_value = _fm_field(fm_lines, "description")
     if len(desc_value) > 1024:
         vr.warn(f"{name} - Description exceeds 1024 characters")
@@ -377,6 +426,7 @@ def validate_skills(tk_dir: Path, vr: ValidationResult) -> int:
     print("## Skills")
     skills_dir = tk_dir / "app" / "skills"
     skill_count = 0
+    largest_body = (0, "")
 
     if not skills_dir.is_dir():
         vr.error("app/skills directory not found")
@@ -401,8 +451,15 @@ def validate_skills(tk_dir: Path, vr: ValidationResult) -> int:
         fm_lines = _parse_frontmatter_lines(skill_file)
         _validate_skill_frontmatter(tk_dir, skill_path, fm_lines, vr)
         _validate_skill_references(tk_dir, skill_path, fm_lines, vr)
+        largest_body = max(largest_body, (_body_bytes(skill_file), name))
 
     print(f"  Found: {skill_count} skills")
+    if skill_count:
+        size, worst = largest_body
+        print(
+            f"  Body budget: largest is {worst} at {size} bytes "
+            f"(warn {SKILL_BODY_BUDGET_WARN}, error {SKILL_BODY_BUDGET_ERROR})"
+        )
     print()
     return skill_count
 
@@ -730,6 +787,31 @@ def validate_kb_documents(tk_dir: Path, vr: ValidationResult) -> None:
             vr.error(f"{kb_name} - Invalid category '{kb_category}' (valid: {', '.join(sorted(VALID_KB_CATEGORIES))})")
             kb_errors += 1
 
+        # `section` is a legacy alias for `category`, not a second axis. A
+        # document carrying both with different values is indexed twice and
+        # found once, and nothing about that is visible to its author.
+        kb_section = _fm_field(fm_lines, "section").strip()
+        if kb_section and kb_category and kb_section != kb_category:
+            vr.error(
+                f"{kb_name} - section '{kb_section}' disagrees with category "
+                f"'{kb_category}'; section is an alias and must match"
+            )
+            kb_errors += 1
+
+        # A document filed under a taxonomy directory must declare that
+        # category, or a browsing reader and a search hit disagree about what it
+        # is. The rule is scoped to directories that ARE category names on
+        # purpose: `kb/history/completed/` is a lifecycle location, not a type,
+        # and a finished plan is still a `planning` document.
+        relative = kb_file.relative_to(kb_dir).parts
+        top = relative[0] if len(relative) > 1 else ""
+        if kb_category and top in VALID_KB_CATEGORIES and top != kb_category:
+            vr.error(
+                f"{kb_name} - is in {top}/ but declares category "
+                f"'{kb_category}'; they must agree"
+            )
+            kb_errors += 1
+
         # Validate tags is not empty
         tags_val = _fm_field(fm_lines, "tags")
         if tags_val: