@softspark/ai-toolkit 2.7.3 → 2.9.0

package/scripts/audit_skills.py

@@ -7,9 +7,11 @@ Detects dangerous code patterns, hardcoded secrets, and permission issues.
 Stdlib-only. JSON output to stdout. Non-zero exit on HIGH findings.
 
 Usage:
-    python3 scripts/audit_skills.py [toolkit-dir]          # scan all
-    python3 scripts/audit_skills.py [toolkit-dir] --json   # JSON output
-    python3 scripts/audit_skills.py [toolkit-dir] --ci     # exit 1 on HIGH
+    python3 scripts/audit_skills.py [toolkit-dir]               # scan all
+    python3 scripts/audit_skills.py [toolkit-dir] --json        # JSON output
+    python3 scripts/audit_skills.py [toolkit-dir] --sarif       # SARIF 2.1.0 output
+    python3 scripts/audit_skills.py [toolkit-dir] --permissions # per-skill tool permission report
+    python3 scripts/audit_skills.py [toolkit-dir] --ci          # exit 1 on HIGH
 
 Exit codes:
     0  no HIGH findings
@@ -71,6 +73,13 @@ SECRET_PATTERNS = [
     (r'api_key\s*=\s*["\'][^"\']{8,}["\']', "WARN", "Hardcoded API key"),
 ]
 
+# Placeholder values that the WARN-severity secret patterns must ignore to cut
+# false positives on docs, fixtures, and .env.example-style files.
+SECRET_PLACEHOLDER_PREFIXES = (
+    "REPLACE_", "CHANGEME_", "CHANGE_ME", "YOUR_", "EXAMPLE_", "PLACEHOLDER_",
+    "${", "{{", "$ENV_", "$(", "<", "xxx", "XXX",
+)
+
 # ---------------------------------------------------------------------------
 # Scanner
 # ---------------------------------------------------------------------------
@@ -110,6 +119,16 @@ def scan_file_patterns(filepath: Path, patterns: list[tuple],
                 findings.append(Finding(severity, rel, lineno, regex, desc))
 
 
+def _is_placeholder_value(match: re.Match) -> bool:
+    """Return True if the matched value looks like a docs placeholder."""
+    # Extract content between quotes (group 0 is the full match)
+    inner = re.search(r'["\']([^"\']+)["\']', match.group(0))
+    if not inner:
+        return False
+    value = inner.group(1)
+    return value.startswith(SECRET_PLACEHOLDER_PREFIXES)
+
+
 def scan_secrets(filepath: Path, findings: list[Finding]) -> None:
     """Scan a file for hardcoded secrets."""
     try:
@@ -119,8 +138,80 @@ def scan_secrets(filepath: Path, findings: list[Finding]) -> None:
     rel = str(filepath)
     for lineno, line in enumerate(text.splitlines(), 1):
         for regex, severity, desc in SECRET_PATTERNS:
-            if re.search(regex, line):
-                findings.append(Finding(severity, rel, lineno, regex, desc))
+            m = re.search(regex, line)
+            if not m:
+                continue
+            # Skip placeholder values for WARN-level hardcoded-* patterns.
+            # HIGH-level patterns (real AWS/GitHub/etc. keys) are structural —
+            # no need to allowlist.
+            if severity == "WARN" and _is_placeholder_value(m):
+                continue
+            findings.append(Finding(severity, rel, lineno, regex, desc))
+
+
+# Description quality — per Anthropic docs (code.claude.com/docs/en/skills.md):
+# description + when_to_use combined ≤ 1536 chars; first sentence carries
+# the trigger keywords that let the LLM route to this skill.
+DESCRIPTION_MAX_CHARS = 1536
+DESCRIPTION_MIN_CHARS = 80
+
+# Weak patterns — fail-fast on the historical "Loaded when user asks about X"
+# shape that carries no action verb and no concrete keywords.
+WEAK_DESCRIPTION_PATTERNS = [
+    (r'^Loaded when user asks about ', "starts with 'Loaded when user asks about' — no action verb, no trigger keywords"),
+    (r'^Loaded when user asks to ', "starts with 'Loaded when user asks to' — no action verb, no trigger keywords"),
+]
+
+
+def check_description(skill_md: Path, findings: list[Finding]) -> None:
+    """Check that SKILL.md description is specific enough for auto-loading.
+
+    Only enforces for knowledge skills (user-invocable: false). Task skills
+    invoked via `/name` do not rely on description for routing.
+    """
+    user_invocable = frontmatter_field(skill_md, "user-invocable")
+    disable_model = frontmatter_field(skill_md, "disable-model-invocation")
+
+    # Skip strictly-task skills — description is a menu label, not a trigger.
+    if disable_model == "true" and user_invocable != "false":
+        return
+
+    desc = frontmatter_field(skill_md, "description")
+    when = frontmatter_field(skill_md, "when_to_use")
+    combined_len = len(desc) + len(when)
+    rel = str(skill_md)
+
+    if not desc:
+        findings.append(Finding(
+            "WARN", rel, 0, "missing-description",
+            "Skill has no description — auto-loaded skills need one to be routable",
+        ))
+        return
+
+    if combined_len > DESCRIPTION_MAX_CHARS:
+        findings.append(Finding(
+            "WARN", rel, 0, "description-too-long",
+            f"description + when_to_use is {combined_len} chars (limit {DESCRIPTION_MAX_CHARS}). "
+            "Anthropic truncates past this — trigger keywords may be lost.",
+        ))
+
+    # Knowledge skills must have routable descriptions.
+    if user_invocable == "false":
+        if len(desc) < DESCRIPTION_MIN_CHARS:
+            findings.append(Finding(
+                "WARN", rel, 0, "description-too-short",
+                f"Knowledge skill description is {len(desc)} chars — needs "
+                f"≥{DESCRIPTION_MIN_CHARS} with concrete trigger keywords",
+            ))
+
+        for regex, reason in WEAK_DESCRIPTION_PATTERNS:
+            if re.match(regex, desc):
+                findings.append(Finding(
+                    "WARN", rel, 0, "description-weak-pattern",
+                    f"Weak description: {reason}. "
+                    "Use the shape '[capability]. Triggers: [keywords]. Load when [...].'",
+                ))
+                break
 
 
 def check_frontmatter(skill_dir: Path, findings: list[Finding]) -> None:
@@ -130,6 +221,9 @@ def check_frontmatter(skill_dir: Path, findings: list[Finding]) -> None:
         return
     rel = str(skill_md)
 
+    # Description quality (routability)
+    check_description(skill_md, findings)
+
     allowed = frontmatter_field(skill_md, "allowed-tools")
     user_invocable = frontmatter_field(skill_md, "user-invocable")
     disable_model = frontmatter_field(skill_md, "disable-model-invocation")
@@ -176,6 +270,78 @@ def check_agent(agent_md: Path, findings: list[Finding]) -> None:
         ))
 
 
+# ---------------------------------------------------------------------------
+# Per-skill permission report
+# ---------------------------------------------------------------------------
+
+def collect_permissions(toolkit_root: Path) -> list[dict]:
+    """Read each SKILL.md frontmatter and return permission metadata per skill."""
+    skills = toolkit_root / "app" / "skills"
+    rows: list[dict] = []
+    if not skills.is_dir():
+        return rows
+    for skill_dir in sorted(skills.iterdir()):
+        if not skill_dir.is_dir() or skill_dir.name.startswith("_"):
+            continue
+        skill_md = skill_dir / "SKILL.md"
+        if not skill_md.is_file():
+            continue
+        allowed_raw = frontmatter_field(skill_md, "allowed-tools") or ""
+        tools = [t.strip() for t in allowed_raw.split(",") if t.strip()]
+        rows.append({
+            "name": skill_dir.name,
+            "tools": tools,
+            "user_invocable": frontmatter_field(skill_md, "user-invocable") or "",
+            "disable_model_invocation": frontmatter_field(
+                skill_md, "disable-model-invocation"
+            ) or "",
+        })
+    return rows
+
+
+def print_permissions(rows: list[dict], json_mode: bool = False) -> None:
+    """Emit the per-skill permission report."""
+    # Aggregate tool usage counts
+    by_tool: dict[str, list[str]] = {}
+    for row in rows:
+        for tool in row["tools"]:
+            by_tool.setdefault(tool, []).append(row["name"])
+    broad = [
+        row["name"] for row in rows
+        if {"Bash", "Write", "Edit"}.issubset(set(row["tools"]))
+    ]
+
+    if json_mode:
+        report = {
+            "total": len(rows),
+            "by_tool": {k: sorted(v) for k, v in by_tool.items()},
+            "broad_access": sorted(broad),
+            "skills": rows,
+        }
+        print(json.dumps(report, indent=2))
+        return
+
+    print("Skill Permissions Report")
+    print("=" * 40)
+    print(f"Total skills: {len(rows)}")
+    print()
+    print("By tool (skill count):")
+    for tool in sorted(by_tool, key=lambda t: (-len(by_tool[t]), t)):
+        print(f"  {tool:<12} {len(by_tool[tool])}")
+    print()
+    if broad:
+        print(f"Skills with Bash + Write + Edit ({len(broad)}):")
+        for name in sorted(broad):
+            print(f"  - {name}")
+        print()
+    print("Full table:")
+    print(f"  {'skill':<32} {'invocable':<10} {'tools'}")
+    for row in rows:
+        inv = row["user_invocable"] or "-"
+        tools = ",".join(row["tools"]) or "(none declared)"
+        print(f"  {row['name']:<32} {inv:<10} {tools}")
+
+
 # ---------------------------------------------------------------------------
 # Main
 # ---------------------------------------------------------------------------
@@ -260,23 +426,97 @@ def print_json(findings: list[Finding]) -> None:
     print(json.dumps(report, indent=2))
 
 
+# SARIF severity maps to GitHub Advanced Security Code Scanning levels.
+_SARIF_LEVEL = {"HIGH": "error", "WARN": "warning", "INFO": "note"}
+
+
+def _sarif_rules(findings: list[Finding]) -> list[dict]:
+    """Build the tool.driver.rules array from unique (severity, description) pairs."""
+    seen: dict[str, dict] = {}
+    for f in findings:
+        rule_id = f"{f.severity}-{hash(f.description) & 0xFFFFFFFF:08x}"
+        if rule_id in seen:
+            continue
+        seen[rule_id] = {
+            "id": rule_id,
+            "name": f.description.split(" — ")[0].replace(" ", "-").lower()[:64],
+            "shortDescription": {"text": f.description},
+            "defaultConfiguration": {"level": _SARIF_LEVEL.get(f.severity, "note")},
+        }
+    return list(seen.values())
+
+
+def print_sarif(findings: list[Finding], toolkit_root: Path) -> None:
+    """Print SARIF 2.1.0 report for GitHub Code Scanning ingestion."""
+    rules = _sarif_rules(findings)
+    rule_index = {r["shortDescription"]["text"]: i for i, r in enumerate(rules)}
+    results = []
+    for f in findings:
+        idx = rule_index.get(f.description, 0)
+        rule_id = rules[idx]["id"] if rules else "unknown"
+        try:
+            rel = str(Path(f.file).resolve().relative_to(toolkit_root.resolve()))
+        except ValueError:
+            rel = f.file
+        results.append({
+            "ruleId": rule_id,
+            "ruleIndex": idx,
+            "level": _SARIF_LEVEL.get(f.severity, "note"),
+            "message": {"text": f.description},
+            "locations": [{
+                "physicalLocation": {
+                    "artifactLocation": {"uri": rel},
+                    "region": {"startLine": max(1, f.line)},
+                }
+            }],
+        })
+    sarif = {
+        "$schema": "https://json.schemastore.org/sarif-2.1.0.json",
+        "version": "2.1.0",
+        "runs": [{
+            "tool": {
+                "driver": {
+                    "name": "ai-toolkit-audit-skills",
+                    "informationUri": "https://github.com/softspark/ai-toolkit",
+                    "rules": rules,
+                }
+            },
+            "results": results,
+        }],
+    }
+    print(json.dumps(sarif, indent=2))
+
+
 def main() -> None:
     args = sys.argv[1:]
     toolkit_root = default_toolkit_dir
     json_mode = False
+    sarif_mode = False
+    permissions_mode = False
     ci_mode = False
 
     for arg in args:
         if arg == "--json":
             json_mode = True
+        elif arg == "--sarif":
+            sarif_mode = True
+        elif arg == "--permissions":
+            permissions_mode = True
         elif arg == "--ci":
             ci_mode = True
         elif not arg.startswith("-"):
             toolkit_root = Path(arg)
 
+    if permissions_mode:
+        rows = collect_permissions(toolkit_root)
+        print_permissions(rows, json_mode=json_mode)
+        return
+
     findings = audit(toolkit_root)
 
-    if json_mode:
+    if sarif_mode:
+        print_sarif(findings, toolkit_root)
+    elif json_mode:
         print_json(findings)
     else:
         print_text(findings)

package/scripts/config_resolver.py

@@ -302,9 +302,12 @@ def _extract_tarball(tarball: Path, dest: Path) -> None:
     """Extract npm tarball (which has a package/ prefix) to dest.
 
     Validates that extracted paths stay within dest to prevent path traversal.
-    Rejects symlinks and absolute paths.
+    Rejects symlinks and absolute paths. Uses tarfile filter="data" on 3.12+
+    as defense in depth (Python 3.14 will require it).
     """
     dest_resolved = dest.resolve()
+    # filter="data" landed in 3.12 and becomes the default in 3.14
+    supports_filter = sys.version_info >= (3, 12)
     with tarfile.open(tarball, "r:gz") as tf:
         for member in tf.getmembers():
             # npm tarballs have a "package/" prefix
@@ -320,7 +323,10 @@ def _extract_tarball(tarball: Path, dest: Path) -> None:
             target = (dest / member.name).resolve()
             if not str(target).startswith(str(dest_resolved)):
                 continue
-            tf.extract(member, dest)
+            if supports_filter:
+                tf.extract(member, dest, filter="data")
+            else:
+                tf.extract(member, dest)
 
 
 def _extract_version_from_tarball(filename: str, package_name: str) -> str:

package/scripts/hook_sources.py

@@ -10,6 +10,7 @@ Stdlib-only — no external dependencies.
 """
 from __future__ import annotations
 
+import hashlib
 import json
 import os
 import sys
@@ -78,17 +79,43 @@ def save_sources(hooks_dir: Path | None = None,
 # CRUD
 # ---------------------------------------------------------------------------
 
-def register_url_source(hooks_dir: Path | None, hook_name: str, url: str) -> None:
-    """Add or update a URL source entry."""
+def register_url_source(
+    hooks_dir: Path | None,
+    hook_name: str,
+    url: str,
+    content: bytes | None = None,
+) -> None:
+    """Add or update a URL source entry.
+
+    When ``content`` is supplied, its sha256 is persisted. If a previous
+    sha256 exists and differs from the new one, a warning is printed
+    (and the process fails with exit 2 when ``AI_TOOLKIT_STRICT_PIN=1``).
+    """
     import re
     if not hook_name or not re.fullmatch(r"[a-zA-Z0-9_-]+", hook_name):
         raise ValueError(f"Invalid hook name: {hook_name!r}")
     hooks_dir = hooks_dir or EXTERNAL_HOOKS_DIR
     sources = load_sources(hooks_dir)
-    sources[hook_name] = {
+    entry: dict[str, Any] = {
         "url": url,
         "fetched_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
     }
+    if content is not None:
+        new_hash = hashlib.sha256(content).hexdigest()
+        prev = sources.get(hook_name) or {}
+        prev_hash = prev.get("sha256")
+        if prev_hash and prev_hash != new_hash:
+            msg = (
+                f"  CHECKSUM CHANGED: hook '{hook_name}' sha256 "
+                f"{prev_hash[:12]}... -> {new_hash[:12]}..."
+            )
+            print(msg)
+            if os.environ.get("AI_TOOLKIT_STRICT_PIN") == "1":
+                raise SystemExit(
+                    f"Refusing to update '{hook_name}' under AI_TOOLKIT_STRICT_PIN=1."
+                )
+        entry["sha256"] = new_hash
+    sources[hook_name] = entry
     save_sources(hooks_dir, sources)
 
 

package/scripts/inject_hook_cli.py

@@ -284,7 +284,7 @@ def _fetch_and_cache(url: str, source: str) -> str:
 
     cached_path = EXTERNAL_HOOKS_DIR / f"{source}.json"
     cached_path.write_bytes(data)
-    register_url_source(None, source, url)
+    register_url_source(None, source, url, content=data)
 
     return str(cached_path)
 

package/scripts/install_steps/ai_tools.py

@@ -194,7 +194,11 @@ def inject_with_rules(
     else:
         cmd = ["bash", str(scripts_dir / generator_script)]
 
-    result = subprocess.run(cmd, capture_output=True, text=True)
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
+    except subprocess.TimeoutExpired:
+        print(f"  ERROR: {generator_script} timed out after 120s")
+        return
     if result.returncode != 0:
         print(f"  ERROR: {generator_script} failed: {result.stderr.strip()}")
         return
@@ -240,7 +244,11 @@ def run_script(script_name: str, *args: str, capture: bool = False) -> str:
         cmd = ["python3", str(scripts_dir / py_name), *args]
     else:
         cmd = ["bash", str(scripts_dir / script_name), *args]
-    result = subprocess.run(cmd, capture_output=capture, text=True)
+    try:
+        result = subprocess.run(cmd, capture_output=capture, text=True, timeout=120)
+    except subprocess.TimeoutExpired:
+        print(f"  ERROR: {script_name} timed out after 120s")
+        return ""
     return result.stdout if capture else ""
 
 

package/scripts/install_steps/hooks.py

@@ -64,7 +64,7 @@ def _copy_hook_scripts(claude_dir: Path, hooks_scripts_dir: Path) -> None:
 
 def _run_merge_hooks(action: str, *args: str) -> None:
     cmd = ["python3", str(toolkit_dir / "scripts" / "merge-hooks.py"), action, *args]
-    subprocess.run(cmd, check=True)
+    subprocess.run(cmd, check=True, timeout=120)
 
 
 def _install_output_styles(claude_dir: Path) -> None:

package/scripts/install_steps/markers.py

@@ -88,7 +88,7 @@ def _refresh_url_rules(rules_dir: Path) -> None:
         try:
             data = fetch_url(url)
             rule_file.write_bytes(data)
-            register_url_source(rules_dir, rule_name, url)
+            register_url_source(rules_dir, rule_name, url, content=data)
             print(f"  Refreshed: {rule_name} (from {url})")
         except Exception as exc:
             if rule_file.is_file():
@@ -124,7 +124,7 @@ def refresh_url_hooks(target_dir: str | None = None) -> None:
             # Validate JSON before caching
             json.loads(data)
             cached_file.write_bytes(data)
-            register_url_source(None, hook_name, url)
+            register_url_source(None, hook_name, url, content=data)
             print(f"  Refreshed: {hook_name} (from {url})")
         except Exception as exc:
             if cached_file.is_file():

package/scripts/rule_sources.py

@@ -10,6 +10,7 @@ Stdlib-only — no external dependencies.
 """
 from __future__ import annotations
 
+import hashlib
 import json
 import os
 import sys
@@ -79,17 +80,43 @@ def save_sources(rules_dir: Path | None = None,
 # CRUD
 # ---------------------------------------------------------------------------
 
-def register_url_source(rules_dir: Path | None, rule_name: str, url: str) -> None:
-    """Add or update a URL source entry."""
+def register_url_source(
+    rules_dir: Path | None,
+    rule_name: str,
+    url: str,
+    content: bytes | None = None,
+) -> None:
+    """Add or update a URL source entry.
+
+    When ``content`` is supplied, its sha256 is persisted. If a previous
+    sha256 exists and differs from the new one, a warning is printed
+    (and the process fails with exit 2 when ``AI_TOOLKIT_STRICT_PIN=1``).
+    """
     import re
     if not rule_name or not re.fullmatch(r"[a-zA-Z0-9_-]+", rule_name):
         raise ValueError(f"Invalid rule name: {rule_name!r}")
     rules_dir = rules_dir or RULES_DIR
     sources = load_sources(rules_dir)
-    sources[rule_name] = {
+    entry: dict[str, Any] = {
         "url": url,
         "fetched_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
     }
+    if content is not None:
+        new_hash = hashlib.sha256(content).hexdigest()
+        prev = sources.get(rule_name) or {}
+        prev_hash = prev.get("sha256")
+        if prev_hash and prev_hash != new_hash:
+            msg = (
+                f"  CHECKSUM CHANGED: rule '{rule_name}' sha256 "
+                f"{prev_hash[:12]}... -> {new_hash[:12]}..."
+            )
+            print(msg)
+            if os.environ.get("AI_TOOLKIT_STRICT_PIN") == "1":
+                raise SystemExit(
+                    f"Refusing to update '{rule_name}' under AI_TOOLKIT_STRICT_PIN=1."
+                )
+        entry["sha256"] = new_hash
+    sources[rule_name] = entry
     save_sources(rules_dir, sources)